From 816e00519953c5186636d44f1ef8cf69aa3c1d84 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:25:32 +0400 Subject: [PATCH 01/33] Serve v2 metadata responses with an ETag and a cache lifetime The metadata routers now use CacheableMetadataRoute, which gives every successful response a strong ETag (a hash of its exact body, so any content or format change gives a new one) and Cache-Control public: a day when the request names its catalog version, which never changes, and five minutes for the running version, which changes on deploy. A matching If-None-Match (strong or weak, alone or in a list) gets 304 with no body. Failures carry neither header, and the bodies are unchanged. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/migration/stage-9-v2-metadata.md | 12 +++ .../fastapi_routes/v2/metadata/common.py | 88 +++++++++++++++++- .../v2/metadata/geography_routes.py | 7 +- .../v2/metadata/model_routes.py | 7 +- .../v2/metadata/parameter_routes.py | 7 +- tests/unit/v2/test_metadata_routes.py | 92 +++++++++++++++++++ 7 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md new file mode 100644 index 000000000..e6b747a86 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -0,0 +1 @@ +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0ad3eb455..fc3e1fc5f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,18 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Response caching + +Every successful metadata response carries a strong `ETag`, a hash of its +exact body, and `Cache-Control: public`. A request that names its catalog with +`policyengine_version` may be reused for a day (`max-age=86400`): a published +catalog version never changes, so only a change to the route's response format +can change such a response, and the `ETag` lets a client revalidate after the +day. A request without it selects the running version, which changes on +deploy, and may be reused for five minutes (`max-age=300`). A request whose +`If-None-Match` names the current `ETag` (strong or weak form, alone or in a +list) gets HTTP 304 with no body. Error responses carry neither header. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index fd4eaab5d..9cd6ff688 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,13 @@ from __future__ import annotations import contextlib +import hashlib from typing import TYPE_CHECKING, Any, TypeVar +from fastapi.routing import APIRoute from pydantic import BaseModel +from starlette.responses import Response +from starlette.status import HTTP_200_OK from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, @@ -24,8 +28,9 @@ ) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Coroutine + from fastapi import Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.dependencies import ( @@ -62,6 +67,71 @@ ResponseT = TypeVar("ResponseT", bound=BaseModel) +ReadT = TypeVar("ReadT") + +EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 +"""How long a response for a named catalog version may be reused (a day). + +A published catalog version never changes, so such a response stays valid +until its code's response format changes; the ETag lets a client revalidate +it cheaply after that. +""" +RUNNING_VERSION_MAX_AGE_SECONDS = 300 +"""How long a response for the running catalog version may be reused. + +Without ``policyengine_version`` the running PolicyEngine.py version answers, +and it changes whenever the API deploys a new one. +""" + + +class CacheableMetadataRoute(APIRoute): + """A metadata route whose successful responses carry caching headers. + + See ``with_cache_headers``. Failures are served unchanged, without them. + """ + + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + """Wrap FastAPI's handler so a 200 response gains its caching headers.""" + handle = super().get_route_handler() + + async def handle_with_cache_headers(request: Request) -> Response: + response = await handle(request) + if response.status_code != HTTP_200_OK: + return response + return with_cache_headers(request, response) + + return handle_with_cache_headers + + +def with_cache_headers(request: Request, response: Response) -> Response: + """Give a JSON response a strong ETag and a public cache lifetime. + + The ETag is a hash of the exact body, so it changes with any change to the + content, including the catalog format. A request naming its catalog with + ``policyengine_version`` may be reused for a day, one that takes the + running version for five minutes. A request whose ``If-None-Match`` names + the ETag gets a 304 with no body. + """ + entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' + max_age = ( + EXPLICIT_VERSION_MAX_AGE_SECONDS + if "policyengine_version" in request.query_params + else RUNNING_VERSION_MAX_AGE_SECONDS + ) + headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} + if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return response + + +def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: + if if_none_match is None: + return False + candidates = {candidate.strip() for candidate in if_none_match.split(",")} + return "*" in candidates or bool( + {entity_tag, f"W/{entity_tag}"}.intersection(candidates) + ) def read_resource( @@ -75,6 +145,20 @@ def read_resource( (400, 404 or 503, and 500 for anything unexpected), and the reader is closed whatever happens. """ + return run_metadata_read( + dependencies, lambda reader: response_type(result=operation(reader)) + ) + + +def run_metadata_read( + dependencies: NativeRouteDependencies, + operation: Callable[[V2MetadataResourceReader], ReadT], +) -> ReadT | JSONResponse: + """Run one metadata read and return its result unchanged. + + Failures become the typed v2 error responses described by read_resource, + and the reader is closed whatever happens. + """ reader = None try: factory = dependencies.v2_metadata_reader_factory @@ -85,7 +169,7 @@ def read_resource( factory = _default_v2_metadata_reader_factory reader = factory() - return response_type(result=operation(reader)) + return operation(reader) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 17ff62e27..f5add9640 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -36,7 +37,11 @@ def build_v2_metadata_geography_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/datasets", diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 927b52536..cbb3a8ef5 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -37,7 +38,11 @@ def build_v2_metadata_model_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/tax-benefit-models", diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 0f4714e5c..5895c8564 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -11,6 +11,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -49,7 +50,11 @@ def build_v2_metadata_parameter_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/parameters", diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..8f2084547 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -583,3 +583,95 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "application/json" ]["schema"] assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" + + +def _every_resource_path(ids: dict[str, object]) -> list[str]: + return [ + "/v2/tax-benefit-models?country_id=us", + "/v2/tax-benefit-models/by-country/us", + f"/v2/tax-benefit-models/{ids['model_id']}?country_id=us", + "/v2/tax-benefit-model-versions?country_id=us", + f"/v2/tax-benefit-model-versions/{ids['model_version_id']}?country_id=us", + "/v2/variables?country_id=us", + f"/v2/variables/{ids['variable_id']}?country_id=us", + "/v2/parameters?country_id=us", + "/v2/parameters/children?country_id=us&parent_path=gov.example", + f"/v2/parameters/{ids['parameter_id']}?country_id=us", + "/v2/parameter-values?country_id=us", + f"/v2/parameter-values/{ids['parameter_value_id']}?country_id=us", + "/v2/datasets?country_id=us", + f"/v2/datasets/{ids['dataset_id']}?country_id=us", + "/v2/regions?country_id=us", + "/v2/regions/by-code/us?country_id=us", + f"/v2/regions/{ids['region_id']}?country_id=us", + "/v2/economy-options?country_id=us", + ] + + +def test_every_resource_route_serves_an_entity_tag_and_cache_lifetime() -> None: + results, ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + + for path in _every_resource_path(ids): + response = client.get(path) + assert response.status_code == 200, path + assert response.headers["cache-control"] == "public, max-age=300", path + assert response.headers["etag"].startswith('"'), path + assert response.headers["etag"].endswith('"'), path + + +def test_a_named_catalog_version_is_reusable_for_a_day() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + query = {"country_id": "us", "policyengine_version": "4.20.3"} + + first = client.get("/v2/variables", params=query) + second = client.get("/v2/variables", params=query) + + assert first.headers["cache-control"] == "public, max-age=86400" + assert first.headers["etag"] == second.headers["etag"] + + +def test_a_matching_entity_tag_gets_a_not_modified_response_without_a_body() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + path = "/v2/variables?country_id=us&policyengine_version=4.20.3" + entity_tag = client.get(path).headers["etag"] + + revalidated = client.get(path, headers={"If-None-Match": entity_tag}) + weakly_revalidated = client.get(path, headers={"If-None-Match": f"W/{entity_tag}"}) + listed = client.get(path, headers={"If-None-Match": f'"other", {entity_tag}'}) + stale = client.get(path, headers={"If-None-Match": '"other"'}) + + for not_modified in (revalidated, weakly_revalidated, listed): + assert not_modified.status_code == 304 + assert not_modified.content == b"" + assert not_modified.headers["etag"] == entity_tag + assert not_modified.headers["cache-control"] == "public, max-age=86400" + assert stale.status_code == 200 + assert stale.json()["status"] == "ok" + + +def test_different_content_gets_a_different_entity_tag() -> None: + results, _ids = _resource_results() + changed_results, _changed_ids = _resource_results() + + first = _client(lambda: ResourceReader(results)).get("/v2/variables?country_id=us") + changed = _client(lambda: ResourceReader(changed_results)).get( + "/v2/variables?country_id=us" + ) + + assert first.headers["etag"] != changed.headers["etag"] + + +def test_failed_reads_carry_no_cache_headers() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get( + "/v2/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert response.status_code == 404 + assert "etag" not in response.headers + assert "cache-control" not in response.headers From 7af658237d5027ad8beee1727172ec42fc3c0966 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:33:08 +0400 Subject: [PATCH 02/33] Serve the small parts of a catalog version at /v2/model-catalog GET /v2/model-catalog returns one catalog version's model and model version, entities, basic inputs, every region and logical input dataset, and the economy choices, in three queries and without reading variables, parameters or their values. A catalog without regions, such as a household-only country's, gets empty region, dataset and choice lists rather than an error; a version published before catalog format 2 has a null catalog_format and empty entities and basic inputs. The result names the selected PolicyEngine.py version so a client can ask for the other parts by that exact version. The query uses the shared CatalogQuery model, the route joins the metadata route group and the app's metadata workflow contract, and the migration contract documents are regenerated. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 5 +- docs/generated/migration_contracts.json | 21 ++- docs/migration/stage-9-v2-metadata.md | 15 ++ .../fastapi_routes/dependencies.py | 15 ++ .../v2/metadata/catalog_routes.py | 67 +++++++++ .../v2/metadata/response_models.py | 7 + policyengine_api/fastapi_routes/v2/routes.py | 4 + policyengine_api/migration_registry.py | 1 + .../database_connectors/reads_datasets.py | 14 ++ .../services/v2/metadata/services.py | 50 +++++++ .../services/v2/metadata/transformations.py | 52 +++++++ .../services/v2/metadata/types.py | 45 ++++++ tests/contract/registry.py | 17 +++ tests/integration/test_v2_metadata_routes.py | 50 +++++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 104 +++++++++++++ tests/unit/v2/test_metadata_service.py | 140 +++++++++++++++++- 18 files changed, 605 insertions(+), 5 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index e6b747a86..e1d3d7b8b 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1 +1,2 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. +GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..c9fcbd3a3 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 89 | +| request count | 90 | | db entity count | 6 | | sim flow count | 3 | @@ -18,7 +18,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | @@ -215,6 +215,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/regions/by-code/state/ca?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.item` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..d1550bdc5 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 90, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -33,6 +33,7 @@ "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", @@ -1614,6 +1615,24 @@ "result.time_period", "result.datasets" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index fc3e1fc5f..cc2ae7b25 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,21 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Loading a catalog version on demand + +`GET /v2/model-catalog?country_id=[&policyengine_version=]` +returns only the small parts of one catalog version: the model and model +version (with the current-law policy id and the years offered), the entities +(`key`, `plural`, `is_person`), the basic inputs, every region and logical +input dataset, and the economy choices. A catalog without regions, such as a +household-only country's, has empty region, dataset and economy-choice lists +rather than an error. A version published before catalog format 2 has a null +`catalog_format` and empty entities and basic inputs. The result names the +selected PolicyEngine.py version, so a client that starts without a version +learns the running one and requests every other part for that exact version. +The query uses the shared `CatalogQuery` model: unknown and repeated +parameters are rejected. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..59745fec8 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -38,6 +38,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -311,6 +312,20 @@ def get_economy_options( """ ... + def get_model_catalog( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + These are the model and model version, entities, basic inputs, regions, + logical input datasets and economy choices, which are empty for a + catalog without regions. Raise ``MetadataCatalogUnavailableError`` when + a catalog with regions lacks what ``get_economy_options`` needs. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py new file mode 100644 index 000000000..2c73c44d1 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -0,0 +1,67 @@ +"""API v2 routes that serve a catalog version in parts loaded on demand.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import APIRouter, Depends +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.metadata.common import ( + ERROR_RESPONSES, + CacheableMetadataRoute, + read_resource, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataModelCatalogResponse, +) +from policyengine_api.query_parameters import CatalogQuery + +if TYPE_CHECKING: + from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] + + +def build_v2_model_catalog_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the routes that serve one catalog version in parts. + + Each request opens and closes its own metadata reader, so building the + router touches no database. + """ + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) + + @router.get( + "/model-catalog", + response_model=MetadataModelCatalogResponse, + summary="Get the small parts of one PolicyEngine.py catalog version", + description=( + "Returns the selected catalog version's model, model version, " + "entities, basic inputs, regions, logical input datasets and " + "economy choices, which are empty lists for a catalog without " + "regions. The result names the selected PolicyEngine.py version, " + "so a client can request the version's other parts by that exact " + "version; a response for a named version may be reused for a day." + ), + ) + def get_model_catalog( + query: CatalogQueryDependency, + ) -> MetadataModelCatalogResponse | JSONResponse: + return read_resource( + dependencies, + MetadataModelCatalogResponse, + lambda reader: reader.get_model_catalog( + query.country_id, query.policyengine_version + ), + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index d53660834..be095bb8a 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -10,6 +10,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -139,3 +140,9 @@ class MetadataEconomyOptionsResponse( MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] ): """Success response holding the options for configuring an economy comparison.""" + + +class MetadataModelCatalogResponse( + MetadataResourceSuccessResponse[MetadataModelCatalogResult] +): + """Success response holding the small parts of one catalog version.""" diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..b55b57436 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -12,6 +12,9 @@ build_v2_household_router, ) from policyengine_api.fastapi_routes.v2.me.routes import build_v2_me_router +from policyengine_api.fastapi_routes.v2.metadata.catalog_routes import ( + build_v2_model_catalog_router, +) from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) @@ -56,6 +59,7 @@ def build_v2_router( router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) + router.include_router(build_v2_model_catalog_router(dependencies)) @router.get( "/v2/openapi.json", diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..25fd541f6 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -39,6 +39,7 @@ class RouteGroupConfig: "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py index cd5dbc276..b6d79b6fa 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -48,3 +48,17 @@ def read_dataset( ), ) return rows[0] if rows else None + + +def read_input_datasets(session: Session, *, model_version_id: UUID) -> list[Dataset]: + """Return every logical input dataset of a model version in name order.""" + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)), + ) diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 938eb660e..199b12683 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -14,6 +14,7 @@ from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( read_dataset, read_datasets, + read_input_datasets, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, @@ -37,6 +38,7 @@ metadata_dataset, metadata_economy_options, metadata_model, + metadata_model_catalog, metadata_model_selection, metadata_model_version, metadata_parameter, @@ -53,6 +55,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -541,3 +544,50 @@ def get_economy_options( return metadata_economy_options( selected, regions=regions, national_dataset=dataset ) + + def get_model_catalog( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + They are the model and model version, entities, basic inputs, every + region and logical input dataset, and the economy choices. A catalog + without regions offers no economy calculation, so its choices are + empty; one with regions raises MetadataCatalogUnavailableError when it + lacks the national region, its default dataset or the model-version + options, as get_economy_options does. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + datasets = read_input_datasets( + session, model_version_id=selected.model_version.id + ) + national_dataset = None + if regions: + national_region = next( + (region for region in regions if region.code == country_id), None + ) + national_dataset = validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + next( + ( + dataset + for dataset in datasets + if dataset.id == national_region.default_dataset_id + ), + None, + ) + if national_region is not None + else None + ), + ) + return metadata_model_catalog( + selected, + regions=regions, + datasets=datasets, + national_dataset=national_dataset, + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 68ebe21fe..ce8eb7bd2 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -10,8 +10,11 @@ MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -221,9 +224,22 @@ def metadata_economy_options( The national dataset is the only dataset choice, and it is the default. """ + options = _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) return MetadataEconomyOptionsResult( policyengine_version=selected.policyengine_version, current_law_id=selected.model_version.current_law_id, + region=options.region, + time_period=options.time_period, + datasets=options.datasets, + ) + + +def _economy_option_lists( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptions: + return MetadataEconomyOptions( region=[ MetadataRegionOption( name=region.code, @@ -243,3 +259,39 @@ def metadata_economy_options( ) ], ) + + +def metadata_model_catalog( + selected: SelectedCatalog, + *, + regions: list[Region], + datasets: list[Dataset], + national_dataset: Dataset | None, +) -> MetadataModelCatalogResult: + """Build the small parts of the selected catalog version. + + Without a national dataset (a catalog without regions) the economy choices + are empty. Entities and basic inputs a version published before catalog + format 2 lacks are empty lists. + """ + model_version = selected.model_version + return MetadataModelCatalogResult( + policyengine_version=selected.policyengine_version, + catalog_format=model_version.catalog_format, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + entities=[ + MetadataEntity.model_validate(entity) + for entity in model_version.entities or [] + ], + basic_inputs=list(model_version.basic_inputs or []), + regions=[metadata_region(region) for region in regions], + datasets=[metadata_dataset(dataset) for dataset in datasets], + economy_options=( + _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) + if national_dataset is not None + else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) + ), + ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index a7336d7e9..e810a663b 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -239,3 +239,48 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] + + +class MetadataEntity(StrictResponseModel): + """A kind of member a household holds, such as a person or a tax unit. + + plural is the key the household's members of this kind are listed under, + and is_person marks the entity every household input about a person uses. + """ + + key: str + plural: str + is_person: bool + + +class MetadataEconomyOptions(StrictResponseModel): + """The region, year and dataset choices for an economy calculation. + + Every list is empty for a catalog without regions, such as a + household-only country's, which offers no economy calculation. + """ + + region: list[MetadataRegionOption] + time_period: list[MetadataTimePeriodOption] + datasets: list[MetadataDatasetOption] + + +class MetadataModelCatalogResult(StrictResponseModel): + """The small parts of one catalog version, read when a client starts. + + policyengine_version names the selected catalog version, so a client can + request its other parts for that exact version. catalog_format is null, + and entities and basic_inputs are empty, for a version published before + catalog format 2. Variables, parameters and their values are served by + their own routes. + """ + + policyengine_version: str + catalog_format: int | None + model: MetadataModel + model_version: MetadataModelVersionDetail + entities: list[MetadataEntity] + basic_inputs: list[str] + regions: list[MetadataRegion] + datasets: list[MetadataDataset] + economy_options: MetadataEconomyOptions diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..c1fbe02e8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -671,6 +671,23 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 326eb78d9..efeb702b5 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -360,3 +360,53 @@ def test_postgres_parameter_collection_remains_available_without_values( assert parameters.status_code == values.status_code == 200 assert parameters.json()["result"]["items"] assert values.json()["result"]["items"] == [] + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_postgres_model_catalog_serves_the_small_parts_of_a_version( + published_engine: Engine, + country_id: str, +) -> None: + client = _client(published_engine) + country = normalized_catalog().country(country_id) + + running = client.get("/v2/model-catalog", params={"country_id": country_id}) + named = client.get( + "/v2/model-catalog", + params={"country_id": country_id, "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert running.status_code == 200 + assert running.headers["cache-control"] == "public, max-age=300" + assert named.headers["cache-control"] == "public, max-age=86400" + assert running.headers["etag"] == named.headers["etag"] + result = running.json()["result"] + assert result["policyengine_version"] == POLICYENGINE_VERSION + assert result["catalog_format"] == country.model_version.catalog_format + assert result["entities"] == [ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ] + assert result["basic_inputs"] == list(country.model_version.basic_inputs) + assert [region["code"] for region in result["regions"]] == sorted( + region.code for region in country.regions + ) + assert [dataset["name"] for dataset in result["datasets"]] == sorted( + dataset.name for dataset in country.datasets + ) + assert [option["name"] for option in result["economy_options"]["region"]] == [ + region["code"] for region in result["regions"] + ] + assert "variables" not in result + assert "parameters" not in result + + revalidated = client.get( + "/v2/model-catalog", + params={"country_id": country_id}, + headers={"If-None-Match": running.headers["etag"]}, + ) + assert revalidated.status_code == 304 diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..4125c6dcb 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 89, + "request_count": 90, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 8f2084547..81ff8669e 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -29,8 +29,11 @@ MetadataDataset, MetadataDatasetOption, MetadataDetailResult, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -193,6 +196,31 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_model_catalog": MetadataModelCatalogResult( + policyengine_version=version, + catalog_format=2, + model=model, + model_version=model_version, + entities=[ + MetadataEntity(key="person", plural="people", is_person=True) + ], + basic_inputs=["employment_income"], + regions=[region], + datasets=[dataset], + economy_options=MetadataEconomyOptions( + region=[ + MetadataRegionOption( + name="us", label="United States", type="national" + ) + ], + time_period=[MetadataTimePeriodOption(name=2026, label="2026")], + datasets=[ + MetadataDatasetOption( + name="populace_us_2024", label="Microcosm" + ) + ], + ), + ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, current_law_id=2, @@ -521,6 +549,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", + "/v2/model-catalog", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -605,6 +634,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: "/v2/regions/by-code/us?country_id=us", f"/v2/regions/{ids['region_id']}?country_id=us", "/v2/economy-options?country_id=us", + "/v2/model-catalog?country_id=us", ] @@ -675,3 +705,77 @@ def test_failed_reads_carry_no_cache_headers() -> None: assert response.status_code == 404 assert "etag" not in response.headers assert "cache-control" not in response.headers + + +def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", params={"country_id": "US"} + ) + + assert response.status_code == 200 + result = response.json()["result"] + assert result["policyengine_version"] == "4.20.3" + assert result["catalog_format"] == 2 + assert result["entities"] == [ + {"key": "person", "plural": "people", "is_person": True} + ] + assert result["basic_inputs"] == ["employment_income"] + assert [region["code"] for region in result["regions"]] == ["us"] + assert result["economy_options"]["datasets"][0]["default"] is True + assert reader.calls == [("get_model_catalog", ("us", None), {})] + assert reader.closed + + +def test_model_catalog_route_names_an_explicit_version_for_a_day() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", + params={"country_id": "uk", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + assert reader.calls == [("get_model_catalog", ("uk", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + [ + "", + "country_id=ca", + "country_id=us&country_id=uk", + "country_id=us&limit=10", + "country_id=us&policyengine_version=latest", + ], +) +def test_model_catalog_route_rejects_invalid_queries(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog?{query}") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert calls == [] + + +def test_model_catalog_route_declares_its_query_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version"} + assert parameters["country_id"]["required"] is True + assert parameters["policyengine_version"]["required"] is False + success = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 1f80d1b3f..996ba1ef5 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -34,7 +34,10 @@ MetadataDatabaseSession, ) from policyengine_api.services.v2.metadata.services import V2MetadataService -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.types import ( + MetadataEntity, + ParameterValueFilter, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -74,6 +77,16 @@ def _insert_country( if time_periods is None else time_periods ), + catalog_format=country.model_version.catalog_format, + entities=[ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ], + basic_inputs=list(country.model_version.basic_inputs), ) ) session.add_all( @@ -89,6 +102,19 @@ def _insert_country( default_value=record.default_value, adds=record.adds, subtracts=record.subtracts, + is_input_variable=record.is_input_variable, + hidden_input=record.hidden_input, + module_name=record.module_name, + adds_parameter_path=record.adds_parameter_path, + subtracts_parameter_path=record.subtracts_parameter_path, + enum_values=( + None + if record.enum_values is None + else [ + {"value": enum_value.value, "label": enum_value.label} + for enum_value in record.enum_values + ] + ), ) for record in country.variables ) @@ -111,6 +137,8 @@ def _insert_country( description=record.description, data_type=record.data_type, unit=record.unit, + economy=record.economy, + household=record.household, ) for record in country.parameters ) @@ -560,6 +588,115 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") +def _recorded_statements(session: Session, read): + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement.lower()) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record_statement) + try: + result = read() + finally: + event.remove(bind, "before_cursor_execute", record_statement) + return result, statements + + +def test_model_catalog_holds_only_the_small_parts_of_a_catalog_version( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, lambda: _service(catalog_session).get_model_catalog("us") + ) + us_catalog = normalized_catalog().country("us") + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert result.model.name == "policyengine-us" + assert ( + result.model_version.current_law_id == us_catalog.model_version.current_law_id + ) + assert result.entities == [ + MetadataEntity(key="person", plural="people", is_person=True), + MetadataEntity(key="household", plural="households", is_person=False), + ] + assert result.basic_inputs == ["employment_income"] + assert [region.code for region in result.regions] == sorted( + region.code for region in us_catalog.regions + ) + assert [dataset.name for dataset in result.datasets] == sorted( + dataset.name for dataset in us_catalog.datasets + ) + assert [option.name for option in result.economy_options.region] == [ + region.code for region in result.regions + ] + assert [option.name for option in result.economy_options.time_period] == list( + us_catalog.model_version.metadata_time_periods + ) + assert [option.label for option in result.economy_options.datasets] == ["Microcosm"] + assert len(statements) == 3 + for table in ("variables", "parameters", "parameter_values", "parameter_nodes"): + assert all(f"from {table}" not in statement for statement in statements) + + +def test_model_catalog_without_regions_offers_no_economy_calculation( + catalog_session: Session, +) -> None: + # A household-only country publishes no regions or datasets; its catalog + # still serves entities and basic inputs, with empty economy choices. + model_version = _us_model_version(catalog_session) + for region in catalog_session.exec( + select(Region).where(Region.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(region) + catalog_session.flush() + for dataset in catalog_session.exec( + select(Dataset).where(Dataset.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(dataset) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.regions == [] + assert result.datasets == [] + assert result.economy_options.region == [] + assert result.economy_options.time_period == [] + assert result.economy_options.datasets == [] + assert result.basic_inputs == ["employment_income"] + + +def test_model_catalog_published_before_format_2_has_empty_new_fields( + catalog_session: Session, +) -> None: + model_version = _us_model_version(catalog_session) + model_version.catalog_format = None + model_version.entities = None + model_version.basic_inputs = None + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.catalog_format is None + assert result.entities == [] + assert result.basic_inputs == [] + + +def test_model_catalog_with_regions_requires_the_national_region( + catalog_session: Session, +) -> None: + national_region = catalog_session.exec( + select(Region).where(Region.code == "us") + ).one() + catalog_session.delete(national_region) + catalog_session.commit() + + with pytest.raises(MetadataCatalogUnavailableError, match="national v2 region"): + _service(catalog_session).get_model_catalog("us") + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -624,6 +761,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region", "get_region_by_code", "get_economy_options", + "get_model_catalog", } for method_name in method_names: From 290b042df403cf0c2395b8392f1f7de7cea6014e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:40:25 +0400 Subject: [PATCH 03/33] Serve every variable of a catalog version with its part parameters' values GET /v2/model-catalog/variables returns all of a catalog version's variables in name order, in one response, and adds the catalog format 2 fields (input and hidden-input flags, module, adds and subtracts parameter paths, and enum values with their labels) to every variable response, including /v2/variables pages. part_parameter_values maps each parameter path a variable's adds or subtracts names to that parameter's current-law values in start-date order, read in one query (none when no variable names one), so a client picks the list for the year it shows; a path the version lacks maps to an empty list. Times are served in UTC whatever the database session's time zone, so a response's bytes and ETag do not depend on it. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 3 +- docs/generated/migration_contracts.json | 17 +- docs/migration/stage-9-v2-metadata.md | 12 ++ .../fastapi_routes/dependencies.py | 13 ++ .../v2/metadata/catalog_routes.py | 24 +++ .../v2/metadata/response_models.py | 7 + .../database_connectors/reads_parameters.py | 33 +++- .../database_connectors/reads_variables.py | 10 ++ .../services/v2/metadata/services.py | 27 ++++ .../services/v2/metadata/transformations.py | 66 ++++++++ .../services/v2/metadata/types.py | 50 +++++- tests/contract/registry.py | 14 ++ tests/integration/test_v2_metadata_routes.py | 98 ++++++++++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 62 ++++++++ tests/unit/v2/test_metadata_service.py | 147 ++++++++++++++++++ 17 files changed, 581 insertions(+), 5 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index e1d3d7b8b..5e897c7ed 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,2 +1,3 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index c9fcbd3a3..33212c71d 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 90 | +| request count | 91 | | db entity count | 6 | | sim flow count | 3 | @@ -216,6 +216,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | | `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | +| `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index d1550bdc5..74592e7fc 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 90, + "request_count": 91, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -1633,6 +1633,21 @@ "result.datasets", "result.economy_options" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/variables?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index cc2ae7b25..ed8456fc4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -42,6 +42,18 @@ learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated parameters are rejected. +`GET /v2/model-catalog/variables` (same query) returns every variable of the +version in name order, in one response, with the catalog format 2 fields: +`is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, +`subtracts_parameter_path` and `enum_values` (each value's name and its label, +in the type the country package declares). Where a variable's `adds` or +`subtracts` list comes from a parameter, `part_parameter_values` maps that +parameter's path to its current-law values in start-date order +(`start_date`, `end_date`, `value`), so a client picks the list for the year +it shows instead of the list resolved on 2025-01-01; a path whose parameter +the version lacks maps to an empty list. `GET /v2/variables` pages carry the +same variable fields. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 59745fec8..7de4f62e0 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -34,6 +34,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -326,6 +327,18 @@ def get_model_catalog( """ ... + def get_catalog_variables( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also carries the current-law values of each parameter a variable's + adds or subtracts names. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 2c73c44d1..a648ece21 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -14,6 +14,7 @@ read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, ) from policyengine_api.query_parameters import CatalogQuery @@ -64,4 +65,27 @@ def get_model_catalog( ), ) + @router.get( + "/model-catalog/variables", + response_model=MetadataCatalogVariablesResponse, + summary="Get every variable of one PolicyEngine.py catalog version", + description=( + "Returns all of the selected catalog version's variables in name " + "order, in one response. Where a variable's adds or subtracts list " + "comes from a parameter, part_parameter_values carries that " + "parameter's current-law values by date, so a client can pick the " + "list for any year." + ), + ) + def get_catalog_variables( + query: CatalogQueryDependency, + ) -> MetadataCatalogVariablesResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogVariablesResponse, + lambda reader: reader.get_catalog_variables( + query.country_id, query.policyengine_version + ), + ) + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index be095bb8a..56791d2bb 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,6 +6,7 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -146,3 +147,9 @@ class MetadataModelCatalogResponse( MetadataResourceSuccessResponse[MetadataModelCatalogResult] ): """Success response holding the small parts of one catalog version.""" + + +class MetadataCatalogVariablesResponse( + MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] +): + """Success response holding every variable of one catalog version.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 02062798a..739b6b0cb 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import sqlalchemy as sa from sqlmodel import Session, col, select @@ -11,6 +11,7 @@ from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows if TYPE_CHECKING: + from collections.abc import Collection from uuid import UUID from policyengine_api.services.v2.metadata.types import ParameterValueSelection @@ -131,3 +132,33 @@ def read_parameter_value( ), ) return rows[0] if rows else None + + +def read_current_law_values_by_parameter_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Any]: + """Return the current-law values of the named parameters of a model version. + + Each row has the parameter's name, the value's value_json, start_date and + end_date. Values tied to a policy or dynamic are excluded. Rows are ordered + by parameter name, then start date. + """ + if not parameter_names: + return [] + return read_rows( + session, + select( + col(Parameter.name), + col(ParameterValue.value_json), + col(ParameterValue.start_date), + col(ParameterValue.end_date), + ) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + .order_by(col(Parameter.name), col(ParameterValue.start_date)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py index 4d84be396..1378f6528 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -62,3 +62,13 @@ def read_variable( ), ) return rows[0] if rows else None + + +def read_all_variables(session: Session, *, model_version_id: UUID) -> list[Variable]: + """Return every variable of a model version in name order.""" + return read_rows( + session, + select(Variable) + .where(col(Variable.tax_benefit_model_version_id) == model_version_id) + .order_by(col(Variable.name)), + ) diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 199b12683..e6d4506c7 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -20,6 +20,7 @@ read_parameter_children, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_current_law_values_by_parameter_name, read_parameter, read_parameter_value, read_parameter_values, @@ -31,10 +32,12 @@ read_regions, ) from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_all_variables, read_variable, read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + metadata_catalog_variables, metadata_dataset, metadata_economy_options, metadata_model, @@ -48,9 +51,11 @@ metadata_variable, page_result, parameter_value_selection, + part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -591,3 +596,25 @@ def get_model_catalog( datasets=datasets, national_dataset=national_dataset, ) + + def get_catalog_variables( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also returns the current-law values of each parameter a variable's + adds or subtracts names, so a client can pick the list for any year. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + variables = read_all_variables( + session, model_version_id=selected.model_version.id + ) + part_values = read_current_law_values_by_parameter_name( + session, + model_version_id=selected.model_version.id, + parameter_names=part_parameter_paths(variables), + ) + return metadata_catalog_variables( + selected, variables=variables, part_values=part_values + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index ce8eb7bd2..651f26414 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,11 +8,14 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataEconomyOptions, MetadataEconomyOptionsResult, MetadataEntity, + MetadataEnumValue, MetadataModel, MetadataModelCatalogResult, MetadataModelSelectionResult, @@ -106,6 +109,57 @@ def metadata_variable(variable: Variable) -> MetadataVariable: default_value=variable.default_value, adds=variable.adds, subtracts=variable.subtracts, + is_input_variable=variable.is_input_variable, + hidden_input=variable.hidden_input, + module_name=variable.module_name, + adds_parameter_path=variable.adds_parameter_path, + subtracts_parameter_path=variable.subtracts_parameter_path, + enum_values=( + None + if variable.enum_values is None + else [ + MetadataEnumValue.model_validate(enum_value) + for enum_value in variable.enum_values + ] + ), + ) + + +def part_parameter_paths(variables: list[Variable]) -> set[str]: + """Return every parameter path the variables' adds or subtracts names.""" + return { + path + for variable in variables + for path in (variable.adds_parameter_path, variable.subtracts_parameter_path) + if path is not None + } + + +def metadata_catalog_variables( + selected: SelectedCatalog, *, variables: list[Variable], part_values: list[Any] +) -> MetadataCatalogVariablesResult: + """Build every variable of the selected version and its part parameters' values. + + part_values holds rows of read_current_law_values_by_parameter_name, in + parameter-name and start-date order. Each path a variable names gets an + entry, empty when no row carries its name. + """ + values_by_path: dict[str, list[MetadataDatedValue]] = { + path: [] for path in sorted(part_parameter_paths(variables)) + } + for row in part_values: + values_by_path.setdefault(row.name, []).append( + MetadataDatedValue( + start_date=as_utc(row.start_date), + end_date=None if row.end_date is None else as_utc(row.end_date), + value=row.value_json, + ) + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, ) @@ -132,6 +186,18 @@ def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParamete ) +def as_utc(moment: datetime) -> datetime: + """Return moment in UTC, treating a naive time as UTC. + + The database may return times in its session's time zone (or, in SQLite, + without one); serving them in UTC keeps a response's bytes, and so its + ETag, the same whatever that setting is. + """ + if moment.tzinfo is None: + return moment.replace(tzinfo=UTC) + return moment.astimezone(UTC) + + def utc_day_start(selected_time: datetime) -> datetime: """Return midnight UTC on the day of selected_time, treating a naive time as UTC.""" if selected_time.tzinfo is None: diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index e810a663b..d6633eeaf 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -71,11 +71,26 @@ class MetadataModelVersion(StrictResponseModel): description: str | None +class MetadataEnumValue(StrictResponseModel): + """One value of an enum variable: its stored name and its display label. + + The label keeps the type the country package declares it with. + """ + + value: str + label: str | int | float | bool | None + + class MetadataVariable(StrictResponseModel): """A variable the tax-benefit model computes or accepts as input. adds and subtracts, when present, list the variables summed into or - subtracted from this one. + subtracted from this one. When the model takes that list from a + parameter, adds_parameter_path or subtracts_parameter_path names the + parameter, whose dated values give the list for each year (adds and + subtracts hold its value on 2025-01-01). The catalog format 2 fields + (is_input_variable, hidden_input, module_name, the two parameter paths + and enum_values) are null in versions published before that format. """ id: UUID @@ -88,6 +103,12 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None = None + hidden_input: bool | None = None + module_name: str | None = None + adds_parameter_path: str | None = None + subtracts_parameter_path: str | None = None + enum_values: list[MetadataEnumValue] | None = None class MetadataParameterSummary(StrictResponseModel): @@ -284,3 +305,30 @@ class MetadataModelCatalogResult(StrictResponseModel): regions: list[MetadataRegion] datasets: list[MetadataDataset] economy_options: MetadataEconomyOptions + + +class MetadataDatedValue(StrictResponseModel): + """A current-law parameter value from start_date until end_date. + + A null end_date means the value applies until a later value starts, or + indefinitely. + """ + + start_date: datetime + end_date: datetime | None + value: JsonValue + + +class MetadataCatalogVariablesResult(StrictResponseModel): + """Every variable of one catalog version, in name order. + + part_parameter_values has an entry for each parameter path a variable's + adds_parameter_path or subtracts_parameter_path names: that parameter's + current-law values in start-date order, or an empty list when the + catalog has no such parameter. + """ + + policyengine_version: str + catalog_format: int | None + variables: list[MetadataVariable] + part_parameter_values: dict[str, list[MetadataDatedValue]] diff --git a/tests/contract/registry.py b/tests/contract/registry.py index c1fbe02e8..b126a838e 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -688,6 +688,20 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog/variables?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index efeb702b5..68a44ebda 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 @@ -20,8 +21,11 @@ from policyengine_api.data.v2.catalog.publication import publish_catalog from policyengine_api.data.v2.models import ( Dataset, + Parameter, + ParameterValue, TaxBenefitModel, TaxBenefitModelVersion, + Variable, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -410,3 +414,97 @@ def test_postgres_model_catalog_serves_the_small_parts_of_a_version( headers={"If-None-Match": running.headers["etag"]}, ) assert revalidated.status_code == 304 + + +def _us_model_version_id(engine: Engine): + with Session(engine) as session: + return session.exec( + select(TaxBenefitModelVersion.id) + .join( + TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id + ) + .where(TaxBenefitModel.name == "policyengine-us") + ).one() + + +def test_postgres_catalog_variables_carry_part_parameter_values_by_date( + published_engine: Engine, +) -> None: + model_version_id = _us_model_version_id(published_engine) + parts_id = uuid4() + with Session(published_engine) as session: + session.add( + Parameter( + id=parts_id, + tax_benefit_model_version_id=model_version_id, + name="gov.example.parts", + label="Income parts", + economy=False, + household=True, + ) + ) + session.flush() + session.add_all( + [ + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + ), + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + ), + Variable( + tax_benefit_model_version_id=model_version_id, + name="total_income", + entity="person", + data_type="float", + default_value=0, + adds=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + ), + ] + ) + session.commit() + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income", + "total_income", + ] + assert result["variables"][0]["module_name"] == ( + "household.income.employment_income" + ) + assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2024-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + }, + { + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income", "pension_income"], + }, + ] + } + uk = client.get("/v2/model-catalog/variables", params={"country_id": "uk"}) + assert [variable["name"] for variable in uk.json()["result"]["variables"]] == [ + "employment_income" + ] + assert uk.json()["result"]["part_parameter_values"] == {} diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 4125c6dcb..a52730e6e 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 90, + "request_count": 91, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 81ff8669e..e522a114b 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -26,8 +26,10 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataDetailResult, MetadataEconomyOptions, MetadataEconomyOptionsResult, @@ -196,6 +198,20 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_variables": MetadataCatalogVariablesResult( + policyengine_version=version, + catalog_format=2, + variables=[variable], + part_parameter_values={ + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income"], + ) + ] + }, + ), "get_model_catalog": MetadataModelCatalogResult( policyengine_version=version, catalog_format=2, @@ -550,6 +566,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets/{dataset_id}", "/v2/economy-options", "/v2/model-catalog", + "/v2/model-catalog/variables", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -635,6 +652,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: f"/v2/regions/{ids['region_id']}?country_id=us", "/v2/economy-options?country_id=us", "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", ] @@ -779,3 +797,47 @@ def test_model_catalog_route_declares_its_query_in_openapi() -> None: assert parameters["policyengine_version"]["required"] is False success = operation["responses"]["200"]["content"]["application/json"]["schema"] assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} + + +def test_catalog_variables_route_serves_every_variable_and_part_values() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["catalog_format"] == 2 + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income" + ] + assert result["variables"][0]["module_name"] is None + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + } + ] + } + assert reader.calls == [("get_catalog_variables", ("us", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + ["", "country_id=us&offset=0", "country_id=us&search=income"], +) +def test_catalog_variables_route_rejects_paging_and_unknown_queries( + query: str, +) -> None: + results, _ids = _resource_results() + response = _client(lambda: ResourceReader(results)).get( + f"/v2/model-catalog/variables?{query}" + ) + + assert response.status_code == 422 diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 996ba1ef5..d30ba98b8 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -35,7 +35,9 @@ ) from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.types import ( + MetadataDatedValue, MetadataEntity, + MetadataEnumValue, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -697,6 +699,150 @@ def test_model_catalog_with_regions_requires_the_national_region( _service(catalog_session).get_model_catalog("us") +def _add_variable_with_parameter_named_parts(session: Session) -> None: + model_version = _us_model_version(session) + parts = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.parts", + label="Income parts", + description=None, + data_type="list", + unit=None, + economy=False, + household=True, + ) + session.add(parts) + session.add_all( + [ + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["reform_only_income"], + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ), + ] + ) + session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="total_income", + label="Total income", + entity="person", + description=None, + data_type="float", + possible_values=["LOW", "HIGH"], + default_value=0, + adds=["employment_income"], + subtracts=None, + is_input_variable=False, + hidden_input=True, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + subtracts_parameter_path="gov.example.unpublished_parts", + enum_values=[ + {"value": "LOW", "label": "Low"}, + {"value": "HIGH", "label": 2}, + ], + ) + ) + session.commit() + + +def test_catalog_variables_hold_every_variable_with_its_format_2_fields( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("us"), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [variable.name for variable in result.variables] == [ + "employment_income", + "total_income", + ] + employment_income, total_income = result.variables + assert employment_income.is_input_variable is True + assert employment_income.hidden_input is False + assert employment_income.module_name == "household.income.employment_income" + assert employment_income.adds_parameter_path is None + assert total_income.hidden_input is True + assert total_income.adds_parameter_path == "gov.example.parts" + assert total_income.subtracts_parameter_path == "gov.example.unpublished_parts" + assert total_income.enum_values == [ + MetadataEnumValue(value="LOW", label="Low"), + MetadataEnumValue(value="HIGH", label=2), + ] + assert len(statements) == 3 + + +def test_catalog_variables_carry_the_dated_values_of_parameter_named_parts( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result = _service(catalog_session).get_catalog_variables("us") + + assert result.part_parameter_values == { + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + value=["employment_income"], + ), + MetadataDatedValue( + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income", "pension_income"], + ), + ], + "gov.example.unpublished_parts": [], + } + + +def test_catalog_variables_without_parameter_named_parts_read_no_values( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("uk"), + ) + + assert [variable.name for variable in result.variables] == ["employment_income"] + assert result.part_parameter_values == {} + assert len(statements) == 2 + assert all("parameter_values" not in statement for statement in statements) + + +def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> None: + variable = _service(catalog_session).list_variables("us").items[0] + + assert variable.is_input_variable is True + assert variable.module_name == "household.income.employment_income" + assert variable.enum_values is None + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -762,6 +908,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region_by_code", "get_economy_options", "get_model_catalog", + "get_catalog_variables", } for method_name in method_names: From adff8a2560ae106cb4c68e97c1f0ad320db16624 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:53:39 +0400 Subject: [PATCH 04/33] Fetch several parameters by name with their values and breadcrumb labels GET /v2/model-catalog/parameters takes up to 100 repeated name keys (the shared CatalogParameterQuery) and returns those parameters in name order, each with its economy and household flags and its current-law values by date, the stored labels of the parameter nodes above them (a bracket index such as rates[0] counting as a child of its schedule rates), and the requested names the version lacks, in four queries on the existing (version, name) unique indexes. A client showing or editing a reform loads all of its parameters in this one request. The economy and household flags also join every parameter summary, so /v2/parameters pages and details carry them. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 3 +- docs/generated/migration_contracts.json | 18 ++- docs/migration/stage-9-v2-metadata.md | 12 ++ .../fastapi_routes/dependencies.py | 17 ++- .../v2/metadata/catalog_routes.py | 30 +++- .../v2/metadata/response_models.py | 7 + policyengine_api/query_parameters.py | 17 +++ .../reads_parameter_nodes.py | 34 +++++ .../database_connectors/reads_parameters.py | 20 +++ .../services/v2/metadata/services.py | 47 ++++++ .../services/v2/metadata/transformations.py | 84 +++++++++-- .../services/v2/metadata/types.py | 29 +++- tests/contract/registry.py | 18 +++ tests/integration/test_v2_metadata_routes.py | 35 +++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 87 +++++++++++ tests/unit/v2/test_metadata_service.py | 139 ++++++++++++++++++ 18 files changed, 584 insertions(+), 16 deletions(-) create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 5e897c7ed..1e4e961ec 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,3 +1,4 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 33212c71d..411656a62 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 91 | +| request count | 92 | | db entity count | 6 | | sim flow count | 3 | @@ -217,6 +217,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | | `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | | `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | +| `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 74592e7fc..9232097e8 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 91, + "request_count": 92, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -1648,6 +1648,22 @@ "result.variables", "result.part_parameter_values" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index ed8456fc4..a5aaaba4c 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -54,6 +54,18 @@ it shows instead of the list resolved on 2025-01-01; a path whose parameter the version lacks maps to an empty list. `GET /v2/variables` pages carry the same variable fields. +`GET /v2/model-catalog/parameters` takes the same query plus one or more +`name` keys (at most 100, each an exact parameter path) and returns those +parameters in name order, each with its `economy` and `household` flags and its +current-law values in start-date order, in one response: a client showing or +editing a reform loads all of its parameters at once. `node_labels` maps each +ancestor of a returned parameter that the version stores as a parameter node +to its stored label (null when it has none), for breadcrumbs; a bracket index +such as `rates[0]` counts as a child of its schedule `rates`. `missing_names` +lists requested names the version has no parameter for, so a reform saved +against an older version still opens. `GET /v2/parameters` pages and details +carry the same `economy` and `household` flags. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 7de4f62e0..6631c8630 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -15,7 +15,7 @@ from policyengine_api.readiness import is_ready if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from uuid import UUID from policyengine_api.json_types import JSONObject @@ -34,6 +34,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -339,6 +340,20 @@ def get_catalog_variables( """ ... + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values; the result also carries the + stored labels of their ancestor nodes and the names the version lacks. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index a648ece21..8d91e8630 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -14,10 +14,11 @@ read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, ) -from policyengine_api.query_parameters import CatalogQuery +from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -25,6 +26,9 @@ CatalogQueryDependency = Annotated[ CatalogQuery, Depends(query_dependency(CatalogQuery)) ] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] def build_v2_model_catalog_router( @@ -88,4 +92,28 @@ def get_catalog_variables( ), ) + @router.get( + "/model-catalog/parameters", + response_model=MetadataCatalogParametersResponse, + summary="Get several parameters of one catalog version by name", + description=( + "Returns the named parameters (repeat name for each, at most 100) " + "in name order, each with its economy and household flags and its " + "current-law values by date, plus the stored labels of their " + "ancestor nodes for breadcrumbs and the requested names the " + "version has no parameter for. A client showing or editing a " + "reform loads all of its parameters in this one request." + ), + ) + def get_catalog_parameters( + query: CatalogParameterQueryDependency, + ) -> MetadataCatalogParametersResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogParametersResponse, + lambda reader: reader.get_catalog_parameters( + query.country_id, query.policyengine_version, names=query.name + ), + ) + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index 56791d2bb..697ab6f76 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,6 +6,7 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -153,3 +154,9 @@ class MetadataCatalogVariablesResponse( MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] ): """Success response holding every variable of one catalog version.""" + + +class MetadataCatalogParametersResponse( + MetadataResourceSuccessResponse[MetadataCatalogParametersResult] +): + """Success response holding parameters fetched by name, with their values.""" diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index e06e9cdcb..a9f0816ae 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -232,6 +232,23 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +MAXIMUM_PARAMETER_NAMES = 100 +ParameterName = Annotated[ + str, + Field(min_length=1, max_length=512, description="Exact parameter path"), +] + + +class CatalogParameterQuery(CatalogQuery): + """Query contract for fetching several parameters of a catalog by name.""" + + name: list[ParameterName] = Field( + min_length=1, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Exact parameter paths; repeat the key for each one", + ) + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py new file mode 100644 index 000000000..154a1a6eb --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -0,0 +1,34 @@ +"""Database selections for parameter-node labels.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + +if TYPE_CHECKING: + from collections.abc import Collection + from uuid import UUID + + +def read_parameter_node_labels( + session: Session, *, model_version_id: UUID, node_names: Collection[str] +) -> list[Any]: + """Return the name and stored label of each named node of a model version. + + Names the version has no node for are absent. Rows are in name order. + """ + if not node_names: + return [] + return read_rows( + session, + select(col(ParameterNode.name), col(ParameterNode.label)) + .where( + col(ParameterNode.tax_benefit_model_version_id) == model_version_id, + col(ParameterNode.name).in_(sorted(node_names)), + ) + .order_by(col(ParameterNode.name)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 739b6b0cb..e16bca767 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -53,6 +53,26 @@ def read_parameters( ) +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Parameter]: + """Return the named parameters of a model version in name order. + + Names the version has no parameter for are absent. + """ + if not parameter_names: + return [] + return read_rows( + session, + select(Parameter) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + ) + .order_by(col(Parameter.name)), + ) + + def read_parameter( session: Session, *, model_version_id: UUID, parameter_id: UUID ) -> Parameter | None: diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index e6d4506c7..33f9ef07f 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -16,6 +16,9 @@ read_datasets, read_input_datasets, ) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 # a dotted module path cannot be wrapped + read_parameter_node_labels, +) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, ) @@ -25,6 +28,7 @@ read_parameter_value, read_parameter_values, read_parameters, + read_parameters_by_name, ) from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( read_input_dataset, @@ -37,6 +41,8 @@ read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + ancestor_paths, + metadata_catalog_parameters, metadata_catalog_variables, metadata_dataset, metadata_economy_options, @@ -55,6 +61,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -77,6 +84,7 @@ ) if TYPE_CHECKING: + from collections.abc import Sequence from uuid import UUID from policyengine_api.services.v2.metadata.database_session import ( @@ -618,3 +626,42 @@ def get_catalog_variables( return metadata_catalog_variables( selected, variables=variables, part_values=part_values ) + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values, and the result carries the stored + labels of their ancestor nodes and the requested names the version + lacks. Repeated names count once. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + model_version_id = selected.model_version.id + requested_names = set(names) + parameters = read_parameters_by_name( + session, model_version_id=model_version_id, parameter_names=requested_names + ) + found_names = {parameter.name for parameter in parameters} + value_rows = read_current_law_values_by_parameter_name( + session, model_version_id=model_version_id, parameter_names=found_names + ) + node_rows = read_parameter_node_labels( + session, + model_version_id=model_version_id, + node_names={ + ancestor for name in found_names for ancestor in ancestor_paths(name) + }, + ) + return metadata_catalog_parameters( + selected, + requested_names=requested_names, + parameters=parameters, + value_rows=value_rows, + node_rows=node_rows, + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 651f26414..cedd21ab2 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,6 +8,8 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, @@ -144,23 +146,37 @@ def metadata_catalog_variables( parameter-name and start-date order. Each path a variable names gets an entry, empty when no row carries its name. """ - values_by_path: dict[str, list[MetadataDatedValue]] = { - path: [] for path in sorted(part_parameter_paths(variables)) + values_by_path = _dated_values_by_parameter_name( + part_values, parameter_names=part_parameter_paths(variables) + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, + ) + + +def _dated_values_by_parameter_name( + value_rows: list[Any], *, parameter_names: set[str] +) -> dict[str, list[MetadataDatedValue]]: + """Group value rows by parameter name, with an empty list for each other name. + + value_rows holds rows of read_current_law_values_by_parameter_name, in + parameter-name and start-date order. + """ + values_by_name: dict[str, list[MetadataDatedValue]] = { + name: [] for name in sorted(parameter_names) } - for row in part_values: - values_by_path.setdefault(row.name, []).append( + for row in value_rows: + values_by_name.setdefault(row.name, []).append( MetadataDatedValue( start_date=as_utc(row.start_date), end_date=None if row.end_date is None else as_utc(row.end_date), value=row.value_json, ) ) - return MetadataCatalogVariablesResult( - policyengine_version=selected.policyengine_version, - catalog_format=selected.model_version.catalog_format, - variables=[metadata_variable(variable) for variable in variables], - part_parameter_values=values_by_path, - ) + return values_by_name def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: @@ -172,6 +188,54 @@ def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: description=parameter.description, data_type=parameter.data_type, unit=parameter.unit, + economy=parameter.economy, + household=parameter.household, + ) + + +def ancestor_paths(parameter_name: str) -> list[str]: + """Return the paths of every node above parameter_name, from the root down. + + A dot separates a child, and so does the ``[`` of a bracket index: + ``gov.tax.rates[0].rate`` has the ancestors ``gov``, ``gov.tax``, + ``gov.tax.rates`` and ``gov.tax.rates[0]``. + """ + return [ + parameter_name[:position] + for position, character in enumerate(parameter_name) + if character in ".[" and position > 0 + ] + + +def metadata_catalog_parameters( + selected: SelectedCatalog, + *, + requested_names: set[str], + parameters: list[Parameter], + value_rows: list[Any], + node_rows: list[Any], +) -> MetadataCatalogParametersResult: + """Build the requested parameters with their values and ancestor labels. + + value_rows holds rows of read_current_law_values_by_parameter_name and + node_rows rows of read_parameter_node_labels. + """ + values_by_name = _dated_values_by_parameter_name( + value_rows, parameter_names={parameter.name for parameter in parameters} + ) + found_names = {parameter.name for parameter in parameters} + return MetadataCatalogParametersResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + parameters=[ + MetadataCatalogParameter( + **metadata_parameter(parameter).model_dump(), + values=values_by_name[parameter.name], + ) + for parameter in parameters + ], + node_labels={row.name: row.label for row in node_rows}, + missing_names=sorted(requested_names - found_names), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index d6633eeaf..472d270e2 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -114,7 +114,9 @@ class MetadataVariable(StrictResponseModel): class MetadataParameterSummary(StrictResponseModel): """A policy parameter, described without its values. - Its current-law values are listed by GET /v2/parameter-values. + Its current-law values are listed by GET /v2/parameter-values. economy and + household say whether a reform of an economy or of a household may change + it; they are null in versions published before catalog format 2. """ id: UUID @@ -123,6 +125,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None + economy: bool | None = None + household: bool | None = None class MetadataCanonicalParameterValue(StrictResponseModel): @@ -332,3 +336,26 @@ class MetadataCatalogVariablesResult(StrictResponseModel): catalog_format: int | None variables: list[MetadataVariable] part_parameter_values: dict[str, list[MetadataDatedValue]] + + +class MetadataCatalogParameter(MetadataParameterSummary): + """A parameter with its current-law values in start-date order.""" + + values: list[MetadataDatedValue] + + +class MetadataCatalogParametersResult(StrictResponseModel): + """Requested parameters of one catalog version, in name order. + + node_labels maps each ancestor of a requested parameter that the version + stores as a parameter node to the node's stored label (null when it has + none), for breadcrumbs; a bracket index such as ``rates[0]`` counts as a + child of its schedule ``rates``. missing_names lists requested names the + version has no parameter for. + """ + + policyengine_version: str + catalog_format: int | None + parameters: list[MetadataCatalogParameter] + node_labels: dict[str, str | None] + missing_names: list[str] diff --git a/tests/contract/registry.py b/tests/contract/registry.py index b126a838e..b84980a1c 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -702,6 +702,24 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path=( + "/v2/model-catalog/parameters?country_id=us" + "&name=gov.irs.income.bracket.rates.1" + ), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 68a44ebda..5f5e6af26 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -508,3 +508,38 @@ def test_postgres_catalog_variables_carry_part_parameter_values_by_date( "employment_income" ] assert uk.json()["result"]["part_parameter_values"] == {} + + +def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( + published_engine: Engine, +) -> None: + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/parameters", + params=[ + ("country_id", "us"), + ("policyengine_version", POLICYENGINE_VERSION), + ("name", "gov.example.rate"), + ("name", "gov.removed.amount"), + ], + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [parameter["name"] for parameter in result["parameters"]] == [ + "gov.example.rate" + ] + parameter = result["parameters"][0] + assert (parameter["economy"], parameter["household"]) == (True, False) + assert parameter["values"] == [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": "2025-12-31T00:00:00Z", + "value": 0.1, + }, + {"start_date": "2026-01-01T00:00:00Z", "end_date": None, "value": 0.2}, + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed.amount"] diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index a52730e6e..e35785e12 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 91, + "request_count": 92, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e522a114b..6b1b252e9 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -26,6 +26,8 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, @@ -198,6 +200,24 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_parameters": MetadataCatalogParametersResult( + policyengine_version=version, + catalog_format=2, + parameters=[ + MetadataCatalogParameter( + **parameter.model_dump(), + values=[ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=0.1, + ) + ], + ) + ], + node_labels={"gov.example": "Example policy"}, + missing_names=["gov.removed"], + ), "get_catalog_variables": MetadataCatalogVariablesResult( policyengine_version=version, catalog_format=2, @@ -567,6 +587,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/economy-options", "/v2/model-catalog", "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -653,6 +674,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: "/v2/economy-options?country_id=us", "/v2/model-catalog?country_id=us", "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/parameters?country_id=us&name=gov.example.rate", ] @@ -841,3 +863,68 @@ def test_catalog_variables_route_rejects_paging_and_unknown_queries( ) assert response.status_code == 422 + + +def test_catalog_parameters_route_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/parameters" + "?country_id=us&policyengine_version=4.20.3" + "&name=gov.example.rate&name=gov.removed" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["parameters"][0]["name"] == "gov.example.rate" + assert result["parameters"][0]["values"] == [ + {"start_date": "2025-01-01T00:00:00Z", "end_date": None, "value": 0.1} + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed"] + assert reader.calls == [ + ( + "get_catalog_parameters", + ("us", "4.20.3"), + {"names": ["gov.example.rate", "gov.removed"]}, + ) + ] + + +@pytest.mark.parametrize( + "query", + [ + "country_id=us", + "country_id=us&name=", + "country_id=us&name=gov.example.rate&offset=0", + "country_id=us&" + "&".join(f"name=gov.p{index}" for index in range(101)), + ], +) +def test_catalog_parameters_route_rejects_invalid_name_lists(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog/parameters?{query}") + + assert response.status_code == 422 + assert calls == [] + + +def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog/parameters"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version", "name"} + assert parameters["name"]["required"] is True + name_schema = parameters["name"]["schema"] + assert name_schema["type"] == "array" + assert name_schema["minItems"] == 1 + assert name_schema["maxItems"] == 100 diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d30ba98b8..d8813b960 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -34,6 +34,7 @@ MetadataDatabaseSession, ) from policyengine_api.services.v2.metadata.services import V2MetadataService +from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( MetadataDatedValue, MetadataEntity, @@ -843,6 +844,142 @@ def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> N assert variable.enum_values is None +def _add_bracket_schedule(session: Session) -> None: + model_version = _us_model_version(session) + session.add_all( + [ + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates", + label="Example rates", + description=None, + ), + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0]", + label=None, + description=None, + ), + ] + ) + bracket_rate = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0].rate", + label="Rate", + description=None, + data_type="float", + unit="/1", + economy=True, + household=True, + ) + session.add(bracket_rate) + session.add( + ParameterValue( + id=uuid4(), + parameter_id=bracket_rate.id, + value_json=0.2, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ) + session.commit() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("gov", []), + ("gov.example.rate", ["gov", "gov.example"]), + ( + "gov.example.rates[0].rate", + ["gov", "gov.example", "gov.example.rates", "gov.example.rates[0]"], + ), + ("gov.example.rates[10]", ["gov", "gov.example", "gov.example.rates"]), + ], +) +def test_ancestor_paths_treat_a_bracket_index_as_a_child( + name: str, expected: list[str] +) -> None: + assert ancestor_paths(name) == expected + + +def test_named_parameters_load_with_values_and_ancestor_labels_in_one_read( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_parameters( + "us", + names=["gov.example.rates[0].rate", "gov.example.rate", "gov.example.rate"], + ), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [parameter.name for parameter in result.parameters] == [ + "gov.example.rate", + "gov.example.rates[0].rate", + ] + example_rate, bracket_rate = result.parameters + assert (example_rate.economy, example_rate.household) == (True, False) + assert [value.value for value in example_rate.values] == [0.1, 0.2] + assert example_rate.values[0].start_date == datetime(2025, 1, 1, tzinfo=UTC) + assert example_rate.values[0].end_date == datetime(2025, 12, 31, tzinfo=UTC) + assert [value.value for value in bracket_rate.values] == [0.2] + assert result.node_labels == { + "gov.example": "Example policy", + "gov.example.rates": "Example rates", + "gov.example.rates[0]": None, + } + assert result.missing_names == [] + assert len(statements) == 4 + + +def test_named_parameters_list_names_the_version_lacks( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_catalog_parameters( + "uk", names=["gov.example.rate", "gov.removed.amount"] + ) + + assert [parameter.name for parameter in result.parameters] == ["gov.example.rate"] + assert result.missing_names == ["gov.removed.amount"] + + +def test_named_parameters_exclude_policy_and_dynamic_values( + catalog_session: Session, +) -> None: + model_version = _us_model_version(catalog_session) + parameter = catalog_session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version.id, + Parameter.name == "gov.example.rate", + ) + ).one() + catalog_session.add( + ParameterValue( + id=uuid4(), + parameter_id=parameter.id, + value_json=0.9, + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ) + ) + catalog_session.commit() + + result = _service(catalog_session).get_catalog_parameters( + "us", names=["gov.example.rate"] + ) + + assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -852,6 +989,7 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package / "data" / "v2" / "catalog" / "catalog_selection.py", connector_directory / "reads.py", connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_nodes.py", connector_directory / "reads_parameter_tree.py", connector_directory / "reads_parameters.py", connector_directory / "reads_regions.py", @@ -909,6 +1047,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_economy_options", "get_model_catalog", "get_catalog_variables", + "get_catalog_parameters", } for method_name in method_names: From 830ee5b8992f29ebedfb374f9f635755e0dddf57 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:59:16 +0400 Subject: [PATCH 05/33] Filter parameter pages and values by exact parameter names GET /v2/parameters accepts repeated name keys and GET /v2/parameter-values repeated parameter_name keys (at most 100 each), so a client can page only those parameters or their values. Because their query contracts change, both routes now read their query through the shared ParameterCollectionQuery and ParameterValueCollectionQuery models: unknown and repeated scalar parameters, an unsupported country_id and a malformed policyengine_version are rejected with 422 before any read (the last two returned 400 from the read before, and unknown parameters were ignored). The service and connector take the search and the names as one ParameterFilter, like ParameterValueFilter. Co-Authored-By: Claude Opus 5.5 --- ...1f-116-catalog-routes-on-demand.changed.md | 1 + docs/migration/stage-9-v2-metadata.md | 9 ++ .../fastapi_routes/dependencies.py | 13 +- .../v2/metadata/parameter_routes.py | 54 ++++---- policyengine_api/query_parameters.py | 29 +++++ .../database_connectors/reads_parameters.py | 28 +++- .../services/v2/metadata/services.py | 17 ++- .../services/v2/metadata/transformations.py | 1 + .../services/v2/metadata/types.py | 23 +++- tests/unit/v2/test_metadata_routes.py | 121 ++++++++++++++++++ tests/unit/v2/test_metadata_service.py | 31 ++++- 11 files changed, 281 insertions(+), 46 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md new file mode 100644 index 000000000..62dc783e3 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -0,0 +1 @@ +/v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. Both read their query through the shared query models, so unknown or repeated parameters, an unsupported country_id and a malformed policyengine_version now return 422. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index a5aaaba4c..028a6713e 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -66,6 +66,15 @@ lists requested names the version has no parameter for, so a reform saved against an older version still opens. `GET /v2/parameters` pages and details carry the same `economy` and `household` flags. +`GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page +only those exact parameters, and `GET /v2/parameter-values` repeated +`parameter_name` keys to page only those parameters' values. Both routes now +read their query through the shared query models (`ParameterCollectionQuery` +and `ParameterValueCollectionQuery`): unknown and repeated scalar parameters, +an unsupported `country_id` and a malformed `policyengine_version` are +rejected with HTTP 422 before any read; before, the last two returned HTTP 400 +from the read and unknown parameters were ignored. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 6631c8630..1dfcdaaa5 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -48,6 +48,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.policies.types import ( @@ -193,11 +194,12 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: - """Return parameters in name order, optionally filtered by ``search``. + """Return parameters in name order, optionally filtered. - ``search`` is a case-insensitive substring of the name, label or description. + The filter's ``search`` is a case-insensitive substring of the name, + label or description; its ``names`` keep only those exact names. """ ... @@ -234,9 +236,10 @@ def list_parameter_values( offset: int = 0, limit: int = 100, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - """Return canonical values, optionally for one parameter. + """Return canonical values, optionally for one or several parameters. - With the filter's ``current``, only values in effect on the UTC day of its + The filter's ``parameter_id`` or ``parameter_names`` select parameters. + With its ``current``, only values in effect on the UTC day of its ``now`` (default: the present) are returned. """ ... diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 5895c8564..252682bb0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,13 +2,13 @@ from __future__ import annotations -from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated from uuid import UUID from fastapi import APIRouter, Depends, Query from starlette.responses import JSONResponse +from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, @@ -21,7 +21,14 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.query_parameters import ( + ParameterCollectionQuery, + ParameterValueCollectionQuery, +) +from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueFilter, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -30,16 +37,13 @@ Limit = Annotated[int, Query(ge=1, le=500)] -@dataclass(frozen=True) -class ParameterValueListingQuery: - """The query parameters of the canonical parameter-value listing route.""" - - country_id: str - policyengine_version: str | None = None - parameter_id: UUID | None = None - current: bool = False - offset: Offset = 0 - limit: Limit = 100 +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] def build_v2_metadata_parameter_router( @@ -62,21 +66,20 @@ def build_v2_metadata_parameter_router( summary="List parameters from a selected PolicyEngine.py catalog", ) def list_parameters( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: ParameterCollectionQueryDependency, ) -> MetadataParameterPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterPageResponse, lambda reader: reader.list_parameters( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + parameter_filter=ParameterFilter( + search=query.search, + names=None if query.name is None else tuple(query.name), + ), ), ) @@ -130,7 +133,7 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( - query: Annotated[ParameterValueListingQuery, Depends()], + query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: return read_resource( dependencies, @@ -141,6 +144,11 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + parameter_names=( + None + if query.parameter_name is None + else tuple(query.parameter_name) + ), ), offset=query.offset, limit=query.limit, diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index a9f0816ae..ec614f109 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -249,6 +249,35 @@ class CatalogParameterQuery(CatalogQuery): ) +class ParameterCollectionQuery(CatalogQuery, PaginationQuery): + """Query contract for a page of a catalog's parameters.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths; repeat the key", + ) + + +class ParameterValueCollectionQuery(CatalogQuery, PaginationQuery): + """Query contract for a page of a catalog's current-law parameter values.""" + + parameter_id: ResourceId | None = None + parameter_name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths' values; repeat the key", + ) + current: bool = Field( + default=False, + description="Keep only the values in effect today (UTC)", + ) + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index e16bca767..39ab71bd5 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -14,7 +14,10 @@ from collections.abc import Collection from uuid import UUID - from policyengine_api.services.v2.metadata.types import ParameterValueSelection + from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueSelection, + ) def _escape_like(value: str) -> str: @@ -27,17 +30,23 @@ def read_parameters( model_version_id: UUID, offset: int, limit: int, - search: str | None, + parameter_filter: ParameterFilter, ) -> list[Parameter]: """Return a page of a model version's parameters in name order. - search matches name, label or description as a literal, case-insensitive - substring. Fetch limit + 1 rows so the caller can tell whether another page - exists. + The filter's search matches name, label or description as a literal, + case-insensitive substring, and its names keep only parameters with those + exact names. Fetch limit + 1 rows so the caller can tell whether another + page exists. """ statement = select(Parameter).where( col(Parameter.tax_benefit_model_version_id) == model_version_id ) + if parameter_filter.names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(parameter_filter.names)) + ) + search = parameter_filter.search if search: pattern = f"%{_escape_like(search)}%" statement = statement.where( @@ -98,8 +107,9 @@ def read_parameter_values( """Return a page of a model version's current-law parameter values. Values tied to a policy or dynamic are excluded. The selection's - parameter_id keeps one parameter's values and its selected_day keeps those - in effect on that day. Rows are ordered by parameter name, then newest + parameter_id keeps one parameter's values, its parameter_names the values + of the parameters with those names, and its selected_day those in effect + on that day. Rows are ordered by parameter name, then newest first, and limit + 1 are fetched so the caller can tell whether another page exists. """ @@ -116,6 +126,10 @@ def read_parameter_values( ) if parameter_id is not None: statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selection.parameter_names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(selection.parameter_names)) + ) if selected_day is not None: statement = statement.where( col(ParameterValue.start_date) <= selected_day, diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 33f9ef07f..cdae4758e 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -75,6 +75,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -271,12 +272,13 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: """Return a page of the catalog's parameters in name order. - search keeps parameters whose name, label or description contains it, - ignoring case. + The filter's search keeps parameters whose name, label or description + contains it, ignoring case, and its names keep only parameters with + those exact names. No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -286,7 +288,9 @@ def list_parameters( model_version_id=selected.model_version.id, offset=offset, limit=limit, - search=search, + parameter_filter=( + parameter_filter if parameter_filter is not None else ParameterFilter() + ), ) return page_result( selected, @@ -359,8 +363,9 @@ def list_parameter_values( """Return a page of current-law parameter values, newest first per parameter. Values belonging to a reform or dynamic are never listed. The filter's - parameter_id keeps one parameter's values, and current=True keeps those in - effect on the UTC day of now (today by default). No filter lists them all. + parameter_id keeps one parameter's values, parameter_names the values of + the parameters with those exact names, and current=True those in effect + on the UTC day of now (today by default). No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index cedd21ab2..4b9167845 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -286,6 +286,7 @@ def parameter_value_selection( return ParameterValueSelection( parameter_id=value_filter.parameter_id, selected_day=selected_day, + parameter_names=value_filter.parameter_names, ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 472d270e2..36dee6c43 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -11,28 +11,43 @@ from pydantic import BaseModel, ConfigDict, JsonValue +@dataclass(frozen=True) +class ParameterFilter: + """Which parameters a listing keeps. + + ``search`` keeps parameters whose name, label or description contains it, + ignoring case, and ``names`` the parameters with those exact names. + """ + + search: str | None = None + names: tuple[str, ...] | None = None + + @dataclass(frozen=True) class ParameterValueFilter: """Which current-law parameter values a listing keeps. - ``parameter_id`` keeps one parameter's values, and ``current`` keeps the - values in effect on the UTC day of ``now`` (today when ``now`` is None). + ``parameter_id`` keeps one parameter's values, ``parameter_names`` the + values of the parameters with those exact names, and ``current`` the values + in effect on the UTC day of ``now`` (today when ``now`` is None). """ parameter_id: UUID | None = None current: bool = False now: datetime | None = None + parameter_names: tuple[str, ...] | None = None @dataclass(frozen=True) class ParameterValueSelection: - """The parameter and effective day a parameter-value read is narrowed to. + """The parameters and effective day a parameter-value read is narrowed to. - None for either leaves the read unnarrowed by it. + None for any of them leaves the read unnarrowed by it. """ parameter_id: UUID | None selected_day: datetime | None + parameter_names: tuple[str, ...] | None = None class StrictResponseModel(BaseModel): diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 6b1b252e9..366647f29 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -47,6 +47,7 @@ MetadataRegionOption, MetadataTimePeriodOption, MetadataVariable, + ParameterFilter, ) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, @@ -928,3 +929,123 @@ def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: assert name_schema["type"] == "array" assert name_schema["minItems"] == 1 assert name_schema["maxItems"] == 100 + + +def test_parameter_collection_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + client = _client(lambda: reader) + + named = client.get( + "/v2/parameters?country_id=US&name=gov.example.rate&name=gov.other.rate" + ) + paged = client.get("/v2/parameters?country_id=us&offset=5&limit=50&search=rate") + + assert named.status_code == 200 + assert paged.status_code == 200 + assert reader.calls == [ + ( + "list_parameters", + ("us", None), + { + "offset": 0, + "limit": 100, + "parameter_filter": ParameterFilter( + search=None, names=("gov.example.rate", "gov.other.rate") + ), + }, + ), + ( + "list_parameters", + ("us", None), + { + "offset": 5, + "limit": 50, + "parameter_filter": ParameterFilter(search="rate", names=None), + }, + ), + ] + + +def test_parameter_values_fetch_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3" + "¶meter_name=gov.example.rate¶meter_name=gov.other.rate¤t=true" + ) + + assert response.status_code == 200 + name, args, kwargs = reader.calls[0] + assert (name, args) == ("list_parameter_values", ("us", "4.20.3")) + assert kwargs["value_filter"].parameter_names == ( + "gov.example.rate", + "gov.other.rate", + ) + assert kwargs["value_filter"].parameter_id is None + assert kwargs["value_filter"].current is True + assert (kwargs["offset"], kwargs["limit"]) == (0, 100) + + +@pytest.mark.parametrize( + "path", + [ + "/v2/parameters?country_id=us¶meter_id=1", + "/v2/parameters?country_id=us&limit=5&limit=6", + "/v2/parameters?country_id=us&name=", + "/v2/parameters?country_id=ca", + "/v2/parameters?country_id=us&" + + "&".join(f"name=gov.p{index}" for index in range(101)), + "/v2/parameter-values?country_id=us&name=gov.example.rate", + "/v2/parameter-values?country_id=us¤t=true¤t=false", + "/v2/parameter-values?country_id=us¶meter_id=not-a-uuid", + ], +) +def test_parameter_collections_reject_unknown_repeated_and_invalid_queries( + path: str, +) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(path) + + assert response.status_code == 422 + assert calls == [] + + +def test_parameter_collections_declare_their_name_filters_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + def query_parameters(path: str) -> dict[str, dict]: + return { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + } + + parameters = query_parameters("/v2/parameters") + values = query_parameters("/v2/parameter-values") + assert set(parameters) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "search", + "name", + } + assert parameters["name"]["required"] is False + assert set(values) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "parameter_id", + "parameter_name", + "current", + } + assert values["parameter_name"]["required"] is False diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d8813b960..fc0827b06 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -39,6 +39,7 @@ MetadataDatedValue, MetadataEntity, MetadataEnumValue, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -452,7 +453,9 @@ def test_resource_filters_and_details_remain_inside_selected_catalog( ) -> None: service = _service(catalog_session) variables = service.list_variables("us", search="employment") - parameters = service.list_parameters("us", search="example rate") + parameters = service.list_parameters( + "us", parameter_filter=ParameterFilter(search="example rate") + ) states = service.list_regions("us", region_type="state") region = service.get_region_by_code("us", "state/ca") @@ -980,6 +983,32 @@ def test_named_parameters_exclude_policy_and_dynamic_values( assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] +def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + service = _service(catalog_session) + + parameters = service.list_parameters( + "us", + parameter_filter=ParameterFilter( + names=("gov.example.rates[0].rate", "gov.removed.amount") + ), + ) + values = service.list_parameter_values( + "us", + value_filter=ParameterValueFilter( + parameter_names=("gov.example.rate", "gov.example.rates[0].rate") + ), + ) + + assert [parameter.name for parameter in parameters.items] == [ + "gov.example.rates[0].rate" + ] + assert parameters.items[0].economy is True + assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 17300e7bc7b4d15781019eb58c5a089214428039 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:07:34 +0400 Subject: [PATCH 06/33] Keep whole catalog-version responses once built in each process Measured against the full 6.10.0 catalog in a local database, every new query uses an existing (version, name) or (version, code) unique index and takes under 4 ms; the cost of /v2/model-catalog/variables (about 160 ms, 3.9 MB, 0.58 MB compressed) is reading 6,167 rows and building their models. A published catalog version never changes, so each process now keeps the serialized bodies of /v2/model-catalog and /v2/model-catalog/variables once built, in a bounded store (eight route, country and requested-version combinations, least recently used dropped first) shared by every request of the default dependencies. Concurrent first requests wait for one read, and failed reads are not kept. A kept variables response takes about 25 ms, most of it compression. Tests pass their own store or none. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/migration/stage-9-v2-metadata.md | 10 ++ .../fastapi_routes/dependencies.py | 13 ++ .../v2/metadata/catalog_routes.py | 13 +- .../fastapi_routes/v2/metadata/common.py | 32 +++++ .../v2/metadata/response_cache.py | 73 ++++++++++++ tests/unit/v2/test_metadata_response_cache.py | 111 ++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 59 +++++++++- 8 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/response_cache.py create mode 100644 tests/unit/v2/test_metadata_response_cache.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 1e4e961ec..4f665f71d 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -2,3 +2,4 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 028a6713e..3cbe58fab 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -87,6 +87,16 @@ deploy, and may be reused for five minutes (`max-age=300`). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. +Each API process also keeps the serialized bodies of `/v2/model-catalog` and +`/v2/model-catalog/variables` once built, for up to eight route, country and +requested-version combinations (least recently used dropped first; a request +without `policyengine_version` is keyed to the process's running version, +which never changes while it runs). Only the first successful request per +process reads the database; concurrent first requests wait for one read, and +a failed read is not kept. Measured locally against the full 6.10.0 US +catalog, the variables route takes about 160 ms when read and about 25 ms +when kept, most of it compression. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 1dfcdaaa5..43f6fbf4f 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -10,6 +10,9 @@ from fastapi import Depends from fastapi.security import APIKeyHeader, HTTPAuthorizationCredentials, HTTPBearer +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready @@ -773,6 +776,12 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_metadata_response_cache() -> MetadataResponseCache: + """Return the whole catalog-version responses this process keeps.""" + return MetadataResponseCache() + + def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -978,6 +987,9 @@ class NativeRouteDependencies: None ) v2_account_service_factory: Callable[[], V2AccountResourceService] | None = None + # Whole catalog-version responses (/v2/model-catalog and its variables) are + # kept here once built; None reads them on every request. + v2_metadata_response_cache: MetadataResponseCache | None = None # User-owned routes attach this at router level. The default validates the # Auth0 bearer token, or else accepts the unauthenticated-user header, and # fails closed (401/403/503); tests replace it with a callable returning a @@ -993,6 +1005,7 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_metadata_response_cache=_process_metadata_response_cache(), v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 8d91e8630..bef2ba326 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -5,13 +5,14 @@ from typing import TYPE_CHECKING, Annotated from fastapi import APIRouter, Depends -from starlette.responses import JSONResponse +from starlette.responses import JSONResponse, Response from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, read_resource, + read_whole_version_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataCatalogParametersResponse, @@ -60,9 +61,10 @@ def build_v2_model_catalog_router( ) def get_model_catalog( query: CatalogQueryDependency, - ) -> MetadataModelCatalogResponse | JSONResponse: - return read_resource( + ) -> MetadataModelCatalogResponse | Response: + return read_whole_version_resource( dependencies, + ("model-catalog", query.country_id, query.policyengine_version), MetadataModelCatalogResponse, lambda reader: reader.get_model_catalog( query.country_id, query.policyengine_version @@ -83,9 +85,10 @@ def get_model_catalog( ) def get_catalog_variables( query: CatalogQueryDependency, - ) -> MetadataCatalogVariablesResponse | JSONResponse: - return read_resource( + ) -> MetadataCatalogVariablesResponse | Response: + return read_whole_version_resource( dependencies, + ("model-catalog/variables", query.country_id, query.policyengine_version), MetadataCatalogVariablesResponse, lambda reader: reader.get_catalog_variables( query.country_id, query.policyengine_version diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 9cd6ff688..d44c66f8a 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -37,6 +37,9 @@ NativeRouteDependencies, V2MetadataResourceReader, ) + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + ResponseKey, + ) ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { @@ -150,6 +153,35 @@ def read_resource( ) +def read_whole_version_resource( + dependencies: NativeRouteDependencies, + key: ResponseKey, + response_type: type[ResponseT], + operation: Callable[[V2MetadataResourceReader], object], +) -> ResponseT | Response: + """Serve a whole catalog-version response, built once per process when kept. + + With the dependencies' response cache, the serialized body for ``key`` is + read and built only on the first successful request; failures are served + as ``read_resource`` serves them and never kept. Without a cache this is + ``read_resource``. + """ + cache = dependencies.v2_metadata_response_cache + if cache is None: + return read_resource(dependencies, response_type, operation) + + def build() -> bytes | JSONResponse: + result = read_resource(dependencies, response_type, operation) + if isinstance(result, BaseModel): + return result.model_dump_json().encode() + return result + + built = cache.get_or_build(key, build) + if isinstance(built, bytes): + return Response(content=built, media_type="application/json") + return built + + def run_metadata_read( dependencies: NativeRouteDependencies, operation: Callable[[V2MetadataResourceReader], ReadT], diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py new file mode 100644 index 000000000..ae688ae4c --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -0,0 +1,73 @@ +"""A bounded, process-wide store of whole catalog-version response bodies.""" + +from __future__ import annotations + +import threading +from collections import OrderedDict +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + from starlette.responses import JSONResponse + +ResponseKey = tuple[str, str, str | None] +"""A route name, a country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + +DEFAULT_RESPONSE_CAPACITY = 8 + + +class MetadataResponseCache: + """Keep the serialized bodies of whole-version responses, built once each. + + A published catalog version never changes, so a body built for a route, + country and version stays valid for the life of the process. The most + recently used ``capacity`` bodies are kept. Requests for a body being built + wait for that build instead of starting another, and a failed read (a v2 + error response) is returned but not kept, so the next request tries again. + """ + + def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: + """Keep at most ``capacity`` bodies (at least one).""" + if capacity < 1: + msg = "capacity must be positive" + raise ValueError(msg) + self._capacity = capacity + self._bodies: OrderedDict[ResponseKey, bytes] = OrderedDict() + self._lock = threading.Lock() + self._build_locks: dict[ResponseKey, threading.Lock] = {} + + def _stored(self, key: ResponseKey) -> bytes | None: + with self._lock: + body = self._bodies.get(key) + if body is not None: + self._bodies.move_to_end(key) + return body + + def get_or_build( + self, + key: ResponseKey, + build: Callable[[], bytes | JSONResponse], + ) -> bytes | JSONResponse: + """Return the body kept for ``key``, calling ``build`` only when none is.""" + body = self._stored(key) + if body is not None: + return body + with self._lock: + build_lock = self._build_locks.setdefault(key, threading.Lock()) + with build_lock: + body = self._stored(key) + if body is not None: + return body + built = build() + with self._lock: + self._build_locks.pop(key, None) + if isinstance(built, bytes): + self._bodies[key] = built + self._bodies.move_to_end(key) + while len(self._bodies) > self._capacity: + self._bodies.popitem(last=False) + return built diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py new file mode 100644 index 000000000..a96d7b005 --- /dev/null +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -0,0 +1,111 @@ +"""The process-wide store of whole catalog-version response bodies.""" + +from __future__ import annotations + +import threading +import time + +import pytest +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) + + +def _counting_build(body: bytes, builds: list[str]): + def build() -> bytes: + builds.append("built") + return body + + return build + + +def test_a_body_is_built_once_per_key_and_reused() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + key = ("variables", "us", "6.10.0") + + first = cache.get_or_build(key, _counting_build(b'{"a":1}', builds)) + second = cache.get_or_build(key, _counting_build(b'{"b":2}', builds)) + + assert first == b'{"a":1}' + assert second == b'{"a":1}' + assert builds == ["built"] + + +def test_different_routes_countries_and_versions_are_kept_apart() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + for key in ( + ("variables", "us", "6.10.0"), + ("variables", "uk", "6.10.0"), + ("variables", "us", None), + ("model-catalog", "us", "6.10.0"), + ): + cache.get_or_build(key, _counting_build(repr(key).encode(), builds)) + + assert len(builds) == 4 + + +def test_a_failed_read_is_returned_but_not_kept() -> None: + cache = MetadataResponseCache() + failure = JSONResponse({"status": "error"}, status_code=503) + builds: list[str] = [] + + def failing_build() -> JSONResponse: + builds.append("failed") + return failure + + first = cache.get_or_build(("variables", "us", None), failing_build) + second = cache.get_or_build( + ("variables", "us", None), _counting_build(b"{}", builds) + ) + + assert first is failure + assert second == b"{}" + assert builds == ["failed", "built"] + + +def test_the_least_recently_used_body_is_dropped_beyond_capacity() -> None: + cache = MetadataResponseCache(capacity=2) + builds: list[str] = [] + oldest, middle, newest = (("variables", country, None) for country in "abc") + + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(middle, _counting_build(b"2", builds)) + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(newest, _counting_build(b"3", builds)) + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(middle, _counting_build(b"2", builds)) + + assert builds == ["built", "built", "built", "built"] + + +def test_concurrent_requests_for_one_key_build_it_once() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + started = threading.Barrier(4) + + def slow_build() -> bytes: + builds.append("built") + time.sleep(0.05) + return b"{}" + + def request() -> None: + started.wait() + cache.get_or_build(("variables", "us", None), slow_build) + + threads = [threading.Thread(target=request) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert builds == ["built"] + + +def test_capacity_must_be_positive() -> None: + with pytest.raises(ValueError, match="capacity"): + MetadataResponseCache(capacity=0) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 366647f29..c3b50b6e5 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -20,6 +20,9 @@ from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -286,7 +289,7 @@ def detail(item: object) -> MetadataDetailResult: ) -def _client(factory) -> TestClient: +def _client(factory, *, response_cache=None) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -305,6 +308,7 @@ def v1_metadata(country_id: str): metadata_reader_factory=lambda: None, specification_provider=dict, v2_metadata_reader_factory=factory, + v2_metadata_response_cache=response_cache, ) settings = RouteImplementationSettings( health=RouteImplementation.FLASK_FALLBACK, @@ -1049,3 +1053,56 @@ def query_parameters(path: str) -> dict[str, dict]: "current", } assert values["parameter_name"]["required"] is False + + +def test_whole_version_responses_are_read_once_per_process_and_version() -> None: + results, _ids = _resource_results() + readers: list[ResourceReader] = [] + + def factory() -> ResourceReader: + readers.append(ResourceReader(results)) + return readers[-1] + + client = _client(factory, response_cache=MetadataResponseCache()) + paths = [ + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3", + "/v2/model-catalog/variables?country_id=uk", + ] + + first_responses = [client.get(path) for path in paths] + repeated_responses = [client.get(path) for path in paths] + + assert len(readers) == len(paths) + for first, repeated in zip(first_responses, repeated_responses, strict=True): + assert repeated.status_code == 200 + assert repeated.content == first.content + assert repeated.headers["etag"] == first.headers["etag"] + assert repeated.headers["cache-control"] == first.headers["cache-control"] + assert all(reader.closed for reader in readers) + + +def test_failed_whole_version_reads_are_retried() -> None: + results, _ids = _resource_results() + readers = [ + ResourceReader(results, error=MetadataCatalogUnavailableError("down")), + ResourceReader(results), + ] + client = _client(lambda: readers.pop(0), response_cache=MetadataResponseCache()) + + failed = client.get("/v2/model-catalog/variables?country_id=us") + recovered = client.get("/v2/model-catalog/variables?country_id=us") + + assert failed.status_code == 503 + assert "etag" not in failed.headers + assert recovered.status_code == 200 + assert readers == [] + + +def test_default_dependencies_share_one_response_cache() -> None: + first = route_dependencies.NativeRouteDependencies.defaults() + second = route_dependencies.NativeRouteDependencies.defaults() + + assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) + assert first.v2_metadata_response_cache is second.v2_metadata_response_cache From 08d990d3c5f9a083a38d433442d5a7cf3d26bb7f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:09:30 +0400 Subject: [PATCH 07/33] Check the on-demand catalog routes on each release and record their times The release workflow's live metadata test, run against each staging and production Cloud Run candidate, now loads each country's catalog in parts: /v2/model-catalog for the running version, its variables for that exact version twice and once with If-None-Match (which must get 304), and one parameter by name. It appends each request's time and body size to the GitHub job summary (GITHUB_STEP_SUMMARY, set by the runner in every step), so every release records the routes' times against its own catalog. The new paths join its OpenAPI check. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 8 ++ tests/integration/test_live_v2_metadata.py | 109 ++++++++++++++++++++- 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3cbe58fab..7d0f0a36f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -340,6 +340,14 @@ service error. None of these outcomes reads or repairs v1. Unsupported countries and methods return typed client errors. +The same test loads each country's catalog in parts as a client starting +cold would: `GET /v2/model-catalog` for the running version, then +`/v2/model-catalog/variables` for that exact version twice (the second served +from the process's kept body) and once more with `If-None-Match`, which must +return 304, then `/v2/model-catalog/parameters` for one parameter. It appends +each request's time and body size to the GitHub job summary, so every staging +and production release records these route times against its own catalog. + Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for success and error responses. diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 235304156..803e43b05 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -1,8 +1,11 @@ """Read-only checks for deployed Cloud Run v2 metadata preview routes.""" +import os +import time import tomllib from pathlib import Path +import httpx import pytest REPO = Path(__file__).parents[2] @@ -44,7 +47,15 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: assert response.status_code == 200, response.text[:500] document = response.json() expected_paths = {f"/v2/{resource}" for resource in PAGED_RESOURCES} - expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) + expected_paths.update( + { + "/v2/parameters/children", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", + } + ) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) assert "V2ErrorResponse" in document["components"]["schemas"] @@ -104,3 +115,99 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: assert payload["status"] == "error" assert payload["message"] assert "result" not in payload or payload["result"] is None + + +def _timed_get( + api_client: httpx.Client, path: str, **request_options: object +) -> tuple[httpx.Response, float]: + started = time.perf_counter() + response = api_client.get(path, **request_options) + return response, time.perf_counter() - started + + +def _record_route_times( + country_id: str, route_times: list[tuple[str, float, httpx.Response]] +) -> None: + """Append each route's time and size to the GitHub job summary, when there is one. + + The release workflow runs this test against each Cloud Run candidate, so + the summary keeps every release's staging and production route times. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + lines = [ + f"\n#### v2 catalog route times ({country_id})\n", + "| Request | Time (ms) | Body (bytes, uncompressed) |", + "| --- | ---: | ---: |", + ] + lines.extend( + f"| {label} | {seconds * 1000:.0f} | {len(response.content)} |" + for label, seconds, response in route_times + ) + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write("\n".join(lines) + "\n") + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> None: + expected_version = _installed_policyengine_version() + route_times: list[tuple[str, float, httpx.Response]] = [] + + catalog_response, seconds = _timed_get( + api_client, "/v2/model-catalog", params={"country_id": country_id} + ) + route_times.append( + ("/v2/model-catalog (running version)", seconds, catalog_response) + ) + catalog = _assert_ok(catalog_response, expected_version) + assert catalog["regions"] + assert catalog["economy_options"]["region"] + assert "variables" not in catalog + pinned = {"country_id": country_id, "policyengine_version": expected_version} + + variables_response, seconds = _timed_get( + api_client, "/v2/model-catalog/variables", params=pinned + ) + route_times.append(("/v2/model-catalog/variables", seconds, variables_response)) + repeated_variables, seconds = _timed_get( + api_client, "/v2/model-catalog/variables", params=pinned + ) + route_times.append( + ("/v2/model-catalog/variables (repeated)", seconds, repeated_variables) + ) + variables = _assert_ok(variables_response, expected_version) + assert variables["variables"] + assert variables_response.headers["cache-control"] == "public, max-age=86400" + assert repeated_variables.headers["etag"] == variables_response.headers["etag"] + revalidated, seconds = _timed_get( + api_client, + "/v2/model-catalog/variables", + params=pinned, + headers={"If-None-Match": variables_response.headers["etag"]}, + ) + route_times.append( + ("/v2/model-catalog/variables (revalidated)", seconds, revalidated) + ) + assert revalidated.status_code == 304 + + parameter_name = _assert_ok( + api_client.get("/v2/parameters", params={**pinned, "limit": 1}), + expected_version, + )["items"][0]["name"] + parameters_response, seconds = _timed_get( + api_client, + "/v2/model-catalog/parameters", + params={**pinned, "name": parameter_name}, + ) + route_times.append( + ("/v2/model-catalog/parameters (1 name)", seconds, parameters_response) + ) + parameters = _assert_ok(parameters_response, expected_version) + assert [parameter["name"] for parameter in parameters["parameters"]] == [ + parameter_name + ] + assert parameters["parameters"][0]["values"] + assert parameters["missing_names"] == [] + + _record_route_times(country_id, route_times) From 4c64114867d7268cdca0eccf26b7b9baef9be11c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:36:17 +0400 Subject: [PATCH 08/33] Reuse current parameter values only until the next UTC midnight GET /v2/parameter-values?current=true returns the values in effect on the request's UTC day, but its response got the same lifetime as any other: a day when it named a version. A cache that fetched it late on 31 December would have served last year's values for up to a day. The route now marks such a request in request.state, and the caching route caps its lifetime at the seconds left until the next UTC midnight, whichever lifetime would otherwise apply. No other metadata route depends on the time of the request. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 16 +++--- .../fastapi_routes/v2/metadata/common.py | 29 ++++++++++- .../v2/metadata/parameter_routes.py | 6 ++- tests/unit/v2/test_metadata_routes.py | 49 +++++++++++++++++++ 5 files changed, 92 insertions(+), 10 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 4f665f71d..a80f72ae6 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,4 +1,4 @@ -Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the next UTC midnight for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 7d0f0a36f..0b9873082 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -80,12 +80,16 @@ from the read and unknown parameters were ignored. Every successful metadata response carries a strong `ETag`, a hash of its exact body, and `Cache-Control: public`. A request that names its catalog with `policyengine_version` may be reused for a day (`max-age=86400`): a published -catalog version never changes, so only a change to the route's response format -can change such a response, and the `ETag` lets a client revalidate after the -day. A request without it selects the running version, which changes on -deploy, and may be reused for five minutes (`max-age=300`). A request whose -`If-None-Match` names the current `ETag` (strong or weak form, alone or in a -list) gets HTTP 304 with no body. Error responses carry neither header. +catalog version never changes, so such a response changes only with the +route's response format or with the day it is read on, and the `ETag` lets a +client revalidate after the day. A request without it selects the running +version, which changes on deploy, and may be reused for five minutes +(`max-age=300`). A response that depends on the request's UTC day, which is +`GET /v2/parameter-values?current=true` (the values in effect today), may be +reused only until the next UTC midnight, whichever lifetime would otherwise +apply. A request whose `If-None-Match` names the current `ETag` (strong or +weak form, alone or in a list) gets HTTP 304 with no body. Error responses +carry neither header. Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, for up to eight route, country and diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d44c66f8a..ce5314c0d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -4,6 +4,7 @@ import contextlib import hashlib +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, TypeVar from fastapi.routing import APIRoute @@ -87,6 +88,26 @@ """ +DEPENDS_ON_UTC_DAY = "depends_on_utc_day" +"""The ``request.state`` attribute a route sets when its result depends on the day. + +Such a response, for example current values on the request's UTC day, may be +reused only until the next UTC midnight, whatever version it names. +""" + + +def utc_now() -> datetime: + """Return the current time in UTC; tests replace it to fix the clock.""" + return datetime.now(UTC) + + +def _seconds_until_next_utc_day(moment: datetime) -> int: + next_day = (moment + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + return max(int((next_day - moment).total_seconds()), 0) + + class CacheableMetadataRoute(APIRoute): """A metadata route whose successful responses carry caching headers. @@ -112,8 +133,10 @@ def with_cache_headers(request: Request, response: Response) -> Response: The ETag is a hash of the exact body, so it changes with any change to the content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the - running version for five minutes. A request whose ``If-None-Match`` names - the ETag gets a 304 with no body. + running version for five minutes, and one whose route marked it as + depending on the day (``DEPENDS_ON_UTC_DAY``) no later than the next UTC + midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with + no body. """ entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' max_age = ( @@ -121,6 +144,8 @@ def with_cache_headers(request: Request, response: Response) -> Response: if "policyengine_version" in request.query_params else RUNNING_VERSION_MAX_AGE_SECONDS ) + if getattr(request.state, DEPENDS_ON_UTC_DAY, False): + max_age = min(max_age, _seconds_until_next_utc_day(utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): return Response(status_code=304, headers=headers) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 252682bb0..ceb22e985 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -5,11 +5,12 @@ from typing import TYPE_CHECKING, Annotated from uuid import UUID -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( + DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, @@ -133,8 +134,11 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( + request: Request, query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: + # Current values are those in effect on the request's UTC day. + setattr(request.state, DEPENDS_ON_UTC_DAY, query.current) return read_resource( dependencies, MetadataParameterValuePageResponse, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index c3b50b6e5..d77571581 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1106,3 +1106,52 @@ def test_default_dependencies_share_one_response_cache() -> None: assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) assert first.v2_metadata_response_cache is second.v2_metadata_response_cache + + +@pytest.mark.parametrize( + ("now", "query", "expected_cache_control"), + [ + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=true", + "public, max-age=3600", + ), + ( + datetime(2026, 12, 31, 23, 58, tzinfo=UTC), + "¤t=true", + "public, max-age=120", + ), + ( + datetime(2026, 6, 1, 12, 0, tzinfo=UTC), + "¤t=true", + "public, max-age=300", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=false", + "public, max-age=86400", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3", + "public, max-age=86400", + ), + ], +) +def test_current_parameter_values_are_reused_only_until_the_next_utc_day( + monkeypatch: pytest.MonkeyPatch, + now: datetime, + query: str, + expected_cache_control: str, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + monkeypatch.setattr(common, "utc_now", lambda: now) + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + f"/v2/parameter-values?country_id=us{query}" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == expected_cache_control From 07383802ca97f938ac12a64f3391d055e6f65975 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:51:13 +0400 Subject: [PATCH 09/33] Read v2 metadata in autocommit sessions on the shared pool With the production engine options and a warm pool, the database's statement log showed five statements for each metadata request: the pool's liveness check, BEGIN, the catalog selection, the read and ROLLBACK. Each metadata read is one statement on rows a published catalog version never changes, so it needs no transaction. The default metadata reader now opens its session from a process-owned read-only factory bound to the v2 engine in autocommit mode (sharing its pool), so psycopg sends no BEGIN and no ROLLBACK: three statements per request. Other v2 sessions keep the ordinary mode, which the pool restores when a connection returns. The pool's liveness check is unchanged. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 1 + docs/migration/stage-9-v2-metadata.md | 12 +++++++ policyengine_api/data/v2/database.py | 35 ++++++++++++++++++- .../fastapi_routes/dependencies.py | 4 +-- tests/integration/test_v2_metadata_routes.py | 33 ++++++++++++++++- tests/unit/v2/test_database.py | 23 ++++++++++++ tests/unit/v2/test_metadata_routes.py | 4 ++- 7 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md new file mode 100644 index 000000000..2392c3e57 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -0,0 +1 @@ +v2 metadata reads run in autocommit read-only sessions on the shared pool, so each request sends no BEGIN or ROLLBACK to the database (two fewer round trips per request). diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0b9873082..ce3efa833 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -75,6 +75,18 @@ an unsupported `country_id` and a malformed `policyengine_version` are rejected with HTTP 422 before any read; before, the last two returned HTTP 400 from the read and unknown parameters were ignored. +## Database round trips per request + +Metadata reads use read-only sessions in autocommit mode on the shared v2 +pool (`get_v2_read_session_factory`): each read is one statement on rows a +published catalog version never changes, so no transaction is needed, and the +database receives no `BEGIN` before the reads and no `ROLLBACK` when the +session closes. Measured with statement logging in a local database, a page of +500 parameters or one parameter's values took five statements per request +before (the pool's liveness check, `BEGIN`, the catalog selection, the read +and `ROLLBACK`) and three after. The pool's liveness check +(`pool_pre_ping`) is unchanged. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 07b88ad1a..3885ab94d 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -28,7 +28,7 @@ @dataclass class _V2EngineState: - """The process-owned v2 engine and Session factory. + """The process-owned v2 engine and Session factories. ``engine_pid`` is the ID of the process that created the engine, and ``engine_url_fingerprint`` identifies its target. ``_state_lock`` guards @@ -39,6 +39,7 @@ class _V2EngineState: engine_pid: int | None = None engine_url_fingerprint: tuple[object, ...] | None = None session_factory: sessionmaker[Session] | None = None + read_session_factory: sessionmaker[Session] | None = None _state_lock = Lock() @@ -82,6 +83,22 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: + """Build a factory of read-only Sessions that run without a transaction. + + The sessions share ``engine``'s pool in autocommit mode, so psycopg sends + each statement on its own: no BEGIN before the first read and no ROLLBACK + when the session closes. Use them only for reads whose statements need no + common snapshot, such as reads of a published catalog version, which never + changes. The pool restores the ordinary mode when a connection returns. + """ + return sessionmaker( + bind=engine.execution_options(isolation_level="AUTOCOMMIT"), + class_=Session, + expire_on_commit=False, + ) + + def _discard_inherited_state(current_pid: int) -> None: """Drop pool state inherited through a process fork without touching parent FDs.""" if _engine_state.engine is not None: @@ -90,6 +107,7 @@ def _discard_inherited_state(current_pid: int) -> None: _engine_state.engine_pid = current_pid _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None def get_v2_engine(settings: V2DatabaseSettings | None = None) -> Engine: @@ -130,6 +148,20 @@ def get_v2_session_factory( return _engine_state.session_factory +def get_v2_read_session_factory( + settings: V2DatabaseSettings | None = None, +) -> sessionmaker[Session]: + """Return the process-owned read-only Session factory without opening one. + + See ``build_v2_read_session_factory``. + """ + engine = get_v2_engine(settings) + with _state_lock: + if _engine_state.read_session_factory is None: + _engine_state.read_session_factory = build_v2_read_session_factory(engine) + return _engine_state.read_session_factory + + def close_v2_database() -> None: """Dispose this process's v2 pool and forget its lazy configuration.""" with _state_lock: @@ -139,6 +171,7 @@ def close_v2_database() -> None: _engine_state.engine_pid = None _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None atexit.register(close_v2_database) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 43f6fbf4f..073052be7 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -784,7 +784,7 @@ def _process_metadata_response_cache() -> MetadataResponseCache: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( - get_v2_session_factory, + get_v2_read_session_factory, ) from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, @@ -792,7 +792,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.services.v2.metadata.services import V2MetadataService return V2MetadataService( - MetadataDatabaseSession(get_v2_session_factory()()), + MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), ) diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 5f5e6af26..e9531fd3f 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -12,7 +12,7 @@ from alembic.config import Config from fastapi.testclient import TestClient from flask import Flask, jsonify -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import Engine, make_url from sqlalchemy.pool import NullPool from sqlmodel import Session, select @@ -543,3 +543,34 @@ def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( ] assert result["node_labels"] == {"gov.example": "Example policy"} assert result["missing_names"] == ["gov.removed.amount"] + + +def test_postgres_metadata_reads_send_no_transaction_statements( + published_engine: Engine, +) -> None: + # Each metadata read is one statement on rows a published catalog never + # changes, so the read session runs in autocommit mode: psycopg then sends + # no BEGIN before the reads and no ROLLBACK when the session closes. + from policyengine_api.data.v2.database import build_v2_read_session_factory + + autocommit_modes: list[bool] = [] + + def record_mode(connection, _cursor, _statement, *_args) -> None: + autocommit_modes.append(connection.connection.dbapi_connection.autocommit) + + event.listen(published_engine, "before_cursor_execute", record_mode) + try: + service = V2MetadataService( + MetadataDatabaseSession(build_v2_read_session_factory(published_engine)()), + running_policyengine_version=POLICYENGINE_VERSION, + ) + result = service.list_parameters("us") + service.close() + finally: + event.remove(published_engine, "before_cursor_execute", record_mode) + + assert [item.name for item in result.items] == ["gov.example.rate"] + assert autocommit_modes + assert all(autocommit_modes) + with published_engine.connect() as connection: + assert connection.connection.dbapi_connection.autocommit is False diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index a00667a49..c1984199c 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -125,3 +125,26 @@ def test_forked_process_state_builds_a_new_engine( assert child_engine is not parent_engine assert database._engine_state.engine_pid == parent_pid + 1 + + +def test_read_sessions_share_the_runtime_pool_without_transactions() -> None: + settings = load_v2_runtime_database_settings(_environment()) + + read_factory = database.get_v2_read_session_factory(settings) + runtime_engine = database.get_v2_engine(settings) + + with read_factory() as session: + bind = session.get_bind() + assert bind.get_execution_options()["isolation_level"] == "AUTOCOMMIT" + assert bind.pool is runtime_engine.pool + assert database.get_v2_read_session_factory(settings) is read_factory + assert runtime_engine.get_execution_options().get("isolation_level") is None + + +def test_read_session_factory_is_rebuilt_with_the_engine() -> None: + settings = load_v2_runtime_database_settings(_environment()) + first = database.get_v2_read_session_factory(settings) + + database.close_v2_database() + + assert database.get_v2_read_session_factory(settings) is not first diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index d77571581..707345d09 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -517,7 +517,9 @@ def query_service(candidate_session, *, running_policyengine_version): captured["version"] = running_policyengine_version return reader - monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) + monkeypatch.setattr( + database, "get_v2_read_session_factory", lambda: lambda: session + ) monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, From bb2071a40eaf6d8230c008d1a61cb85f22681d86 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:57:40 +0400 Subject: [PATCH 10/33] Keep each selected catalog version for the life of the process After the autocommit change, each metadata request still sent three statements: the pool's liveness check, the catalog selection and the read. A published catalog version never changes, so the default metadata reader now keeps every successful selection in a process-wide SelectedCatalogStore, keyed by country and requested version, and later requests skip the selection query: two statements per request in the local statement log. Unsupported countries, invalid versions and absent catalogs are looked up, and fail, each time. Tests pass their own store or none. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 13 ++-- .../fastapi_routes/dependencies.py | 10 +++ .../services/v2/metadata/selected_catalogs.py | 41 ++++++++++ .../services/v2/metadata/services.py | 21 ++++- tests/unit/v2/test_metadata_routes.py | 15 +++- tests/unit/v2/test_metadata_service.py | 78 ++++++++++++++++++- 7 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 policyengine_api/services/v2/metadata/selected_catalogs.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md index 2392c3e57..8eff7221b 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -1 +1 @@ -v2 metadata reads run in autocommit read-only sessions on the shared pool, so each request sends no BEGIN or ROLLBACK to the database (two fewer round trips per request). +v2 metadata reads run in autocommit read-only sessions on the shared pool, and each process keeps the catalog versions it has selected, so a metadata request sends two statements to the database (the pool's liveness check and the read) instead of five. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index ce3efa833..d511af3aa 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -81,11 +81,14 @@ Metadata reads use read-only sessions in autocommit mode on the shared v2 pool (`get_v2_read_session_factory`): each read is one statement on rows a published catalog version never changes, so no transaction is needed, and the database receives no `BEGIN` before the reads and no `ROLLBACK` when the -session closes. Measured with statement logging in a local database, a page of -500 parameters or one parameter's values took five statements per request -before (the pool's liveness check, `BEGIN`, the catalog selection, the read -and `ROLLBACK`) and three after. The pool's liveness check -(`pool_pre_ping`) is unchanged. +session closes. Each process also keeps every catalog version a request has +selected (`SelectedCatalogStore`), since a published version never changes, so +later requests for the same country and version skip the selection query; +failed selections are not kept. Measured with statement logging in a local +database, a page of 500 parameters or one parameter's values took five +statements per request before (the pool's liveness check, `BEGIN`, the catalog +selection, the read and `ROLLBACK`) and two after (the liveness check and the +read). The pool's liveness check (`pool_pre_ping`) is unchanged. ## Response caching diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 073052be7..0be11f8fc 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -16,6 +16,9 @@ from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -776,6 +779,12 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_selected_catalogs() -> SelectedCatalogStore: + """Return the catalog versions this process's metadata reads have selected.""" + return SelectedCatalogStore() + + @lru_cache(maxsize=1) def _process_metadata_response_cache() -> MetadataResponseCache: """Return the whole catalog-version responses this process keeps.""" @@ -794,6 +803,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: return V2MetadataService( MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), + selected_catalogs=_process_selected_catalogs(), ) diff --git a/policyengine_api/services/v2/metadata/selected_catalogs.py b/policyengine_api/services/v2/metadata/selected_catalogs.py new file mode 100644 index 000000000..832705432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/selected_catalogs.py @@ -0,0 +1,41 @@ +"""A process-wide store of the catalog versions metadata reads have selected.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + +CatalogSelectionKey = tuple[str, str | None] +"""A country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + + +class SelectedCatalogStore: + """Keep each catalog version a request has selected, for the process's life. + + A published catalog version never changes, so once a request has found it, + later requests for the same country and version need not query for it + again. Only successful selections are kept: an unsupported country, an + invalid version or an absent catalog is looked up, and fails, each time. + The kept records are detached, fully loaded rows that are only read. + """ + + def __init__(self) -> None: + """Start with no selections kept.""" + self._selections: dict[CatalogSelectionKey, SelectedCatalog] = {} + self._lock = threading.Lock() + + def get(self, key: CatalogSelectionKey) -> SelectedCatalog | None: + """Return the selection kept for ``key``, or None.""" + with self._lock: + return self._selections.get(key) + + def keep(self, key: CatalogSelectionKey, selected: SelectedCatalog) -> None: + """Keep ``selected`` as the selection for ``key``.""" + with self._lock: + self._selections[key] = selected diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index cdae4758e..c5a63a223 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -91,6 +91,9 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) class V2MetadataService: @@ -107,12 +110,18 @@ def __init__( database_session: MetadataDatabaseSession, *, running_policyengine_version: str, + selected_catalogs: SelectedCatalogStore | None = None, ) -> None: - """Bind to a request's session and the running PolicyEngine.py version.""" + """Bind to a request's session and the running PolicyEngine.py version. + + With ``selected_catalogs``, a catalog version found once is taken from + that store instead of being queried again. + """ self._database_session = database_session self._running_policyengine_version = validate_policyengine_version( running_policyengine_version ) + self._selected_catalogs = selected_catalogs def close(self) -> None: """Close the request's database session.""" @@ -121,12 +130,20 @@ def close(self) -> None: def _select_catalog( self, country_id: str, policyengine_version: str | None ) -> SelectedCatalog: - return read_metadata_catalog( + key = (country_id, policyengine_version) + if self._selected_catalogs is not None: + kept = self._selected_catalogs.get(key) + if kept is not None: + return kept + selected = read_metadata_catalog( self._database_session.session, country_id=country_id, running_policyengine_version=self._running_policyengine_version, policyengine_version=policyengine_version, ) + if self._selected_catalogs is not None: + self._selected_catalogs.keep(key, selected) + return selected def _select_paginated_catalog( self, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 707345d09..96d504804 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -27,6 +27,9 @@ RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataCatalogParameter, @@ -512,9 +515,12 @@ def test_default_reader_uses_the_installed_policyengine_version( captured = {} reader = object() - def query_service(candidate_session, *, running_policyengine_version): + def query_service( + candidate_session, *, running_policyengine_version, selected_catalogs + ): captured["session"] = candidate_session.session captured["version"] = running_policyengine_version + captured["selected_catalogs"] = selected_catalogs return reader monkeypatch.setattr( @@ -533,7 +539,12 @@ def query_service(candidate_session, *, running_policyengine_version): route_dependencies._running_policyengine_version.cache_clear() assert result is reader - assert captured == {"session": session, "version": "5.2.0"} + assert captured["session"] is session + assert captured["version"] == "5.2.0" + assert isinstance(captured["selected_catalogs"], SelectedCatalogStore) + assert ( + route_dependencies._process_selected_catalogs() is captured["selected_catalogs"] + ) def test_default_factory_and_close_failures_preserve_success( diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index fc0827b06..d07e94bde 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -33,6 +33,9 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( @@ -239,10 +242,13 @@ def _us_model_version(session: Session) -> TaxBenefitModelVersion: ).one() -def _service(session: Session) -> V2MetadataService: +def _service( + session: Session, *, selected_catalogs: SelectedCatalogStore | None = None +) -> V2MetadataService: return V2MetadataService( MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, ) @@ -1009,6 +1015,76 @@ def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] +def test_a_kept_catalog_selection_saves_its_query_on_later_reads( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + + first, first_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + second, second_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + + assert second == first + assert len(first_statements) == 2 + assert len(second_statements) == 1 + assert "tax_benefit_model_versions" not in second_statements[0] + + +def test_failed_catalog_selections_are_not_kept(catalog_session: Session) -> None: + store = SelectedCatalogStore() + service = _service(catalog_session, selected_catalogs=store) + + for _attempt in range(2): + with pytest.raises(MetadataCatalogVersionNotFoundError): + service.list_parameters("us", "9.9.9") + with pytest.raises(UnsupportedPreviewCountryError): + service.list_parameters("ca") + with pytest.raises(InvalidPolicyEngineVersionError): + service.list_parameters("us", "not a version") + + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + assert service.list_parameters("us", "9.9.9").policyengine_version == "9.9.9" + + +def test_each_country_and_version_keeps_its_own_selection( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + service = _service(catalog_session, selected_catalogs=store) + + running_us = service.get_model_by_country("us") + named_us = service.get_model_by_country("us", "9.9.9") + running_uk = service.get_model_by_country("uk") + + assert (running_us.policyengine_version, named_us.policyengine_version) == ( + POLICYENGINE_VERSION, + "9.9.9", + ) + assert named_us.model_version.current_law_id == 7 + assert running_uk.model.name == "policyengine-uk" + assert service.get_model_by_country("us", "9.9.9") == named_us + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 15355a4efb32997a682677724ee93d7453842e5d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:01:24 +0400 Subject: [PATCH 11/33] Keep each whole-version response's ETag, under the version it was read from The response cache kept only bodies, so the caching route hashed the 3.9 MB US variables body again on every request, and a request without a version and one naming the running version kept two copies. Each kept response now carries its ETag, computed once when it is kept, which the caching route reuses, and responses are kept under the exact version they were read from: a request without a version records which version the process runs, so it and an explicit request for that version share one response. The route helper is generic over the result type, so mypy checks each operation's result against its response model. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 13 +- .../v2/metadata/catalog_routes.py | 15 +- .../fastapi_routes/v2/metadata/common.py | 66 ++++++--- .../v2/metadata/response_cache.py | 133 +++++++++++++----- tests/unit/v2/test_metadata_response_cache.py | 93 ++++++++---- tests/unit/v2/test_metadata_routes.py | 36 ++++- 6 files changed, 258 insertions(+), 98 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d511af3aa..d8978e7cb 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -107,12 +107,13 @@ weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. Each API process also keeps the serialized bodies of `/v2/model-catalog` and -`/v2/model-catalog/variables` once built, for up to eight route, country and -requested-version combinations (least recently used dropped first; a request -without `policyengine_version` is keyed to the process's running version, -which never changes while it runs). Only the first successful request per -process reads the database; concurrent first requests wait for one read, and -a failed read is not kept. Measured locally against the full 6.10.0 US +`/v2/model-catalog/variables` once built, each with its `ETag` computed once, +for up to eight route, country and version combinations (least recently used +dropped first). A response is kept under the exact version it was read from, +so a request without `policyengine_version` (the process's running version, +which never changes while it runs) and one naming that version share it. Only +the first successful request per process reads the database; concurrent first +requests wait for one read, and a failed read is not kept. Measured locally against the full 6.10.0 US catalog, the variables route takes about 160 ms when read and about 25 ms when kept, most of it compression. diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index bef2ba326..6f2259785 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -18,8 +18,13 @@ MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, + MetadataResourceSuccessResponse, ) from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery +from policyengine_api.services.v2.metadata.types import ( + MetadataCatalogVariablesResult, + MetadataModelCatalogResult, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -61,10 +66,11 @@ def build_v2_model_catalog_router( ) def get_model_catalog( query: CatalogQueryDependency, - ) -> MetadataModelCatalogResponse | Response: + ) -> MetadataResourceSuccessResponse[MetadataModelCatalogResult] | Response: return read_whole_version_resource( dependencies, - ("model-catalog", query.country_id, query.policyengine_version), + "model-catalog", + query, MetadataModelCatalogResponse, lambda reader: reader.get_model_catalog( query.country_id, query.policyengine_version @@ -85,10 +91,11 @@ def get_model_catalog( ) def get_catalog_variables( query: CatalogQueryDependency, - ) -> MetadataCatalogVariablesResponse | Response: + ) -> MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] | Response: return read_whole_version_resource( dependencies, - ("model-catalog/variables", query.country_id, query.policyengine_version), + "model-catalog/variables", + query, MetadataCatalogVariablesResponse, lambda reader: reader.get_catalog_variables( query.country_id, query.policyengine_version diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index ce5314c0d..d10f4072b 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,8 @@ from __future__ import annotations import contextlib -import hashlib from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Protocol, TypeVar from fastapi.routing import APIRoute from pydantic import BaseModel @@ -23,6 +22,14 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + body_entity_tag, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataResourceSuccessResponse, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -38,9 +45,7 @@ NativeRouteDependencies, V2MetadataResourceReader, ) - from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( - ResponseKey, - ) + from policyengine_api.query_parameters import CatalogQuery ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { @@ -70,8 +75,18 @@ } +class CatalogVersionResult(Protocol): + """A whole-version result, which names the catalog version it was read from.""" + + @property + def policyengine_version(self) -> str: + """The exact PolicyEngine.py version of the catalog read.""" + ... + + ResponseT = TypeVar("ResponseT", bound=BaseModel) ReadT = TypeVar("ReadT") +CatalogResultT = TypeVar("CatalogResultT", bound=CatalogVersionResult) EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 """How long a response for a named catalog version may be reused (a day). @@ -138,7 +153,7 @@ def with_cache_headers(request: Request, response: Response) -> Response: midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with no body. """ - entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' + entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( EXPLICIT_VERSION_MAX_AGE_SECONDS if "policyengine_version" in request.query_params @@ -180,31 +195,42 @@ def read_resource( def read_whole_version_resource( dependencies: NativeRouteDependencies, - key: ResponseKey, - response_type: type[ResponseT], - operation: Callable[[V2MetadataResourceReader], object], -) -> ResponseT | Response: + route: str, + query: CatalogQuery, + response_type: type[MetadataResourceSuccessResponse[CatalogResultT]], + operation: Callable[[V2MetadataResourceReader], CatalogResultT], +) -> MetadataResourceSuccessResponse[CatalogResultT] | Response: """Serve a whole catalog-version response, built once per process when kept. - With the dependencies' response cache, the serialized body for ``key`` is - read and built only on the first successful request; failures are served - as ``read_resource`` serves them and never kept. Without a cache this is + With the dependencies' response cache, the serialized body and its ETag + for ``route``, the query's country and the version it selects are read + and built only on the first successful request; failures are served as + ``read_resource`` serves them and never kept. Without a cache this is ``read_resource``. """ cache = dependencies.v2_metadata_response_cache if cache is None: return read_resource(dependencies, response_type, operation) - def build() -> bytes | JSONResponse: + def build() -> BuiltResponse | JSONResponse: result = read_resource(dependencies, response_type, operation) - if isinstance(result, BaseModel): - return result.model_dump_json().encode() + if isinstance(result, MetadataResourceSuccessResponse): + return BuiltResponse( + body=result.model_dump_json().encode(), + policyengine_version=result.result.policyengine_version, + ) return result - built = cache.get_or_build(key, build) - if isinstance(built, bytes): - return Response(content=built, media_type="application/json") - return built + kept = cache.get_or_build( + route, query.country_id, query.policyengine_version, build + ) + if isinstance(kept, KeptResponse): + return Response( + content=kept.body, + media_type="application/json", + headers={"ETag": kept.entity_tag}, + ) + return kept def run_metadata_read( diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index ae688ae4c..72d3cf9ba 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -1,9 +1,11 @@ -"""A bounded, process-wide store of whole catalog-version response bodies.""" +"""A bounded, process-wide store of whole catalog-version responses.""" from __future__ import annotations +import hashlib import threading from collections import OrderedDict +from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -11,63 +13,120 @@ from starlette.responses import JSONResponse -ResponseKey = tuple[str, str, str | None] -"""A route name, a country ID and the requested PolicyEngine.py version. +DEFAULT_RESPONSE_CAPACITY = 8 -A version of None means the running version, which is fixed for the process. -""" +KeptResponseKey = tuple[str, str, str] +"""A route name, a country ID and the exact PolicyEngine.py version served.""" + + +def body_entity_tag(body: bytes) -> str: + """Return the strong ETag of a response body: a hash of its exact bytes.""" + return f'"{hashlib.sha256(body).hexdigest()[:32]}"' + + +@dataclass(frozen=True) +class BuiltResponse: + """A serialized success body and the catalog version it was read from.""" + + body: bytes + policyengine_version: str -DEFAULT_RESPONSE_CAPACITY = 8 + +@dataclass(frozen=True) +class KeptResponse: + """A kept success body with its ETag, computed once when it was kept.""" + + body: bytes + entity_tag: str class MetadataResponseCache: - """Keep the serialized bodies of whole-version responses, built once each. + """Keep whole-version responses, built once each per process. A published catalog version never changes, so a body built for a route, - country and version stays valid for the life of the process. The most - recently used ``capacity`` bodies are kept. Requests for a body being built - wait for that build instead of starting another, and a failed read (a v2 - error response) is returned but not kept, so the next request tries again. + country and version stays valid for the life of the process. Responses + are kept under the exact version they were read from: a request without a + version (the running version, fixed for the process) and one naming that + version share one response. The most recently used ``capacity`` responses + are kept. Requests for a response being built wait for that build instead + of starting another, and a failed read (a v2 error response) is returned + but not kept, so the next request tries again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: - """Keep at most ``capacity`` bodies (at least one).""" + """Keep at most ``capacity`` responses (at least one).""" if capacity < 1: msg = "capacity must be positive" raise ValueError(msg) self._capacity = capacity - self._bodies: OrderedDict[ResponseKey, bytes] = OrderedDict() + self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() + self._running_versions: dict[tuple[str, str], str] = {} self._lock = threading.Lock() - self._build_locks: dict[ResponseKey, threading.Lock] = {} + self._build_locks: dict[tuple[str, str, str | None], threading.Lock] = {} - def _stored(self, key: ResponseKey) -> bytes | None: + def _stored( + self, route: str, country_id: str, requested_version: str | None + ) -> KeptResponse | None: with self._lock: - body = self._bodies.get(key) - if body is not None: - self._bodies.move_to_end(key) - return body + version = ( + requested_version + if requested_version is not None + else self._running_versions.get((route, country_id)) + ) + if version is None: + return None + key = (route, country_id, version) + kept = self._responses.get(key) + if kept is not None: + self._responses.move_to_end(key) + return kept def get_or_build( self, - key: ResponseKey, - build: Callable[[], bytes | JSONResponse], - ) -> bytes | JSONResponse: - """Return the body kept for ``key``, calling ``build`` only when none is.""" - body = self._stored(key) - if body is not None: - return body + route: str, + country_id: str, + requested_version: str | None, + build: Callable[[], BuiltResponse | JSONResponse], + ) -> KeptResponse | JSONResponse: + """Return the kept response, calling ``build`` only when none is kept. + + ``requested_version`` None means the running version. + """ + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + build_key = (route, country_id, requested_version) with self._lock: - build_lock = self._build_locks.setdefault(key, threading.Lock()) + build_lock = self._build_locks.setdefault(build_key, threading.Lock()) with build_lock: - body = self._stored(key) - if body is not None: - return body + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept built = build() with self._lock: - self._build_locks.pop(key, None) - if isinstance(built, bytes): - self._bodies[key] = built - self._bodies.move_to_end(key) - while len(self._bodies) > self._capacity: - self._bodies.popitem(last=False) - return built + self._build_locks.pop(build_key, None) + if not isinstance(built, BuiltResponse): + return built + return self._keep(route, country_id, requested_version, built) + + def _keep( + self, + route: str, + country_id: str, + requested_version: str | None, + built: BuiltResponse, + ) -> KeptResponse: + key = (route, country_id, built.policyengine_version) + with self._lock: + if requested_version is None: + self._running_versions[(route, country_id)] = built.policyengine_version + kept = self._responses.get(key) + if kept is None: + kept = KeptResponse( + body=built.body, entity_tag=body_entity_tag(built.body) + ) + self._responses[key] = kept + self._responses.move_to_end(key) + while len(self._responses) > self._capacity: + self._responses.popitem(last=False) + return kept diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index a96d7b005..9cfc8fd8c 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -1,4 +1,4 @@ -"""The process-wide store of whole catalog-version response bodies.""" +"""The process-wide store of whole catalog-version responses.""" from __future__ import annotations @@ -9,42 +9,79 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, MetadataResponseCache, + body_entity_tag, ) +RUNNING_VERSION = "6.10.0" -def _counting_build(body: bytes, builds: list[str]): - def build() -> bytes: + +def _counting_build( + builds: list[str], *, body: bytes = b'{"a":1}', version: str = RUNNING_VERSION +): + def build() -> BuiltResponse: builds.append("built") - return body + return BuiltResponse(body=body, policyengine_version=version) return build -def test_a_body_is_built_once_per_key_and_reused() -> None: +def test_a_response_is_built_once_and_kept_with_its_entity_tag() -> None: cache = MetadataResponseCache() builds: list[str] = [] - key = ("variables", "us", "6.10.0") - first = cache.get_or_build(key, _counting_build(b'{"a":1}', builds)) - second = cache.get_or_build(key, _counting_build(b'{"b":2}', builds)) + first = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds, body=b"{}") + ) - assert first == b'{"a":1}' - assert second == b'{"a":1}' + assert first == KeptResponse( + body=b'{"a":1}', entity_tag=body_entity_tag(b'{"a":1}') + ) + assert second is first assert builds == ["built"] +@pytest.mark.parametrize("requested_first", [None, RUNNING_VERSION]) +def test_the_running_version_and_its_number_share_one_kept_response( + requested_first: str | None, +) -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + requested_second = RUNNING_VERSION if requested_first is None else None + + first = cache.get_or_build( + "variables", "us", requested_first, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + third = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + + assert second is first + assert third is first + assert builds == (["built"] if requested_first is None else ["built", "built"]) + + def test_different_routes_countries_and_versions_are_kept_apart() -> None: cache = MetadataResponseCache() builds: list[str] = [] - for key in ( + for route, country_id, version in ( ("variables", "us", "6.10.0"), ("variables", "uk", "6.10.0"), - ("variables", "us", None), + ("variables", "us", "6.9.0"), ("model-catalog", "us", "6.10.0"), ): - cache.get_or_build(key, _counting_build(repr(key).encode(), builds)) + cache.get_or_build( + route, country_id, version, _counting_build(builds, version=version) + ) assert len(builds) == 4 @@ -58,44 +95,40 @@ def failing_build() -> JSONResponse: builds.append("failed") return failure - first = cache.get_or_build(("variables", "us", None), failing_build) - second = cache.get_or_build( - ("variables", "us", None), _counting_build(b"{}", builds) - ) + first = cache.get_or_build("variables", "us", None, failing_build) + second = cache.get_or_build("variables", "us", None, _counting_build(builds)) assert first is failure - assert second == b"{}" + assert isinstance(second, KeptResponse) assert builds == ["failed", "built"] -def test_the_least_recently_used_body_is_dropped_beyond_capacity() -> None: +def test_the_least_recently_used_response_is_dropped_beyond_capacity() -> None: cache = MetadataResponseCache(capacity=2) builds: list[str] = [] - oldest, middle, newest = (("variables", country, None) for country in "abc") - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(middle, _counting_build(b"2", builds)) - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(newest, _counting_build(b"3", builds)) - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(middle, _counting_build(b"2", builds)) + def get(country_id: str) -> None: + cache.get_or_build("variables", country_id, "6.10.0", _counting_build(builds)) + + for country_id in ("a", "b", "a", "c", "a", "b"): + get(country_id) assert builds == ["built", "built", "built", "built"] -def test_concurrent_requests_for_one_key_build_it_once() -> None: +def test_concurrent_requests_for_one_response_build_it_once() -> None: cache = MetadataResponseCache() builds: list[str] = [] started = threading.Barrier(4) - def slow_build() -> bytes: + def slow_build() -> BuiltResponse: builds.append("built") time.sleep(0.05) - return b"{}" + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) def request() -> None: started.wait() - cache.get_or_build(("variables", "us", None), slow_build) + cache.get_or_build("variables", "us", None, slow_build) threads = [threading.Thread(target=request) for _ in range(4)] for thread in threads: diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 96d504804..c23ca5099 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1087,7 +1087,10 @@ def factory() -> ResourceReader: first_responses = [client.get(path) for path in paths] repeated_responses = [client.get(path) for path in paths] - assert len(readers) == len(paths) + # The explicit request names the running version (4.20.3), so it shares + # the response the request without a version read: three reads in all. + assert len(readers) == len(paths) - 1 + assert first_responses[2].content == first_responses[1].content for first, repeated in zip(first_responses, repeated_responses, strict=True): assert repeated.status_code == 200 assert repeated.content == first.content @@ -1168,3 +1171,34 @@ def test_current_parameter_values_are_reused_only_until_the_next_utc_day( assert response.status_code == 200 assert response.headers["cache-control"] == expected_cache_control + + +def test_a_kept_response_is_served_without_hashing_its_body_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common, response_cache + + hashed_bodies: list[int] = [] + original_entity_tag = response_cache.body_entity_tag + + def counting_entity_tag(body: bytes) -> str: + hashed_bodies.append(len(body)) + return original_entity_tag(body) + + monkeypatch.setattr(response_cache, "body_entity_tag", counting_entity_tag) + monkeypatch.setattr(common, "body_entity_tag", counting_entity_tag) + results, _ids = _resource_results() + client = _client( + lambda: ResourceReader(results), response_cache=MetadataResponseCache() + ) + + first = client.get("/v2/model-catalog/variables?country_id=us") + hashed_after_first = len(hashed_bodies) + second = client.get( + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert hashed_after_first == 1 + assert len(hashed_bodies) == 1 + assert second.headers["etag"] == first.headers["etag"] + assert second.content == first.content From 038d593212db39b4032e0ef07965debc95cfab28 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:18:37 +0400 Subject: [PATCH 12/33] Read every v2 metadata query through the shared query models A malformed policyengine_version returned 422 from the routes that already used the shared query models but 400 from /v2/variables, /v2/regions, /v2/datasets, the model routes, the detail routes, the parameter tree and /v2/economy-options. The API routes guidance makes the shared definitions own each parameter's validation, so every metadata route now reads its query through them (CatalogQuery, CatalogVersionQuery for the route with the country in its path, CatalogPageQuery and resource models for variables, regions and the parameter tree), with the typed dependencies defined once. Unknown and repeated scalar parameters, a missing or unsupported country_id and a malformed version are rejected with 422 before any read on every route. The release check and the migration document now expect 422. Co-Authored-By: Claude Opus 5.5 --- ...1f-116-catalog-routes-on-demand.changed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 20 +++-- .../v2/metadata/catalog_routes.py | 17 ++-- .../v2/metadata/geography_routes.py | 72 ++++++++--------- .../v2/metadata/model_routes.py | 80 ++++++++----------- .../v2/metadata/parameter_routes.py | 57 +++++-------- .../v2/metadata/query_dependencies.py | 53 ++++++++++++ policyengine_api/query_parameters.py | 41 +++++++++- tests/integration/test_live_v2_metadata.py | 2 +- tests/integration/test_v2_metadata_routes.py | 2 +- tests/unit/v2/test_metadata_routes.py | 71 ++++++++++++++++ 11 files changed, 272 insertions(+), 145 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md index 62dc783e3..553c9f234 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -1 +1 @@ -/v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. Both read their query through the shared query models, so unknown or repeated parameters, an unsupported country_id and a malformed policyengine_version now return 422. +Every v2 metadata route reads its query through the shared query models, so unknown or repeated parameters, a missing or unsupported country_id and a malformed policyengine_version return 422 before any read (unsupported countries and malformed versions returned 400 before, and unknown parameters were ignored). /v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d8978e7cb..2ecef27b2 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -68,12 +68,18 @@ carry the same `economy` and `household` flags. `GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page only those exact parameters, and `GET /v2/parameter-values` repeated -`parameter_name` keys to page only those parameters' values. Both routes now -read their query through the shared query models (`ParameterCollectionQuery` -and `ParameterValueCollectionQuery`): unknown and repeated scalar parameters, -an unsupported `country_id` and a malformed `policyengine_version` are -rejected with HTTP 422 before any read; before, the last two returned HTTP 400 -from the read and unknown parameters were ignored. +`parameter_name` keys to page only those parameters' values. + +Every v2 metadata route reads its query through the shared query models in +`policyengine_api.query_parameters` (`CatalogQuery`, `CatalogPageQuery` and the +resource-specific models built on them): unknown parameters, repeated scalar +parameters, a missing or unsupported `country_id`, a malformed +`policyengine_version` and out-of-range `offset` or `limit` values are +rejected with HTTP 422 before any read. Before, the routes other than the +`/v2/model-catalog` ones ignored unknown parameters, and an unsupported country +or malformed version returned HTTP 400 from the read. A country given in the +path (`/v2/tax-benefit-models/by-country/{country_id}`) that is not supported +still returns HTTP 400. ## Database round trips per request @@ -353,7 +359,7 @@ A successful response has HTTP 200, `status: "ok"`, `message: null`, and a typed `result`. Collection results contain `policyengine_version`, `items`, `offset`, `limit`, and `has_more`; detail results contain `policyengine_version` and `item`. A malformed or noncanonical explicit -version returns a typed HTTP 400 error. A valid explicit version that has not +version returns a typed HTTP 422 error. A valid explicit version that has not been published for the country returns a typed HTTP 404 error. An absent catalog for the candidate's installed default version returns a typed HTTP 503 service error. None of these outcomes reads or repairs v1. Unsupported diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 6f2259785..1ef94301e 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -2,25 +2,27 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING -from fastapi import APIRouter, Depends +from fastapi import APIRouter from starlette.responses import JSONResponse, Response -from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, read_resource, read_whole_version_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogParameterQueryDependency, + CatalogQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, MetadataResourceSuccessResponse, ) -from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery from policyengine_api.services.v2.metadata.types import ( MetadataCatalogVariablesResult, MetadataModelCatalogResult, @@ -29,13 +31,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -CatalogQueryDependency = Annotated[ - CatalogQuery, Depends(query_dependency(CatalogQuery)) -] -CatalogParameterQueryDependency = Annotated[ - CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) -] - def build_v2_model_catalog_router( dependencies: NativeRouteDependencies, diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index f5add9640..4665ea67c 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( @@ -13,6 +13,11 @@ CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + RegionCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, @@ -20,14 +25,10 @@ MetadataRegionDetailResponse, MetadataRegionPageResponse, ) -from policyengine_api.services.v2.metadata.types import MetadataRegionType if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_geography_router( dependencies: NativeRouteDependencies, @@ -49,19 +50,16 @@ def build_v2_metadata_geography_router( summary="List logical inputs from a selected PolicyEngine.py catalog", ) def list_datasets( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataDatasetPageResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetPageResponse, lambda reader: reader.list_datasets( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -72,16 +70,15 @@ def list_datasets( ) def get_dataset( dataset_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataDatasetDetailResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetDetailResponse, lambda reader: reader.get_dataset( - country_id, + query.country_id, dataset_id, - policyengine_version, + query.policyengine_version, ), ) @@ -91,21 +88,19 @@ def get_dataset( summary="List regions from a selected PolicyEngine.py catalog", ) def list_regions( - country_id: str, - policyengine_version: str | None = None, - region_type: MetadataRegionType | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: RegionCollectionQueryDependency, ) -> MetadataRegionPageResponse | JSONResponse: return read_resource( dependencies, MetadataRegionPageResponse, lambda reader: reader.list_regions( - country_id, - policyengine_version, - region_type=region_type.value if region_type is not None else None, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + region_type=( + None if query.region_type is None else query.region_type.value + ), + offset=query.offset, + limit=query.limit, ), ) @@ -116,16 +111,15 @@ def list_regions( ) def get_region_by_code( region_code: str, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region_by_code( - country_id, + query.country_id, region_code, - policyengine_version, + query.policyengine_version, ), ) @@ -136,16 +130,15 @@ def get_region_by_code( ) def get_region( region_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region( - country_id, + query.country_id, region_id, - policyengine_version, + query.policyengine_version, ), ) @@ -155,15 +148,14 @@ def get_region( summary="Get compact economy-selection options from a selected catalog", ) def get_economy_options( - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataEconomyOptionsResponse | JSONResponse: return read_resource( dependencies, MetadataEconomyOptionsResponse, lambda reader: reader.get_economy_options( - country_id, - policyengine_version, + query.country_id, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index cbb3a8ef5..63e72ce56 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( @@ -13,6 +13,12 @@ CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + CatalogVersionQueryDependency, + VariableCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, @@ -26,9 +32,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_model_router( dependencies: NativeRouteDependencies, @@ -50,19 +53,16 @@ def build_v2_metadata_model_router( summary="List models for one PolicyEngine.py catalog", ) def list_models( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelPageResponse, lambda reader: reader.list_models( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -73,14 +73,14 @@ def list_models( ) def get_model_by_country( country_id: str, - policyengine_version: str | None = None, + query: CatalogVersionQueryDependency, ) -> MetadataModelSelectionResponse | JSONResponse: return read_resource( dependencies, MetadataModelSelectionResponse, lambda reader: reader.get_model_by_country( country_id, - policyengine_version, + query.policyengine_version, ), ) @@ -91,16 +91,15 @@ def get_model_by_country( ) def get_model( model_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelDetailResponse, lambda reader: reader.get_model( - country_id, + query.country_id, model_id, - policyengine_version, + query.policyengine_version, ), ) @@ -110,19 +109,16 @@ def get_model( summary="List selected PolicyEngine.py model versions", ) def list_model_versions( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelVersionPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionPageResponse, lambda reader: reader.list_model_versions( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -133,16 +129,15 @@ def list_model_versions( ) def get_model_version( version_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelVersionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionDetailResponse, lambda reader: reader.get_model_version( - country_id, + query.country_id, version_id, - policyengine_version, + query.policyengine_version, ), ) @@ -152,21 +147,17 @@ def get_model_version( summary="List variables from a selected PolicyEngine.py catalog", ) def list_variables( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: VariableCollectionQueryDependency, ) -> MetadataVariablePageResponse | JSONResponse: return read_resource( dependencies, MetadataVariablePageResponse, lambda reader: reader.list_variables( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + search=query.search, ), ) @@ -177,16 +168,15 @@ def list_variables( ) def get_variable( variable_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataVariableDetailResponse | JSONResponse: return read_resource( dependencies, MetadataVariableDetailResponse, lambda reader: reader.get_variable( - country_id, + query.country_id, variable_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index ceb22e985..d9ddeb336 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,19 +2,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Depends, Query, Request +from fastapi import APIRouter, Request from starlette.responses import JSONResponse -from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogQueryDependency, + ParameterChildrenQueryDependency, + ParameterCollectionQueryDependency, + ParameterValueCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, MetadataParameterDetailResponse, @@ -22,10 +27,6 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.query_parameters import ( - ParameterCollectionQuery, - ParameterValueCollectionQuery, -) from policyengine_api.services.v2.metadata.types import ( ParameterFilter, ParameterValueFilter, @@ -34,18 +35,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - - -ParameterCollectionQueryDependency = Annotated[ - ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) -] -ParameterValueCollectionQueryDependency = Annotated[ - ParameterValueCollectionQuery, - Depends(query_dependency(ParameterValueCollectionQuery)), -] - def build_v2_metadata_parameter_router( dependencies: NativeRouteDependencies, @@ -90,21 +79,17 @@ def list_parameters( summary="List direct children of one parameter path", ) def list_parameter_children( - country_id: str, - parent_path: str = "", - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: ParameterChildrenQueryDependency, ) -> MetadataParameterChildPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterChildPageResponse, lambda reader: reader.list_parameter_children( - country_id, - policyengine_version, - parent_path=parent_path, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + parent_path=query.parent_path, + offset=query.offset, + limit=query.limit, ), ) @@ -115,16 +100,15 @@ def list_parameter_children( ) def get_parameter( parameter_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterDetailResponse, lambda reader: reader.get_parameter( - country_id, + query.country_id, parameter_id, - policyengine_version, + query.policyengine_version, ), ) @@ -166,16 +150,15 @@ def list_parameter_values( ) def get_parameter_value( value_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterValueDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterValueDetailResponse, lambda reader: reader.get_parameter_value( - country_id, + query.country_id, value_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py new file mode 100644 index 000000000..b52d18d1e --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py @@ -0,0 +1,53 @@ +"""Typed FastAPI dependencies for the shared v2 metadata query contracts. + +Route modules import these aliases into their own globals, because FastAPI +resolves an endpoint's string annotations against its module only. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CatalogPageQuery, + CatalogParameterQuery, + CatalogQuery, + CatalogVersionQuery, + ParameterChildrenQuery, + ParameterCollectionQuery, + ParameterValueCollectionQuery, + RegionCollectionQuery, + VariableCollectionQuery, +) + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] +CatalogVersionQueryDependency = Annotated[ + CatalogVersionQuery, Depends(query_dependency(CatalogVersionQuery)) +] +CatalogPageQueryDependency = Annotated[ + CatalogPageQuery, Depends(query_dependency(CatalogPageQuery)) +] +VariableCollectionQueryDependency = Annotated[ + VariableCollectionQuery, Depends(query_dependency(VariableCollectionQuery)) +] +RegionCollectionQueryDependency = Annotated[ + RegionCollectionQuery, Depends(query_dependency(RegionCollectionQuery)) +] +ParameterChildrenQueryDependency = Annotated[ + ParameterChildrenQuery, Depends(query_dependency(ParameterChildrenQuery)) +] +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index ec614f109..0d94c8484 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -30,6 +30,7 @@ validate_policyengine_version, ) from policyengine_api.services.economic_impact_request import LocalAreaBreakdown +from policyengine_api.services.v2.metadata.types import MetadataRegionType from policyengine_api.spm import SPMSelection from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, @@ -232,6 +233,42 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CatalogVersionQuery(StrictQueryParameters): + """An optional exact catalog version, for routes with the country in the path.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class CatalogPageQuery(CatalogQuery, PaginationQuery): + """Query contract for one page of a catalog's resources.""" + + +class VariableCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's variables.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + + +class RegionCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's regions.""" + + region_type: MetadataRegionType | None = Field( + default=None, description="Keep only regions of this type" + ) + + +class ParameterChildrenQuery(CatalogPageQuery): + """Query contract for a page of one parameter path's direct children.""" + + parent_path: str = Field( + default="", + description="Dotted path whose direct children are listed; empty for the root", + ) + + MAXIMUM_PARAMETER_NAMES = 100 ParameterName = Annotated[ str, @@ -249,7 +286,7 @@ class CatalogParameterQuery(CatalogQuery): ) -class ParameterCollectionQuery(CatalogQuery, PaginationQuery): +class ParameterCollectionQuery(CatalogPageQuery): """Query contract for a page of a catalog's parameters.""" search: str | None = Field( @@ -263,7 +300,7 @@ class ParameterCollectionQuery(CatalogQuery, PaginationQuery): ) -class ParameterValueCollectionQuery(CatalogQuery, PaginationQuery): +class ParameterValueCollectionQuery(CatalogPageQuery): """Query contract for a page of a catalog's current-law parameter values.""" parameter_id: ResourceId | None = None diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 803e43b05..1b2b8e183 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -110,7 +110,7 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: params={"country_id": "us", "policyengine_version": "not a version"}, ) - assert response.status_code == 400, response.text[:500] + assert response.status_code == 422, response.text[:500] payload = response.json() assert payload["status"] == "error" assert payload["message"] diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index e9531fd3f..fa5ea761b 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -328,7 +328,7 @@ def test_postgres_resources_distinguish_invalid_and_absent_versions( params={"country_id": "us", "policyengine_version": "4.99.0"}, ) - assert invalid.status_code == 400 + assert invalid.status_code == 422 assert invalid.json()["status"] == "error" assert invalid.json()["message"] assert absent.status_code == 404 diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index c23ca5099..d3b5ab349 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1202,3 +1202,74 @@ def counting_entity_tag(body: bytes) -> str: assert len(hashed_bodies) == 1 assert second.headers["etag"] == first.headers["etag"] assert second.content == first.content + + +def _query_rejections(path: str) -> list[str]: + separator = "&" if "?" in path else "?" + rejections = [ + f"{path}{separator}policyengine_version=not%20a%20version", + f"{path}{separator}unexpected=1", + f"{path}{separator}policyengine_version=4.20.3&policyengine_version=4.20.3", + ] + if "country_id=us" in path: + rejections.append(path.replace("country_id=us", "country_id=ca")) + rejections.append( + path.replace("country_id=us&", "").replace("?country_id=us", "") + ) + return rejections + + +def test_every_metadata_route_rejects_invalid_queries_before_reading() -> None: + _results, ids = _resource_results() + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + client = _client(factory) + for path in _every_resource_path(ids): + for rejected in _query_rejections(path): + response = client.get(rejected) + assert response.status_code == 422, (rejected, response.text) + assert response.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert calls == [] + + +def test_every_metadata_route_declares_the_shared_catalog_query() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + parameters = { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + if parameter["in"] == "query" + } + version = parameters["policyengine_version"] + assert version["required"] is False, path + assert version["schema"]["anyOf"][0]["maxLength"] == 128, path + if path != "/v2/tax-benefit-models/by-country/{country_id}": + country = parameters["country_id"] + assert country["required"] is True, path + assert country["schema"]["enum"] == ["us", "uk"], path From ea169b046a0679e7b53bf6e212e6faa47897dffa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:27:26 +0400 Subject: [PATCH 13/33] Type the new metadata reads and remove their duplicated lookups The current-law value and node-label connectors returned list[Any]; they now return CurrentLawValueRow and ParameterNodeLabelRow named tuples, so mypy checks every use of their fields. get_model_catalog re-implemented get_economy_options's national-region and default-dataset lookup; both now share _validated_national_dataset with a national_region transformation, and differ only in how they load the dataset. The part parameter paths and the found parameter names are computed once in the service and passed to the transformations instead of being recomputed there. Co-Authored-By: Claude Opus 5.5 --- .../reads_parameter_nodes.py | 8 +- .../database_connectors/reads_parameters.py | 13 ++- .../services/v2/metadata/services.py | 103 +++++++++++------- .../services/v2/metadata/transformations.py | 42 ++++--- .../services/v2/metadata/types.py | 29 ++++- tests/unit/v2/test_metadata_service.py | 32 ++++++ 6 files changed, 160 insertions(+), 67 deletions(-) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py index 154a1a6eb..24968399a 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -2,12 +2,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from sqlmodel import Session, col, select from policyengine_api.data.v2.models import ParameterNode from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import ParameterNodeLabelRow if TYPE_CHECKING: from collections.abc import Collection @@ -16,14 +17,14 @@ def read_parameter_node_labels( session: Session, *, model_version_id: UUID, node_names: Collection[str] -) -> list[Any]: +) -> list[ParameterNodeLabelRow]: """Return the name and stored label of each named node of a model version. Names the version has no node for are absent. Rows are in name order. """ if not node_names: return [] - return read_rows( + rows = read_rows( session, select(col(ParameterNode.name), col(ParameterNode.label)) .where( @@ -32,3 +33,4 @@ def read_parameter_node_labels( ) .order_by(col(ParameterNode.name)), ) + return [ParameterNodeLabelRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 39ab71bd5..eadd87be9 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -2,13 +2,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import sqlalchemy as sa from sqlmodel import Session, col, select from policyengine_api.data.v2.models import Parameter, ParameterValue from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import CurrentLawValueRow if TYPE_CHECKING: from collections.abc import Collection @@ -170,16 +171,15 @@ def read_parameter_value( def read_current_law_values_by_parameter_name( session: Session, *, model_version_id: UUID, parameter_names: Collection[str] -) -> list[Any]: +) -> list[CurrentLawValueRow]: """Return the current-law values of the named parameters of a model version. - Each row has the parameter's name, the value's value_json, start_date and - end_date. Values tied to a policy or dynamic are excluded. Rows are ordered - by parameter name, then start date. + Values tied to a policy or dynamic are excluded. Rows are ordered by + parameter name, then start date. """ if not parameter_names: return [] - return read_rows( + rows = read_rows( session, select( col(Parameter.name), @@ -196,3 +196,4 @@ def read_current_law_values_by_parameter_name( ) .order_by(col(Parameter.name), col(ParameterValue.start_date)), ) + return [CurrentLawValueRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index c5a63a223..017e3e5b4 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -55,11 +55,13 @@ metadata_parameter_value, metadata_region, metadata_variable, + national_region, page_result, parameter_value_selection, part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( + FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParametersResult, MetadataCatalogVariablesResult, @@ -85,9 +87,10 @@ ) if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from uuid import UUID + from policyengine_api.data.v2.models import Dataset, Region from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) @@ -554,27 +557,17 @@ def get_economy_options( region, its default input dataset, the current-law id or the years. """ selected = self._select_catalog(country_id, policyengine_version) - regions = read_regions( - self._database_session.session, - model_version_id=selected.model_version.id, - ) - national_region = next( - (region for region in regions if region.code == country_id), None - ) - national_dataset = ( - read_input_dataset( - self._database_session.session, - model_version_id=selected.model_version.id, - dataset_id=national_region.default_dataset_id, - ) - if national_region is not None - else None - ) - dataset = validate_economy_options( + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + dataset = _validated_national_dataset( selected, country_id=country_id, regions=regions, - national_dataset=national_dataset, + find_dataset=lambda dataset_id: read_input_dataset( + session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), ) return metadata_economy_options( selected, regions=regions, national_dataset=dataset @@ -598,28 +591,19 @@ def get_model_catalog( datasets = read_input_datasets( session, model_version_id=selected.model_version.id ) - national_dataset = None - if regions: - national_region = next( - (region for region in regions if region.code == country_id), None - ) - national_dataset = validate_economy_options( + national_dataset = ( + _validated_national_dataset( selected, country_id=country_id, regions=regions, - national_dataset=( - next( - ( - dataset - for dataset in datasets - if dataset.id == national_region.default_dataset_id - ), - None, - ) - if national_region is not None - else None + find_dataset=lambda dataset_id: next( + (dataset for dataset in datasets if dataset.id == dataset_id), + None, ), ) + if regions + else None + ) return metadata_model_catalog( selected, regions=regions, @@ -640,13 +624,17 @@ def get_catalog_variables( variables = read_all_variables( session, model_version_id=selected.model_version.id ) + part_paths = part_parameter_paths(variables) part_values = read_current_law_values_by_parameter_name( session, model_version_id=selected.model_version.id, - parameter_names=part_parameter_paths(variables), + parameter_names=part_paths, ) return metadata_catalog_variables( - selected, variables=variables, part_values=part_values + selected, + variables=variables, + part_paths=part_paths, + part_values=part_values, ) def get_catalog_parameters( @@ -669,21 +657,52 @@ def get_catalog_parameters( parameters = read_parameters_by_name( session, model_version_id=model_version_id, parameter_names=requested_names ) - found_names = {parameter.name for parameter in parameters} + found_parameters = FoundParameters( + parameters=parameters, + names={parameter.name for parameter in parameters}, + ) value_rows = read_current_law_values_by_parameter_name( - session, model_version_id=model_version_id, parameter_names=found_names + session, + model_version_id=model_version_id, + parameter_names=found_parameters.names, ) node_rows = read_parameter_node_labels( session, model_version_id=model_version_id, node_names={ - ancestor for name in found_names for ancestor in ancestor_paths(name) + ancestor + for name in found_parameters.names + for ancestor in ancestor_paths(name) }, ) return metadata_catalog_parameters( selected, requested_names=requested_names, - parameters=parameters, + found_parameters=found_parameters, value_rows=value_rows, node_rows=node_rows, ) + + +def _validated_national_dataset( + selected: SelectedCatalog, + *, + country_id: str, + regions: list[Region], + find_dataset: Callable[[UUID], Dataset | None], +) -> Dataset: + """Return the national region's default dataset, validated for economy choices. + + find_dataset loads a dataset of the selected version by id. Raise + MetadataCatalogUnavailableError when the national region, its dataset or + the model-version options are missing. + """ + region = national_region(regions, country_id) + return validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + None if region is None else find_dataset(region.default_dataset_id) + ), + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 4b9167845..938027741 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -7,6 +7,8 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, + FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParameter, MetadataCatalogParametersResult, @@ -30,6 +32,7 @@ MetadataRegionType, MetadataTimePeriodOption, MetadataVariable, + ParameterNodeLabelRow, ParameterValueSelection, ) @@ -138,16 +141,20 @@ def part_parameter_paths(variables: list[Variable]) -> set[str]: def metadata_catalog_variables( - selected: SelectedCatalog, *, variables: list[Variable], part_values: list[Any] + selected: SelectedCatalog, + *, + variables: list[Variable], + part_paths: set[str], + part_values: list[CurrentLawValueRow], ) -> MetadataCatalogVariablesResult: """Build every variable of the selected version and its part parameters' values. - part_values holds rows of read_current_law_values_by_parameter_name, in - parameter-name and start-date order. Each path a variable names gets an - entry, empty when no row carries its name. + part_paths holds part_parameter_paths of the variables, and part_values the + current-law values of those parameters in name and start-date order. Each + path gets an entry, empty when no value carries its name. """ values_by_path = _dated_values_by_parameter_name( - part_values, parameter_names=part_parameter_paths(variables) + part_values, parameter_names=part_paths ) return MetadataCatalogVariablesResult( policyengine_version=selected.policyengine_version, @@ -158,12 +165,11 @@ def metadata_catalog_variables( def _dated_values_by_parameter_name( - value_rows: list[Any], *, parameter_names: set[str] + value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] ) -> dict[str, list[MetadataDatedValue]]: """Group value rows by parameter name, with an empty list for each other name. - value_rows holds rows of read_current_law_values_by_parameter_name, in - parameter-name and start-date order. + value_rows are in parameter-name and start-date order. """ values_by_name: dict[str, list[MetadataDatedValue]] = { name: [] for name in sorted(parameter_names) @@ -211,19 +217,20 @@ def metadata_catalog_parameters( selected: SelectedCatalog, *, requested_names: set[str], - parameters: list[Parameter], - value_rows: list[Any], - node_rows: list[Any], + found_parameters: FoundParameters, + value_rows: list[CurrentLawValueRow], + node_rows: list[ParameterNodeLabelRow], ) -> MetadataCatalogParametersResult: """Build the requested parameters with their values and ancestor labels. - value_rows holds rows of read_current_law_values_by_parameter_name and - node_rows rows of read_parameter_node_labels. + value_rows are the found parameters' current-law values and node_rows the + stored labels of their ancestor nodes. """ + parameters = found_parameters.parameters + found_names = found_parameters.names values_by_name = _dated_values_by_parameter_name( - value_rows, parameter_names={parameter.name for parameter in parameters} + value_rows, parameter_names=found_names ) - found_names = {parameter.name for parameter in parameters} return MetadataCatalogParametersResult( policyengine_version=selected.policyengine_version, catalog_format=selected.model_version.catalog_format, @@ -348,6 +355,11 @@ def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild] return items +def national_region(regions: list[Region], country_id: str) -> Region | None: + """Return the country's national region (its code is the country ID), or None.""" + return next((region for region in regions if region.code == country_id), None) + + def metadata_economy_options( selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset ) -> MetadataEconomyOptionsResult: diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 36dee6c43..fecb656fa 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,11 +5,14 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import Generic, Literal, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue +if TYPE_CHECKING: + from policyengine_api.data.v2.models import Parameter + @dataclass(frozen=True) class ParameterFilter: @@ -50,6 +53,30 @@ class ParameterValueSelection: parameter_names: tuple[str, ...] | None = None +class CurrentLawValueRow(NamedTuple): + """One current-law value of a parameter, read with the parameter's name.""" + + name: str + value_json: Any + start_date: datetime + end_date: datetime | None + + +@dataclass(frozen=True) +class FoundParameters: + """The stored parameters a by-name read found, and their names.""" + + parameters: list[Parameter] + names: set[str] + + +class ParameterNodeLabelRow(NamedTuple): + """A parameter node's name and its stored label (None when it has none).""" + + name: str + label: str | None + + class StrictResponseModel(BaseModel): """Base for metadata results that rejects unknown fields and non-finite numbers.""" diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d07e94bde..458ab34a4 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -39,10 +39,12 @@ from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, MetadataDatedValue, MetadataEntity, MetadataEnumValue, ParameterFilter, + ParameterNodeLabelRow, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -1085,6 +1087,36 @@ def test_each_country_and_version_keeps_its_own_selection( assert service.get_model_by_country("us", "9.9.9") == named_us +def test_value_and_node_label_reads_return_typed_rows( + catalog_session: Session, +) -> None: + from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 + read_parameter_node_labels, + ) + from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( # noqa: E501 + read_current_law_values_by_parameter_name, + ) + + model_version = _us_model_version(catalog_session) + values = read_current_law_values_by_parameter_name( + catalog_session, + model_version_id=model_version.id, + parameter_names={"gov.example.rate"}, + ) + labels = read_parameter_node_labels( + catalog_session, + model_version_id=model_version.id, + node_names={"gov.example", "gov.absent"}, + ) + + assert [type(row) for row in values] == [CurrentLawValueRow, CurrentLawValueRow] + assert [(row.name, row.value_json) for row in values] == [ + ("gov.example.rate", 0.1), + ("gov.example.rate", 0.2), + ] + assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 6e7e52ed9f78a76ccc03f06c3e51f2fd0ace1423 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:29:48 +0400 Subject: [PATCH 14/33] Let browser scripts read metadata responses' ETag The CORS middleware exposed only the request and observability IDs and WWW-Authenticate, so a script on another origin could not read the ETag of a v2 metadata response to revalidate it. ETag joins Access-Control-Expose-Headers (Cache-Control is readable without it). Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 4 +++- policyengine_api/asgi_factory.py | 2 ++ tests/unit/v2/test_metadata_routes.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 2ecef27b2..c39707c93 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -110,7 +110,9 @@ version, which changes on deploy, and may be reused for five minutes reused only until the next UTC midnight, whichever lifetime would otherwise apply. A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses -carry neither header. +carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so +browser scripts on other origins can read it (`Cache-Control` is readable +without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 1b986188e..cba4c00bc 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -480,6 +480,8 @@ def create_asgi_app( REQUEST_ID_HEADER, OBSERVABILITY_ID_HEADER, "WWW-Authenticate", + # v2 metadata responses carry an ETag a client may revalidate with. + "ETag", ], allow_credentials=False, max_age=600, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index d3b5ab349..df0465c9d 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1273,3 +1273,22 @@ def test_every_metadata_route_declares_the_shared_catalog_query() -> None: country = parameters["country_id"] assert country["required"] is True, path assert country["schema"]["enum"] == ["us", "uk"], path + + +def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + "/v2/model-catalog?country_id=us", + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert "etag" in exposed_headers From 09bbd994c59626b5acd5c115116ab88b7349092e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:31:05 +0400 Subject: [PATCH 15/33] Record the slow existing routes' times on each release, even after a failure The release live test now also times a page of 500 parameters and one parameter's values, the two routes measured slow in production on 2026-10-21, and writes every time it reached to the job summary from a finally block, so a release whose check fails still records its measurements. The second variables request is labelled as possibly served by another instance, since its time need not be a kept response's. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 13 +-- tests/integration/test_live_v2_metadata.py | 93 ++++++++++++++-------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index c39707c93..50ab0d623 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -370,11 +370,14 @@ return typed client errors. The same test loads each country's catalog in parts as a client starting cold would: `GET /v2/model-catalog` for the running version, then -`/v2/model-catalog/variables` for that exact version twice (the second served -from the process's kept body) and once more with `If-None-Match`, which must -return 304, then `/v2/model-catalog/parameters` for one parameter. It appends -each request's time and body size to the GitHub job summary, so every staging -and production release records these route times against its own catalog. +`/v2/model-catalog/variables` for that exact version twice (the second request +may be served by another instance, so its time is not necessarily a kept +response's) and once more with `If-None-Match`, which must return 304, then a +page of 500 parameters, one parameter's values (the two routes measured at +0.8–2.6 s and 0.5–0.65 s in production on 2026-10-21) and that parameter by +name. It appends each request's time and body size to the GitHub job summary, +even when an assertion fails, so every staging and production release records +these route times against its own catalog. Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 1b2b8e183..00ff143e2 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -154,60 +154,85 @@ def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> No expected_version = _installed_policyengine_version() route_times: list[tuple[str, float, httpx.Response]] = [] - catalog_response, seconds = _timed_get( - api_client, "/v2/model-catalog", params={"country_id": country_id} - ) - route_times.append( - ("/v2/model-catalog (running version)", seconds, catalog_response) + def timed_get(label: str, path: str, **request_options: object) -> httpx.Response: + response, seconds = _timed_get(api_client, path, **request_options) + route_times.append((label, seconds, response)) + return response + + # The times are recorded even when an assertion fails, so every release + # keeps the measurements it reached. + try: + _check_catalog_parts(timed_get, country_id, expected_version) + finally: + _record_route_times(country_id, route_times) + + +def _check_catalog_parts(timed_get, country_id: str, expected_version: str) -> None: + catalog = _assert_ok( + timed_get( + "/v2/model-catalog (running version)", + "/v2/model-catalog", + params={"country_id": country_id}, + ), + expected_version, ) - catalog = _assert_ok(catalog_response, expected_version) assert catalog["regions"] assert catalog["economy_options"]["region"] assert "variables" not in catalog pinned = {"country_id": country_id, "policyengine_version": expected_version} - variables_response, seconds = _timed_get( - api_client, "/v2/model-catalog/variables", params=pinned - ) - route_times.append(("/v2/model-catalog/variables", seconds, variables_response)) - repeated_variables, seconds = _timed_get( - api_client, "/v2/model-catalog/variables", params=pinned + variables_response = timed_get( + "/v2/model-catalog/variables", "/v2/model-catalog/variables", params=pinned ) - route_times.append( - ("/v2/model-catalog/variables (repeated)", seconds, repeated_variables) + second_variables = timed_get( + "/v2/model-catalog/variables (second request; another instance may serve it)", + "/v2/model-catalog/variables", + params=pinned, ) variables = _assert_ok(variables_response, expected_version) assert variables["variables"] assert variables_response.headers["cache-control"] == "public, max-age=86400" - assert repeated_variables.headers["etag"] == variables_response.headers["etag"] - revalidated, seconds = _timed_get( - api_client, + assert second_variables.headers["etag"] == variables_response.headers["etag"] + revalidated = timed_get( + "/v2/model-catalog/variables (revalidated)", "/v2/model-catalog/variables", params=pinned, headers={"If-None-Match": variables_response.headers["etag"]}, ) - route_times.append( - ("/v2/model-catalog/variables (revalidated)", seconds, revalidated) - ) assert revalidated.status_code == 304 - parameter_name = _assert_ok( - api_client.get("/v2/parameters", params={**pinned, "limit": 1}), + # The two existing routes measured slow in production on 2026-10-21. + page = _assert_ok( + timed_get( + "/v2/parameters (page of 500)", + "/v2/parameters", + params={**pinned, "limit": 500}, + ), expected_version, - )["items"][0]["name"] - parameters_response, seconds = _timed_get( - api_client, - "/v2/model-catalog/parameters", - params={**pinned, "name": parameter_name}, ) - route_times.append( - ("/v2/model-catalog/parameters (1 name)", seconds, parameters_response) + first_parameter = page["items"][0] + one_parameter_values = _assert_ok( + timed_get( + "/v2/parameter-values (one parameter)", + "/v2/parameter-values", + params={**pinned, "parameter_id": first_parameter["id"]}, + ), + expected_version, + ) + assert all( + value["parameter_id"] == first_parameter["id"] + for value in one_parameter_values["items"] + ) + + parameters = _assert_ok( + timed_get( + "/v2/model-catalog/parameters (1 name)", + "/v2/model-catalog/parameters", + params={**pinned, "name": first_parameter["name"]}, + ), + expected_version, ) - parameters = _assert_ok(parameters_response, expected_version) assert [parameter["name"] for parameter in parameters["parameters"]] == [ - parameter_name + first_parameter["name"] ] - assert parameters["parameters"][0]["values"] assert parameters["missing_names"] == [] - - _record_route_times(country_id, route_times) From b97982681d155e1806618ffe293d1900d2572a3b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:21:55 +0400 Subject: [PATCH 16/33] Read current parameter values and set their lifetime from one instant The parameter-values route marked a current=true request as depending on the day but passed no instant to the read: the service took the day from its own clock during the read, and the lifetime came from another clock reading after it. A read just before UTC midnight whose headers were written just after got the old day's values with a lifetime of nearly a day. The route now takes the request's instant once, passes it to the read as the filter's now, and records it; the caching route gives the response the seconds left until the UTC midnight after that instant, measured when the headers are written, and zero once that midnight has passed. No other metadata read depends on the time. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 8 +-- .../fastapi_routes/v2/metadata/common.py | 42 ++++++++++----- .../v2/metadata/parameter_routes.py | 8 +-- tests/integration/test_v2_metadata_routes.py | 51 +++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 24 +++++++++ .../v2/test_parameter_value_listing_route.py | 12 ++++- 6 files changed, 124 insertions(+), 21 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 50ab0d623..baf6b24f4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -106,9 +106,11 @@ route's response format or with the day it is read on, and the `ETag` lets a client revalidate after the day. A request without it selects the running version, which changes on deploy, and may be reused for five minutes (`max-age=300`). A response that depends on the request's UTC day, which is -`GET /v2/parameter-values?current=true` (the values in effect today), may be -reused only until the next UTC midnight, whichever lifetime would otherwise -apply. A request whose `If-None-Match` names the current `ETag` (strong or +`GET /v2/parameter-values?current=true` (the values in effect today), takes +the request's instant once: the read selects that day's values, and the +response may be reused only until the UTC midnight after that instant, +whichever lifetime would otherwise apply (`max-age=0` if its headers are +written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d10f4072b..ba7a74418 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -103,11 +103,12 @@ def policyengine_version(self) -> str: """ -DEPENDS_ON_UTC_DAY = "depends_on_utc_day" -"""The ``request.state`` attribute a route sets when its result depends on the day. +UTC_DAY_READ = "utc_day_read" +"""The ``request.state`` attribute holding the instant a day-dependent read used. -Such a response, for example current values on the request's UTC day, may be -reused only until the next UTC midnight, whatever version it names. +A route whose result depends on the UTC day, such as current parameter values, +sets it with ``take_utc_instant_for_read`` and reads with that same instant; +its response may then be reused only until the next UTC midnight after it. """ @@ -116,11 +117,26 @@ def utc_now() -> datetime: return datetime.now(UTC) -def _seconds_until_next_utc_day(moment: datetime) -> int: - next_day = (moment + timedelta(days=1)).replace( +def take_utc_instant_for_read(request: Request) -> datetime: + """Take the instant a day-dependent read uses, and record it on the request. + + The read and its response's cache lifetime then use the same UTC day. + """ + instant = utc_now() + setattr(request.state, UTC_DAY_READ, instant) + return instant + + +def _seconds_left_in_utc_day(read_instant: datetime, now: datetime) -> int: + """Return the seconds from ``now`` to the UTC midnight after ``read_instant``. + + Zero when that midnight has passed, so values read for one day are never + reused into the next. + """ + next_midnight = (read_instant.astimezone(UTC) + timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) - return max(int((next_day - moment).total_seconds()), 0) + return max(int((next_midnight - now).total_seconds()), 0) class CacheableMetadataRoute(APIRoute): @@ -148,10 +164,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: The ETag is a hash of the exact body, so it changes with any change to the content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the - running version for five minutes, and one whose route marked it as - depending on the day (``DEPENDS_ON_UTC_DAY``) no later than the next UTC - midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with - no body. + running version for five minutes, and one whose route read for a UTC day + (``UTC_DAY_READ``) no later than the midnight after that day. A request + whose ``If-None-Match`` names the ETag gets a 304 with no body. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -159,8 +174,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: if "policyengine_version" in request.query_params else RUNNING_VERSION_MAX_AGE_SECONDS ) - if getattr(request.state, DEPENDS_ON_UTC_DAY, False): - max_age = min(max_age, _seconds_until_next_utc_day(utc_now())) + read_instant = getattr(request.state, UTC_DAY_READ, None) + if read_instant is not None: + max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): return Response(status_code=304, headers=headers) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index d9ddeb336..406d470ab 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -9,10 +9,10 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, + take_utc_instant_for_read, ) from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( CatalogQueryDependency, @@ -121,8 +121,9 @@ def list_parameter_values( request: Request, query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: - # Current values are those in effect on the request's UTC day. - setattr(request.state, DEPENDS_ON_UTC_DAY, query.current) + # Current values are those in effect on the request's UTC day; the + # read and the response's lifetime use the same instant. + now = take_utc_instant_for_read(request) if query.current else None return read_resource( dependencies, MetadataParameterValuePageResponse, @@ -132,6 +133,7 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + now=now, parameter_names=( None if query.parameter_name is None diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index fa5ea761b..4930e3abb 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -574,3 +574,54 @@ def record_mode(connection, _cursor, _statement, *_args) -> None: assert all(autocommit_modes) with published_engine.connect() as connection: assert connection.connection.dbapi_connection.autocommit is False + + +@pytest.mark.parametrize( + "case", + [ + # Read for 31 December, headers written after midnight: not reusable. + ( + datetime(2025, 12, 31, 23, 59, 59, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, 1, tzinfo=UTC), + [0.1], + "public, max-age=0", + ), + ( + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + [0.1], + "public, max-age=43200", + ), + ( + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + [0.2], + "public, max-age=86400", + ), + ], +) +def test_postgres_current_values_and_their_lifetime_use_one_instant( + published_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + case: tuple[datetime, datetime, list[float], str], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant, header_instant, expected_values, expected_cache_control = case + instants = iter([read_instant, header_instant]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + + response = _client(published_engine).get( + "/v2/parameter-values", + params={ + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "current": "true", + }, + ) + + assert response.status_code == 200 + assert [item["value"] for item in response.json()["result"]["items"]] == ( + expected_values + ) + assert response.headers["cache-control"] == expected_cache_control diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index df0465c9d..e91e6440b 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1292,3 +1292,27 @@ def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: for value in response.headers["access-control-expose-headers"].split(",") } assert "etag" in exposed_headers + + +def test_current_values_read_just_before_midnight_expire_at_that_midnight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The read and the response headers must use the same day: values read for + # 31 December may not be reused into 1 January, even when the headers are + # written after midnight. + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant = datetime(2026, 12, 31, 23, 59, 59, tzinfo=UTC) + instants = iter([read_instant, datetime(2027, 1, 1, 0, 0, 1, tzinfo=UTC)]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3¤t=true" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=0" + _name, _args, kwargs = reader.calls[0] + assert kwargs["value_filter"].now == read_instant diff --git a/tests/unit/v2/test_parameter_value_listing_route.py b/tests/unit/v2/test_parameter_value_listing_route.py index 39ac27b72..a07aa4ca5 100644 --- a/tests/unit/v2/test_parameter_value_listing_route.py +++ b/tests/unit/v2/test_parameter_value_listing_route.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import UTC, datetime + import pytest from policyengine_api.services.v2.metadata.types import ParameterValueFilter @@ -9,7 +11,13 @@ from .test_metadata_routes import ResourceReader, _client, _resource_results -def test_listing_forwards_every_query_parameter() -> None: +def test_listing_forwards_every_query_parameter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + request_instant = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) + monkeypatch.setattr(common, "utc_now", lambda: request_instant) results, ids = _resource_results() reader = ResourceReader(results) @@ -32,7 +40,7 @@ def test_listing_forwards_every_query_parameter() -> None: ("uk", "4.19.0"), { "value_filter": ParameterValueFilter( - parameter_id=ids["parameter_id"], current=True + parameter_id=ids["parameter_id"], current=True, now=request_instant ), "offset": 5, "limit": 7, From b9d734e3767f724326d3a18a0164cbe2dfad7f2a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:23:41 +0400 Subject: [PATCH 17/33] Vary every cacheable metadata response by Origin The API reflects every origin in Access-Control-Allow-Origin, and the CORS middleware passes a request without Origin straight through, so such a response carried neither that header nor Vary: Origin while marked public for up to a day. A shared cache storing it could serve it to a later cross-origin request, which the browser rejects. Every cacheable metadata response, and every 304, now carries Vary: Origin, added to any existing Vary value such as compression's Accept-Encoding rather than replacing it. Failed reads are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 7 +++- .../fastapi_routes/v2/metadata/common.py | 24 +++++++++-- tests/unit/v2/test_metadata_routes.py | 40 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index baf6b24f4..fa28c9a5a 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -112,7 +112,12 @@ response may be reused only until the UTC midnight after that instant, whichever lifetime would otherwise apply (`max-age=0` if its headers are written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses -carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so +carry neither header. Every cacheable response, and every 304, carries +`Vary: Origin` (added to any other `Vary` value, such as compression's +`Accept-Encoding`): the API reflects every origin in +`Access-Control-Allow-Origin` and a request without `Origin` gets no such +header, so a shared cache must keep responses apart by `Origin`. `ETag` is +listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable without that). diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index ba7a74418..d9eb49159 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -165,8 +165,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the running version for five minutes, and one whose route read for a UTC day - (``UTC_DAY_READ``) no later than the midnight after that day. A request - whose ``If-None-Match`` names the ETag gets a 304 with no body. + (``UTC_DAY_READ``) no later than the midnight after that day. Every such + response, and the 304 a request whose ``If-None-Match`` names the ETag + gets instead, varies by ``Origin``. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -179,11 +180,28 @@ def with_cache_headers(request: Request, response: Response) -> Response: max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): - return Response(status_code=304, headers=headers) + not_modified = Response(status_code=304, headers=headers) + not_modified.headers["Vary"] = _vary_with_origin(None) + return not_modified response.headers.update(headers) + response.headers["Vary"] = _vary_with_origin(response.headers.get("vary")) return response +def _vary_with_origin(existing: str | None) -> str: + """Return a Vary value that keeps ``existing`` and names Origin once. + + The API reflects every origin in Access-Control-Allow-Origin, and a request + without Origin gets no such header, so a shared cache must store responses + per Origin value or it could serve one to a cross-origin request that the + browser then rejects. + """ + values = [value.strip() for value in (existing or "").split(",") if value.strip()] + if not any(value.lower() == "origin" for value in values): + values.append("Origin") + return ", ".join(values) + + def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: if if_none_match is None: return False diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e91e6440b..bedd4a326 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1316,3 +1316,43 @@ def test_current_values_read_just_before_midnight_expire_at_that_midnight( assert response.headers["cache-control"] == "public, max-age=0" _name, _args, kwargs = reader.calls[0] assert kwargs["value_filter"].now == read_instant + + +def _vary_values(response) -> set[str]: + return { + value.strip().lower() + for value in response.headers.get("vary", "").split(",") + if value.strip() + } + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_metadata_responses_vary_by_origin(origin: str | None) -> None: + # The API reflects any origin, so a shared cache must not reuse a response + # stored for a request without Origin (no Access-Control-Allow-Origin) for + # a cross-origin one, or the reverse. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {} if origin is None else {"Origin": origin} + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + large = client.get("/v2/model-catalog?country_id=us", headers=headers) + revalidated = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": large.headers["etag"]}, + ) + + assert revalidated.status_code == 304 + for response in (small, large, revalidated): + assert "origin" in _vary_values(response) + assert "accept-encoding" in _vary_values(large) + + +def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get("/v2/variables?country_id=us") + + assert response.status_code == 404 + assert "origin" not in _vary_values(response) From b921ec2ecc08a72849dee1e94dd214b0a90bcef1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:26:05 +0400 Subject: [PATCH 18/33] Document metadata caching headers, 304 and the 400 case in OpenAPI The metadata routes' OpenAPI entries described 400 as an invalid PolicyEngine.py version, which now returns 422; 400 remains only for an unsupported country in the path of the by-country route. They also declared no 304 and none of the caching headers, so the app's generated client and readers could not see them. Every metadata route now documents ETag, Cache-Control and Vary on its 200 response and a 304 without a body. The statement counts in the changelog and the migration document now say exactly which requests send two statements. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 7 ++- .../v2/metadata/catalog_routes.py | 4 +- .../fastapi_routes/v2/metadata/common.py | 43 ++++++++++++++++++- .../v2/metadata/geography_routes.py | 4 +- .../v2/metadata/model_routes.py | 4 +- .../v2/metadata/parameter_routes.py | 4 +- tests/unit/v2/test_metadata_routes.py | 30 +++++++++++++ 8 files changed, 85 insertions(+), 13 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md index 8eff7221b..4c325f3a5 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -1 +1 @@ -v2 metadata reads run in autocommit read-only sessions on the shared pool, and each process keeps the catalog versions it has selected, so a metadata request sends two statements to the database (the pool's liveness check and the read) instead of five. +v2 metadata reads run in autocommit sessions on the shared pool, and each process keeps the catalog versions it has selected. A request that makes one read for a version the process has already selected, such as a page of parameters or one parameter's values, now sends two statements to the database (the pool's liveness check and the read) instead of five; each further read in a request adds one statement. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index fa28c9a5a..4cfee0b24 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -93,8 +93,11 @@ later requests for the same country and version skip the selection query; failed selections are not kept. Measured with statement logging in a local database, a page of 500 parameters or one parameter's values took five statements per request before (the pool's liveness check, `BEGIN`, the catalog -selection, the read and `ROLLBACK`) and two after (the liveness check and the -read). The pool's liveness check (`pool_pre_ping`) is unchanged. +selection, the read and `ROLLBACK`) and two after, once the process has +selected that version (the liveness check and the read). A route that makes +several reads, such as `/v2/model-catalog` (regions and datasets), sends one +statement per read after the liveness check. The pool's liveness check +(`pool_pre_ping`) is unchanged. ## Response caching diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 1ef94301e..968c802c4 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -8,7 +8,7 @@ from starlette.responses import JSONResponse, Response from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, read_whole_version_resource, @@ -42,7 +42,7 @@ def build_v2_model_catalog_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d9eb49159..3b54fc2fa 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -50,7 +50,11 @@ ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, - "description": "The resource request or PolicyEngine.py version is invalid.", + "description": ( + "The country in the path is not supported. Query parameters are " + "checked first: an invalid one, including the PolicyEngine.py " + "version, returns 422." + ), }, 404: { "model": V2ErrorResponse, @@ -62,7 +66,10 @@ }, 422: { "model": V2ErrorResponse, - "description": "The request parameters do not match the resource schema.", + "description": ( + "The query parameters do not match the resource schema, for example " + "an unknown parameter or a malformed PolicyEngine.py version." + ), }, 500: { "model": V2ErrorResponse, @@ -75,6 +82,38 @@ } +CACHE_HEADER_DESCRIPTIONS: dict[str, dict[str, Any]] = { + "ETag": { + "description": "Strong validator: a hash of the exact response body.", + "schema": {"type": "string"}, + }, + "Cache-Control": { + "description": ( + "public, with max-age 86400 when the request names " + "policyengine_version and 300 otherwise; current parameter values " + "expire at the next UTC midnight." + ), + "schema": {"type": "string"}, + }, + "Vary": { + "description": "Includes Origin, so caches keep responses apart by origin.", + "schema": {"type": "string"}, + }, +} + +METADATA_RESPONSES: dict[int | str, dict[str, Any]] = { + **ERROR_RESPONSES, + 200: {"headers": CACHE_HEADER_DESCRIPTIONS}, + 304: { + "description": ( + "Not modified: If-None-Match names the current ETag. No body is sent." + ), + "headers": CACHE_HEADER_DESCRIPTIONS, + }, +} +"""Every metadata route's documented responses, with its caching headers.""" + + class CatalogVersionResult(Protocol): """A whole-version result, which names the catalog version it was read from.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 4665ea67c..448826c5d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -40,7 +40,7 @@ def build_v2_metadata_geography_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 63e72ce56..3482ee108 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -43,7 +43,7 @@ def build_v2_metadata_model_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 406d470ab..51383ccc0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, take_utc_instant_for_read, @@ -46,7 +46,7 @@ def build_v2_metadata_parameter_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index bedd4a326..792d7c05c 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1356,3 +1356,33 @@ def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: assert response.status_code == 404 assert "origin" not in _vary_values(response) + + +def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + responses = schema["paths"][path]["get"]["responses"] + assert set(responses["200"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "$ref" in responses["200"]["content"]["application/json"]["schema"] + assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "content" not in responses["304"] + assert "policyengine_version" not in responses["400"]["description"] + assert "path" in responses["400"]["description"] From b0e98b6defe998b45494e5c149020e06db631160 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:27:18 +0400 Subject: [PATCH 19/33] Keep a response's build lock until it is stored, and state the cache's size MetadataResponseCache.get_or_build removed the per-key build lock before storing the built response, so a request arriving in that interval found neither and started a second read. The lock is now released only after the response is stored (or the read failed). The concurrency tests no longer depend on timing: one blocks the build on an event, the other starts a second request while the first stores its result. The documentation states the bound's cost: the 6.10.0 US variables body is 3,910,063 bytes uncompressed, so eight entries hold about 11 MB in the expected mix and at most about 31 MB. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 4 +- .../v2/metadata/response_cache.py | 32 +++++++----- tests/unit/v2/test_metadata_response_cache.py | 52 +++++++++++++++---- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 4cfee0b24..e80f289f9 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -127,7 +127,9 @@ without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, for up to eight route, country and version combinations (least recently used -dropped first). A response is kept under the exact version it was read from, +dropped first; the 6.10.0 US variables body is 3,910,063 bytes uncompressed, +so eight entries hold about 11 MB for both countries' two routes at two +versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only the first successful request per process reads the database; concurrent first diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 72d3cf9ba..22ea2d348 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -48,9 +48,13 @@ class MetadataResponseCache: are kept under the exact version they were read from: a request without a version (the running version, fixed for the process) and one naming that version share one response. The most recently used ``capacity`` responses - are kept. Requests for a response being built wait for that build instead - of starting another, and a failed read (a v2 error response) is returned - but not kept, so the next request tries again. + are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes + uncompressed, so the default of eight holds about 11 MB for both + countries' two whole-version routes at two versions, and at most about + 31 MB if every entry were that large. Requests for a response being built + wait for that build, and its storage, instead of starting another, and a + failed read (a v2 error response) is returned but not kept, so the next + request tries again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: @@ -99,15 +103,19 @@ def get_or_build( with self._lock: build_lock = self._build_locks.setdefault(build_key, threading.Lock()) with build_lock: - kept = self._stored(route, country_id, requested_version) - if kept is not None: - return kept - built = build() - with self._lock: - self._build_locks.pop(build_key, None) - if not isinstance(built, BuiltResponse): - return built - return self._keep(route, country_id, requested_version, built) + try: + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + built = build() + if not isinstance(built, BuiltResponse): + return built + # Stored before the build lock is released, so a request + # waiting on it finds the response instead of reading again. + return self._keep(route, country_id, requested_version, built) + finally: + with self._lock: + self._build_locks.pop(build_key, None) def _keep( self, diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index 9cfc8fd8c..4d6704a36 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -3,7 +3,6 @@ from __future__ import annotations import threading -import time import pytest from starlette.responses import JSONResponse @@ -119,26 +118,61 @@ def get(country_id: str) -> None: def test_concurrent_requests_for_one_response_build_it_once() -> None: cache = MetadataResponseCache() builds: list[str] = [] - started = threading.Barrier(4) + building = threading.Event() + release = threading.Event() - def slow_build() -> BuiltResponse: + def blocked_build() -> BuiltResponse: builds.append("built") - time.sleep(0.05) + building.set() + release.wait() return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) def request() -> None: - started.wait() - cache.get_or_build("variables", "us", None, slow_build) + cache.get_or_build("variables", "us", None, blocked_build) - threads = [threading.Thread(target=request) for _ in range(4)] - for thread in threads: + first = threading.Thread(target=request) + first.start() + building.wait() + others = [threading.Thread(target=request) for _ in range(3)] + for thread in others: thread.start() - for thread in threads: + release.set() + for thread in [first, *others]: thread.join() + # Whether the others waited on the build or arrived after it, none read. assert builds == ["built"] +def test_a_request_arriving_while_a_response_is_stored_waits_for_it() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + second_results: list[object] = [] + second_requests: list[threading.Thread] = [] + original_keep = cache._keep + + def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: + second_request = threading.Thread( + target=lambda: second_results.append( + cache.get_or_build("variables", "us", None, _counting_build(builds)) + ) + ) + second_requests.append(second_request) + second_request.start() + # The second request may not build while this one stores its result: + # it waits on the build, whatever the timing. + second_request.join(timeout=0.2) + return original_keep(*keep_arguments) + + cache._keep = keep_after_another_request_arrives # type: ignore[method-assign] + + first = cache.get_or_build("variables", "us", None, _counting_build(builds)) + second_requests[0].join() + + assert builds == ["built"] + assert second_results == [first] + + def test_capacity_must_be_positive() -> None: with pytest.raises(ValueError, match="capacity"): MetadataResponseCache(capacity=0) From 77a12b55de071e6732d5826c2bed832d788ee0a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:28:38 +0400 Subject: [PATCH 20/33] Refuse writes through v2 read sessions and state who uses them The autocommit read sessions were described as read-only, but nothing stopped a flush. Autocommit rules out a per-transaction read-only setting, and a connection-level default would carry over to writers through the shared pool, so the factory now refuses every ORM flush with ReadOnlySessionError, and the docstring says exactly that only the metadata reader opens these sessions and that statements passed to execute are not checked. A test pins that the metadata reader factory is the only user. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 12 ++++--- policyengine_api/data/v2/database.py | 25 ++++++++++++--- tests/unit/v2/test_database.py | 45 ++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index e80f289f9..9357eb48c 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -83,11 +83,13 @@ still returns HTTP 400. ## Database round trips per request -Metadata reads use read-only sessions in autocommit mode on the shared v2 -pool (`get_v2_read_session_factory`): each read is one statement on rows a -published catalog version never changes, so no transaction is needed, and the -database receives no `BEGIN` before the reads and no `ROLLBACK` when the -session closes. Each process also keeps every catalog version a request has +Metadata reads use sessions in autocommit mode on the shared v2 pool +(`get_v2_read_session_factory`), which only the metadata reader opens: each +read is one statement on rows a published catalog version never changes, so no +transaction is needed, and the database receives no `BEGIN` before the reads +and no `ROLLBACK` when the session closes. The sessions refuse to flush added +or changed rows (`ReadOnlySessionError`); a statement passed directly to +`execute` is not checked, so the metadata connectors keep to `SELECT`. Each process also keeps every catalog version a request has selected (`SelectedCatalogStore`), since a published version never changes, so later requests for the same country and version skip the selection query; failed selections are not kept. Measured with statement logging in a local diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 3885ab94d..f9df94461 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -8,6 +8,7 @@ from threading import Lock from typing import TYPE_CHECKING +from sqlalchemy import event from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -83,20 +84,34 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +class ReadOnlySessionError(RuntimeError): + """Raised when code tries to write rows through a v2 read session.""" + + +def _refuse_flush(*_flush_arguments: object) -> None: + msg = "v2 read sessions do not write rows" + raise ReadOnlySessionError(msg) + + def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: - """Build a factory of read-only Sessions that run without a transaction. + """Build a factory of Sessions for v2 metadata reads, run without a transaction. The sessions share ``engine``'s pool in autocommit mode, so psycopg sends each statement on its own: no BEGIN before the first read and no ROLLBACK - when the session closes. Use them only for reads whose statements need no - common snapshot, such as reads of a published catalog version, which never - changes. The pool restores the ordinary mode when a connection returns. + when the session closes. Only the metadata reader uses them: each of its + reads is one statement on a published catalog version, which never + changes, so no common snapshot is needed. The sessions refuse to flush + added or changed rows; statements passed directly to ``execute`` are not + checked, so they must stay reads. The pool restores the ordinary mode + when a connection returns. """ - return sessionmaker( + factory = sessionmaker( bind=engine.execution_options(isolation_level="AUTOCOMMIT"), class_=Session, expire_on_commit=False, ) + event.listen(factory, "before_flush", _refuse_flush) + return factory def _discard_inherited_state(current_pid: int) -> None: diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index c1984199c..fbfa92c72 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -1,11 +1,14 @@ """Tests for lazy process-owned v2 engine and SQLModel Session factories.""" from collections.abc import Iterator +from pathlib import Path import pytest -from sqlmodel import Session +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select from policyengine_api.data.v2 import database +from policyengine_api.data.v2.models import V2_METADATA, TaxBenefitModel from policyengine_api.data.v2.settings import ( V2_MIGRATION_DATABASE_URL, V2_RUNTIME_DATABASE_URL, @@ -148,3 +151,43 @@ def test_read_session_factory_is_rebuilt_with_the_engine() -> None: database.close_v2_database() assert database.get_v2_read_session_factory(settings) is not first + + +def test_read_sessions_refuse_to_write_rows() -> None: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + V2_METADATA.create_all(engine) + read_factory = database.build_v2_read_session_factory(engine) + + with read_factory() as session: + session.add(TaxBenefitModel(name="policyengine-test")) + with pytest.raises(database.ReadOnlySessionError): + session.flush() + + with Session(engine) as session: + assert session.exec(select(TaxBenefitModel)).all() == [] + engine.dispose() + + +def test_only_the_metadata_reader_opens_read_sessions() -> None: + package = Path(database.__file__).parents[2] + users = sorted( + str(module.relative_to(package)) + for module in package.rglob("*.py") + if module.name != "database.py" + and ( + "get_v2_read_session_factory" in module.read_text(encoding="utf-8") + or "build_v2_read_session_factory" in module.read_text(encoding="utf-8") + ) + ) + + assert users == ["fastapi_routes/dependencies.py"] + dependencies_source = (package / "fastapi_routes/dependencies.py").read_text( + encoding="utf-8" + ) + reader_factory = dependencies_source.split( + "def _default_v2_metadata_reader_factory" + )[1].split("\ndef ")[0] + assert "get_v2_read_session_factory" in reader_factory + assert dependencies_source.count("get_v2_read_session_factory") == 2 From 87c17b109c23fde4fa2f92e3eec70497d2d7c296 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:30:44 +0400 Subject: [PATCH 21/33] Describe the metadata stores in the layout guide and drop FoundParameters The layout guide listed neither the metadata route package's query_dependencies and response_cache modules nor the service package's selected_catalogs store; it now describes each and where process-wide state belongs. FoundParameters held ORM rows and a derivable name set in the framework-independent types module; it is removed. The service passes the parameters, their values grouped by name (through the now public dated_values_by_parameter_name transformation) and the missing names to the transformation, which no longer recomputes any of them. Co-Authored-By: Claude Opus 5.5 --- .../skills/v2-code-organization.md | 16 +++++++++++++ .../services/v2/metadata/services.py | 23 ++++++++----------- .../services/v2/metadata/transformations.py | 23 ++++++++----------- .../services/v2/metadata/types.py | 13 +---------- 4 files changed, 35 insertions(+), 40 deletions(-) diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index bbe997b65..b32fe2af1 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -21,9 +21,18 @@ user_policies/ metadata/ response_models.py common.py + query_dependencies.py + response_cache.py *_routes.py ``` +`common.py` also holds the route class that gives metadata responses their +caching headers. `query_dependencies.py` defines the typed FastAPI +dependencies for the shared query models once, for every metadata route +module to import. `response_cache.py` keeps serialized whole-version +responses (bodies and ETags) for the process; it is HTTP-level, so it lives +with the routes. + `errors.py` defines the strict error response and serialization function shared by every API v2 resource. Application-wide exception handling distinguishes API v2 from non-v2 requests but must not select error behavior by matching @@ -52,6 +61,13 @@ services/v2// deletes.py ``` +A resource may also add a module for process-wide state its service reuses +across requests, named for what it keeps, such as the metadata package's +`selected_catalogs.py` (each catalog version a request has selected, which +never changes once published). Such a module holds plain data, constructs no +SQL and owns no session; the composition code creates one instance per process +and injects it into the service. + Only add CRUD connector modules for operations the resource supports. Append a specific descriptor when one operation file would become too broad, such as `reads_variables.py` or `reads_parameter_tree.py`. diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 017e3e5b4..4496e3b30 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -42,6 +42,7 @@ ) from policyengine_api.services.v2.metadata.transformations import ( ancestor_paths, + dated_values_by_parameter_name, metadata_catalog_parameters, metadata_catalog_variables, metadata_dataset, @@ -61,7 +62,6 @@ part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( - FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParametersResult, MetadataCatalogVariablesResult, @@ -657,30 +657,25 @@ def get_catalog_parameters( parameters = read_parameters_by_name( session, model_version_id=model_version_id, parameter_names=requested_names ) - found_parameters = FoundParameters( - parameters=parameters, - names={parameter.name for parameter in parameters}, - ) + found_names = {parameter.name for parameter in parameters} value_rows = read_current_law_values_by_parameter_name( - session, - model_version_id=model_version_id, - parameter_names=found_parameters.names, + session, model_version_id=model_version_id, parameter_names=found_names ) node_rows = read_parameter_node_labels( session, model_version_id=model_version_id, node_names={ - ancestor - for name in found_parameters.names - for ancestor in ancestor_paths(name) + ancestor for name in found_names for ancestor in ancestor_paths(name) }, ) return metadata_catalog_parameters( selected, - requested_names=requested_names, - found_parameters=found_parameters, - value_rows=value_rows, + parameters=parameters, + values_by_name=dated_values_by_parameter_name( + value_rows, parameter_names=found_names + ), node_rows=node_rows, + missing_names=requested_names - found_names, ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 938027741..48efad3a1 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,7 +8,6 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( CurrentLawValueRow, - FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParameter, MetadataCatalogParametersResult, @@ -153,7 +152,7 @@ def metadata_catalog_variables( current-law values of those parameters in name and start-date order. Each path gets an entry, empty when no value carries its name. """ - values_by_path = _dated_values_by_parameter_name( + values_by_path = dated_values_by_parameter_name( part_values, parameter_names=part_paths ) return MetadataCatalogVariablesResult( @@ -164,7 +163,7 @@ def metadata_catalog_variables( ) -def _dated_values_by_parameter_name( +def dated_values_by_parameter_name( value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] ) -> dict[str, list[MetadataDatedValue]]: """Group value rows by parameter name, with an empty list for each other name. @@ -216,21 +215,17 @@ def ancestor_paths(parameter_name: str) -> list[str]: def metadata_catalog_parameters( selected: SelectedCatalog, *, - requested_names: set[str], - found_parameters: FoundParameters, - value_rows: list[CurrentLawValueRow], + parameters: list[Parameter], + values_by_name: dict[str, list[MetadataDatedValue]], node_rows: list[ParameterNodeLabelRow], + missing_names: set[str], ) -> MetadataCatalogParametersResult: """Build the requested parameters with their values and ancestor labels. - value_rows are the found parameters' current-law values and node_rows the - stored labels of their ancestor nodes. + values_by_name maps each parameter's name to its current-law values (see + dated_values_by_parameter_name), node_rows are the stored labels of their + ancestor nodes, and missing_names the requested names the version lacks. """ - parameters = found_parameters.parameters - found_names = found_parameters.names - values_by_name = _dated_values_by_parameter_name( - value_rows, parameter_names=found_names - ) return MetadataCatalogParametersResult( policyengine_version=selected.policyengine_version, catalog_format=selected.model_version.catalog_format, @@ -242,7 +237,7 @@ def metadata_catalog_parameters( for parameter in parameters ], node_labels={row.name: row.label for row in node_rows}, - missing_names=sorted(requested_names - found_names), + missing_names=sorted(missing_names), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index fecb656fa..812f73638 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,14 +5,11 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar +from typing import Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue -if TYPE_CHECKING: - from policyengine_api.data.v2.models import Parameter - @dataclass(frozen=True) class ParameterFilter: @@ -62,14 +59,6 @@ class CurrentLawValueRow(NamedTuple): end_date: datetime | None -@dataclass(frozen=True) -class FoundParameters: - """The stored parameters a by-name read found, and their names.""" - - parameters: list[Parameter] - names: set[str] - - class ParameterNodeLabelRow(NamedTuple): """A parameter node's name and its stored label (None when it has none).""" From 4bbe04f4edc9ecc75b18aede96cf386fd418932f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:42:31 +0400 Subject: [PATCH 22/33] Give parameter-tree leaves the economy and household flags /v2/parameters/children built each leaf's parameter summary by hand from a few selected columns and left economy and household out, so a leaf always reported null, which the shared schema reserves for versions published before catalog format 2. The tree read now selects the stored Parameter, and each leaf is built with metadata_parameter, the same builder the parameter routes use, so the two can no longer differ. Co-Authored-By: Claude Opus 5.5 --- .../reads_parameter_tree.py | 8 ++--- .../services/v2/metadata/transformations.py | 36 +++++++------------ tests/unit/v2/test_metadata_service.py | 20 +++++++++++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index cd945768e..49f232a8e 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -114,11 +114,7 @@ def _parameter_children_statement( ), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), - parameter_id.label("parameter_id"), - col(Parameter.label).label("parameter_label"), - col(Parameter.description).label("parameter_description"), - col(Parameter.data_type).label("parameter_data_type"), - col(Parameter.unit).label("parameter_unit"), + Parameter, ) .select_from( paths.outerjoin( @@ -152,7 +148,7 @@ def read_parameter_children( """Return a page of the direct children of parent_path in a model version's tree. Each row has the child's path, label, type (node or parameter), child count - and, for parameters, the parameter's summary columns. The session's dialect + and the stored Parameter (None for a node). The session's dialect selects SQLite or PostgreSQL string functions; limit + 1 rows are fetched so the caller can tell whether another page exists. """ diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 48efad3a1..8234fb8db 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -323,31 +323,21 @@ def metadata_region(region: Region) -> MetadataRegion: def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: """Build parameter-tree children from read_parameter_children rows. - Parameter leaves carry the parameter's summary, and a child with no label is - labelled with the last segment of its path. + Parameter leaves carry the same summary as the parameter routes, and a child + with no label is labelled with the last segment of its path. """ - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) + return [ + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=( + metadata_parameter(row.Parameter) if row.type == "parameter" else None + ), ) - return items + for row in rows + ] def national_region(regions: list[Region], country_id: str) -> Region | None: diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 458ab34a4..fbe09f72c 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1117,6 +1117,26 @@ def test_value_and_node_label_reads_return_typed_rows( assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] +def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( + catalog_session: Session, +) -> None: + service = _service(catalog_session) + + leaf = next( + child + for child in service.list_parameter_children( + "us", parent_path="gov.example" + ).items + if child.type == "parameter" + ) + page_item = service.list_parameters( + "us", parameter_filter=ParameterFilter(names=("gov.example.rate",)) + ).items[0] + + assert leaf.parameter == page_item + assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From db0b4ae418653aaf2706e61656debe60b27de042 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:50:20 +0400 Subject: [PATCH 23/33] Give every request waiting on a whole-version read that read's outcome MetadataResponseCache held a build lock for the whole read, and when the read failed each waiting request took the lock in turn and read again: ten concurrent first requests with a failing read ran ten reads one after another. A waiting request's cleanup could also delete a newer read's lock. Each key now has at most one read in progress; requests arriving meanwhile wait for it and receive its outcome, the kept response or their own copy of its error response (middleware adds request-specific headers, so error responses are not shared). Nothing is kept on failure, so the first request after it reads again, and a read removes only its own entry. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 11 ++- .../fastapi_routes/v2/metadata/common.py | 2 +- .../v2/metadata/response_cache.py | 98 ++++++++++++++----- tests/unit/v2/test_metadata_response_cache.py | 90 +++++++++++++++++ 4 files changed, 173 insertions(+), 28 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 9357eb48c..bf72dc038 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -134,10 +134,13 @@ so eight entries hold about 11 MB for both countries' two routes at two versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only -the first successful request per process reads the database; concurrent first -requests wait for one read, and a failed read is not kept. Measured locally against the full 6.10.0 US -catalog, the variables route takes about 160 ms when read and about 25 ms -when kept, most of it compression. +the first successful request per process reads the database. Requests that +arrive while that read is in progress wait for it and all receive its outcome, +the kept response or a copy of its error response, so one failing read is not +repeated by each of them; a failed read is not kept, so the first request after +it reads again. Measured locally against the full 6.10.0 US catalog, the +variables route takes about 160 ms when read and about 25 ms when kept, most of +it compression. ## Configuration and credential boundaries diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 3b54fc2fa..3e623cf83 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -285,7 +285,7 @@ def read_whole_version_resource( if cache is None: return read_resource(dependencies, response_type, operation) - def build() -> BuiltResponse | JSONResponse: + def build() -> BuiltResponse | Response: result = read_resource(dependencies, response_type, operation) if isinstance(result, MetadataResourceSuccessResponse): return BuiltResponse( diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 22ea2d348..039c9dd03 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -8,11 +8,11 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from starlette.responses import Response + if TYPE_CHECKING: from collections.abc import Callable - from starlette.responses import JSONResponse - DEFAULT_RESPONSE_CAPACITY = 8 KeptResponseKey = tuple[str, str, str] @@ -40,6 +40,35 @@ class KeptResponse: entity_tag: str +@dataclass(frozen=True) +class FailedRead: + """A failed whole-version read, given to the requests that waited on it.""" + + status_code: int + body: bytes + + def response(self) -> Response: + """Return a new error response for one waiting request. + + Each request gets its own response object, because middleware adds + request-specific headers to the response it sends. + """ + return Response( + content=self.body, + status_code=self.status_code, + media_type="application/json", + ) + + +class _InFlightRead: + """One read in progress for a key, and how many requests wait on it.""" + + def __init__(self) -> None: + self.done = threading.Event() + self.outcome: KeptResponse | FailedRead | None = None + self.waiters = 0 + + class MetadataResponseCache: """Keep whole-version responses, built once each per process. @@ -51,10 +80,10 @@ class MetadataResponseCache: are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes uncompressed, so the default of eight holds about 11 MB for both countries' two whole-version routes at two versions, and at most about - 31 MB if every entry were that large. Requests for a response being built - wait for that build, and its storage, instead of starting another, and a - failed read (a v2 error response) is returned but not kept, so the next - request tries again. + 31 MB if every entry were that large. Requests that arrive while a read + for the same key is in progress wait for it, and all receive its outcome: + the kept response, or a copy of its error response. A failed read is not + kept, so the first request after it reads again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: @@ -66,7 +95,7 @@ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() self._running_versions: dict[tuple[str, str], str] = {} self._lock = threading.Lock() - self._build_locks: dict[tuple[str, str, str | None], threading.Lock] = {} + self._builds: dict[tuple[str, str, str | None], _InFlightRead] = {} def _stored( self, route: str, country_id: str, requested_version: str | None @@ -90,32 +119,55 @@ def get_or_build( route: str, country_id: str, requested_version: str | None, - build: Callable[[], BuiltResponse | JSONResponse], - ) -> KeptResponse | JSONResponse: + build: Callable[[], BuiltResponse | Response], + ) -> KeptResponse | Response: """Return the kept response, calling ``build`` only when none is kept. - ``requested_version`` None means the running version. + ``requested_version`` None means the running version. Requests that + arrive while a read for the same key is in progress wait for it and + receive its outcome: the kept response, or a copy of its error + response. A failed read is not kept, so the next request after it + reads again. """ kept = self._stored(route, country_id, requested_version) if kept is not None: return kept - build_key = (route, country_id, requested_version) + key = (route, country_id, requested_version) with self._lock: - build_lock = self._build_locks.setdefault(build_key, threading.Lock()) - with build_lock: - try: - kept = self._stored(route, country_id, requested_version) - if kept is not None: - return kept + in_flight = self._builds.get(key) + if in_flight is None: + in_flight = _InFlightRead() + self._builds[key] = in_flight + reads_itself = True + else: + in_flight.waiters += 1 + reads_itself = False + if not reads_itself: + in_flight.done.wait() + outcome = in_flight.outcome + if outcome is None: + # The read raised instead of returning a response: try again. + return self.get_or_build(route, country_id, requested_version, build) + return outcome.response() if isinstance(outcome, FailedRead) else outcome + try: + kept = self._stored(route, country_id, requested_version) + if kept is None: built = build() if not isinstance(built, BuiltResponse): + in_flight.outcome = FailedRead( + status_code=built.status_code, body=bytes(built.body) + ) return built - # Stored before the build lock is released, so a request - # waiting on it finds the response instead of reading again. - return self._keep(route, country_id, requested_version, built) - finally: - with self._lock: - self._build_locks.pop(build_key, None) + kept = self._keep(route, country_id, requested_version, built) + in_flight.outcome = kept + return kept + finally: + with self._lock: + # Only this read's own entry is removed; a later read for the + # same key keeps its entry. + if self._builds.get(key) is in_flight: + del self._builds[key] + in_flight.done.set() def _keep( self, diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index 4d6704a36..a0a788c1c 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading +import time import pytest from starlette.responses import JSONResponse @@ -173,6 +174,95 @@ def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: assert second_results == [first] +def _wait_for_waiters(cache: MetadataResponseCache, key: tuple, count: int) -> None: + # Polls until every request has registered as waiting on the read in + # progress, so the result does not depend on thread timing. + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + with cache._lock: + in_flight = cache._builds.get(key) + if in_flight is not None and in_flight.waiters == count: + return + time.sleep(0.001) + pytest.fail(f"{count} requests did not wait on the read in progress") + + +@pytest.mark.parametrize("read_outcome", ["failure", "success"]) +def test_waiting_requests_receive_the_outcome_of_the_one_read( + read_outcome: str, +) -> None: + read_fails = read_outcome == "failure" + cache = MetadataResponseCache() + builds: list[str] = [] + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> BuiltResponse | JSONResponse: + builds.append("built") + release.wait() + if read_fails: + return JSONResponse({"status": "error"}, status_code=503) + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + results: list[object] = [] + + def request() -> None: + results.append(cache.get_or_build(*key, blocked_build)) + + owner = threading.Thread(target=request, daemon=True) + waiters = [threading.Thread(target=request, daemon=True) for _ in range(9)] + try: + owner.start() + while not builds: + time.sleep(0.001) + for thread in waiters: + thread.start() + _wait_for_waiters(cache, key, 9) + finally: + release.set() + for thread in [owner, *waiters]: + thread.join(timeout=10) + + assert builds == ["built"] + assert len(results) == 10 + if read_fails: + assert all(result.status_code == 503 for result in results) + assert {bytes(result.body) for result in results} == {b'{"status":"error"}'} + # Each request gets its own response object, so middleware headers + # added for one request cannot reach another. + assert len({id(result) for result in results}) == 10 + else: + assert all(result is results[0] for result in results) + assert key not in cache._builds + + +def test_a_finished_read_leaves_a_newer_reads_entry_in_place() -> None: + cache = MetadataResponseCache() + building = threading.Event() + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> JSONResponse: + building.set() + release.wait() + return JSONResponse({"status": "error"}, status_code=503) + + owner = threading.Thread( + target=lambda: cache.get_or_build(*key, blocked_build), daemon=True + ) + newer_read = object() + try: + owner.start() + building.wait(timeout=10) + with cache._lock: + cache._builds[key] = newer_read + finally: + release.set() + owner.join(timeout=10) + + assert cache._builds[key] is newer_read + + def test_capacity_must_be_positive() -> None: with pytest.raises(ValueError, match="capacity"): MetadataResponseCache(capacity=0) From ba1b435c66ccd028bc1abf0e775861f4860bca98 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:52:35 +0400 Subject: [PATCH 24/33] Give cacheable 200 and 304 responses the same Vary, each value once with_cache_headers gave a 304 only Vary: Origin, while gzip added Accept-Encoding only to compressed 200 responses; a 304 must carry the Vary its 200 would. Every cacheable metadata response and its 304 now name Origin and Accept-Encoding. Starlette's CORS and gzip middleware append to Vary without checking for values already present, which gave "Origin" twice for cross-origin requests, so a small UniqueVaryMiddleware now wraps the CORS middleware and lists each Vary value once. Tested through the whole middleware stack, with and without Origin. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 11 ++-- policyengine_api/asgi_factory.py | 6 +- .../fastapi_routes/v2/metadata/common.py | 37 +++++++----- policyengine_api/vary_headers.py | 56 +++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 35 ++++++++++++ 5 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 policyengine_api/vary_headers.py diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index bf72dc038..8f7a8431a 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -118,10 +118,13 @@ whichever lifetime would otherwise apply (`max-age=0` if its headers are written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. Every cacheable response, and every 304, carries -`Vary: Origin` (added to any other `Vary` value, such as compression's -`Accept-Encoding`): the API reflects every origin in -`Access-Control-Allow-Origin` and a request without `Origin` gets no such -header, so a shared cache must keep responses apart by `Origin`. `ETag` is +`Vary: Origin, Accept-Encoding`, each value once: the API reflects every +origin in `Access-Control-Allow-Origin` and a request without `Origin` gets no +such header, so a shared cache must keep responses apart by `Origin`, and gzip +compresses bodies for clients that accept it. A 304 carries the same `Vary` as +its 200. The CORS and gzip middleware append to `Vary` without checking for +values already present, so the application's outermost middleware +(`UniqueVaryMiddleware`) lists each value once. `ETag` is listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable without that). diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index cba4c00bc..8f74b1bb2 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -53,6 +53,7 @@ current_observability_id, generate_request_id, ) +from policyengine_api.vary_headers import UniqueVaryMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterable @@ -471,7 +472,9 @@ def create_asgi_app( app.mount("/", WSGIMiddleware(wsgi_app)) # The public API already permits every web origin. Use the standard ASGI # implementation while preserving the existing reflected-origin response. - return CORSMiddleware( + # UniqueVaryMiddleware wraps it, because CORS and gzip each append to Vary + # values a route may already have named. + cors_app = CORSMiddleware( app=app, allow_origin_regex=".*", allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], @@ -486,3 +489,4 @@ def create_asgi_app( allow_credentials=False, max_age=600, ) + return UniqueVaryMiddleware(cors_app) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 3e623cf83..bdffdd410 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -34,6 +34,7 @@ InvalidMetadataPageError, MetadataResourceNotFoundError, ) +from policyengine_api.vary_headers import unique_vary_values if TYPE_CHECKING: from collections.abc import Callable, Coroutine @@ -96,7 +97,10 @@ "schema": {"type": "string"}, }, "Vary": { - "description": "Includes Origin, so caches keep responses apart by origin.", + "description": ( + "Origin, Accept-Encoding: caches keep responses apart by origin " + "and by compression; a 304 carries the same value." + ), "schema": {"type": "string"}, }, } @@ -206,7 +210,7 @@ def with_cache_headers(request: Request, response: Response) -> Response: running version for five minutes, and one whose route read for a UTC day (``UTC_DAY_READ``) no later than the midnight after that day. Every such response, and the 304 a request whose ``If-None-Match`` names the ETag - gets instead, varies by ``Origin``. + gets instead, varies by ``Origin`` and ``Accept-Encoding``. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -220,25 +224,30 @@ def with_cache_headers(request: Request, response: Response) -> Response: headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): not_modified = Response(status_code=304, headers=headers) - not_modified.headers["Vary"] = _vary_with_origin(None) + not_modified.headers["Vary"] = _cacheable_vary(None) return not_modified response.headers.update(headers) - response.headers["Vary"] = _vary_with_origin(response.headers.get("vary")) + response.headers["Vary"] = _cacheable_vary(response.headers.get("vary")) return response -def _vary_with_origin(existing: str | None) -> str: - """Return a Vary value that keeps ``existing`` and names Origin once. +CACHEABLE_VARY_VALUES = ("Origin", "Accept-Encoding") +"""What every cacheable metadata response, and its 304, varies by. + +The API reflects every origin in Access-Control-Allow-Origin, and a request +without Origin gets no such header, so a shared cache must keep responses +apart by Origin; gzip compresses the body for clients that accept it. A 304 +carries the Vary its 200 would. +""" + + +def _cacheable_vary(existing: str | None) -> str: + """Return ``existing`` with Origin and Accept-Encoding added if absent. - The API reflects every origin in Access-Control-Allow-Origin, and a request - without Origin gets no such header, so a shared cache must store responses - per Origin value or it could serve one to a cross-origin request that the - browser then rejects. + The CORS and gzip middleware may append them again; the application's + outermost middleware (UniqueVaryMiddleware) then lists each value once. """ - values = [value.strip() for value in (existing or "").split(",") if value.strip()] - if not any(value.lower() == "origin" for value in values): - values.append("Origin") - return ", ".join(values) + return unique_vary_values(", ".join([existing or "", *CACHEABLE_VARY_VALUES])) def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: diff --git a/policyengine_api/vary_headers.py b/policyengine_api/vary_headers.py new file mode 100644 index 000000000..98d834086 --- /dev/null +++ b/policyengine_api/vary_headers.py @@ -0,0 +1,56 @@ +"""ASGI middleware that lists each Vary value of a response once.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from starlette.datastructures import MutableHeaders + +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +def unique_vary_values(vary: str) -> str: + """Return ``vary`` with each value listed once, in first-seen order. + + Values are compared without regard to case, as HTTP field names are. + """ + values: list[str] = [] + seen: set[str] = set() + for value in vary.split(","): + name = value.strip() + if name and name.lower() not in seen: + seen.add(name.lower()) + values.append(name) + return ", ".join(values) + + +class UniqueVaryMiddleware: + """List each Vary value of every HTTP response once. + + Starlette's CORS and gzip middleware append Origin and Accept-Encoding to + Vary without checking whether a route already named them. This wraps the + outermost middleware, so the response that leaves the application names + each value once. + """ + + def __init__(self, app: ASGIApp) -> None: + """Wrap ``app``.""" + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Pass the request through, rewriting a duplicated Vary header.""" + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_unique_vary(message: Message) -> None: + if message["type"] == "http.response.start": + message.setdefault("headers", []) + headers = MutableHeaders(scope=message) + vary = headers.get("vary") + if vary is not None: + headers["vary"] = unique_vary_values(vary) + await send(message) + + await self.app(scope, receive, send_with_unique_vary) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 792d7c05c..e0ce842ea 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1386,3 +1386,38 @@ def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: assert "content" not in responses["304"] assert "policyengine_version" not in responses["400"]["description"] assert "path" in responses["400"]["description"] + + +def _vary_value_counts(response) -> dict[str, int]: + counts: dict[str, int] = {} + for value in response.headers.get("vary", "").split(","): + if value.strip(): + counts[value.strip().lower()] = counts.get(value.strip().lower(), 0) + 1 + return counts + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( + origin: str | None, +) -> None: + # Through the whole middleware stack (gzip, then CORS): a 304 must carry + # the Vary its 200 would, and each value appears once. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {"Accept-Encoding": "gzip"} + if origin is not None: + headers["Origin"] = origin + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + compressed = client.get("/v2/model-catalog?country_id=us", headers=headers) + not_modified = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": compressed.headers["etag"]}, + ) + + assert compressed.headers.get("content-encoding") == "gzip" + assert not_modified.status_code == 304 + expected = {"origin": 1, "accept-encoding": 1} + assert _vary_value_counts(small) == expected + assert _vary_value_counts(compressed) == expected + assert _vary_value_counts(not_modified) == expected From ec9a9c7c23f40dcc3bb9108ee4bd0ed4a20b8591 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:54:19 +0400 Subject: [PATCH 25/33] Declare 400 only on the metadata route with the country in its path Every metadata route's OpenAPI entry listed 400 for an unsupported country in the path, but only /v2/tax-benefit-models/by-country/ {country_id} takes the country in its path; on every other route the shared query models reject an invalid country, version or page with 422 before any read, so 400 cannot occur there. The description now sits only on that route. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/v2/metadata/common.py | 6 ++++- .../v2/metadata/model_routes.py | 2 ++ tests/unit/v2/test_metadata_routes.py | 23 +++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index bdffdd410..a2dab29ee 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -48,7 +48,7 @@ ) from policyengine_api.query_parameters import CatalogQuery -ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { +PATH_COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, "description": ( @@ -57,6 +57,10 @@ "version, returns 422." ), }, +} +"""The extra response of the one metadata route that takes the country in its path.""" + +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 404: { "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 3482ee108..ae3127725 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( METADATA_RESPONSES, + PATH_COUNTRY_ERROR_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -69,6 +70,7 @@ def list_models( @router.get( "/tax-benefit-models/by-country/{country_id}", response_model=MetadataModelSelectionResponse, + responses=PATH_COUNTRY_ERROR_RESPONSES, summary="Get a country model and selected PolicyEngine.py version", ) def get_model_by_country( diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e0ce842ea..bc5eafb7a 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -650,20 +650,15 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: for path in metadata_paths: operation = schema["paths"][path]["get"] - assert set(operation["responses"]) >= { - "200", - "400", - "404", - "405", - "422", - "500", - "503", - } + error_statuses = ["404", "405", "422", "500", "503"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + error_statuses.append("400") + assert set(operation["responses"]) >= {"200", *error_statuses} success_schema = operation["responses"]["200"]["content"]["application/json"][ "schema" ] assert success_schema["$ref"].startswith("#/components/schemas/Metadata") - for status in ("400", "404", "405", "422", "500", "503"): + for status in error_statuses: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] @@ -1384,8 +1379,12 @@ def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: assert "$ref" in responses["200"]["content"]["application/json"]["schema"] assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} assert "content" not in responses["304"] - assert "policyengine_version" not in responses["400"]["description"] - assert "path" in responses["400"]["description"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + assert "path" in responses["400"]["description"] + else: + # Query parameters are validated first (422), so only a country + # given in the path can return 400. + assert "400" not in responses, path def _vary_value_counts(response) -> dict[str, int]: From 069d739f8db38e9472fea99bfee75f5bc9adf0b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:19 +0400 Subject: [PATCH 26/33] Test autocommit reads on one pooled connection and the production reader The autocommit test used a NullPool engine, so it could not notice a pooled connection left in autocommit mode for the next writer. It now uses a pooled engine with one connection and checks that the writer that takes it next is transactional (with autocommit left on, the test fails). A new test runs the production combination of read sessions, SelectedCatalogStore and the response cache through the routes and counts the statements each request sends. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_v2_metadata_routes.py | 110 +++++++++++++++++-- 1 file changed, 103 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 4930e3abb..e72cbd17f 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -550,30 +550,126 @@ def test_postgres_metadata_reads_send_no_transaction_statements( ) -> None: # Each metadata read is one statement on rows a published catalog never # changes, so the read session runs in autocommit mode: psycopg then sends - # no BEGIN before the reads and no ROLLBACK when the session closes. + # no BEGIN before the reads and no ROLLBACK when the session closes. The + # engine has one pooled connection, so the writer below must get the same + # connection back in the ordinary, transactional mode. + from psycopg.pq import TransactionStatus + from policyengine_api.data.v2.database import build_v2_read_session_factory + del published_engine # publishes the catalog this pooled engine reads + pooled_engine = create_engine(_disposable_url(), pool_size=1, max_overflow=0) + read_connections: list[object] = [] autocommit_modes: list[bool] = [] def record_mode(connection, _cursor, _statement, *_args) -> None: - autocommit_modes.append(connection.connection.dbapi_connection.autocommit) + dbapi_connection = connection.connection.dbapi_connection + read_connections.append(dbapi_connection) + autocommit_modes.append(dbapi_connection.autocommit) - event.listen(published_engine, "before_cursor_execute", record_mode) try: + event.listen(pooled_engine, "before_cursor_execute", record_mode) service = V2MetadataService( - MetadataDatabaseSession(build_v2_read_session_factory(published_engine)()), + MetadataDatabaseSession(build_v2_read_session_factory(pooled_engine)()), running_policyengine_version=POLICYENGINE_VERSION, ) result = service.list_parameters("us") service.close() + event.remove(pooled_engine, "before_cursor_execute", record_mode) + + with Session(pooled_engine) as writer: + writer_connection = writer.connection().connection.dbapi_connection + writer.execute(text("SELECT 1")) + writer_status = writer_connection.info.transaction_status + writer_autocommit = writer_connection.autocommit finally: - event.remove(published_engine, "before_cursor_execute", record_mode) + pooled_engine.dispose() assert [item.name for item in result.items] == ["gov.example.rate"] assert autocommit_modes assert all(autocommit_modes) - with published_engine.connect() as connection: - assert connection.connection.dbapi_connection.autocommit is False + assert {id(connection) for connection in read_connections} == { + id(writer_connection) + } + assert writer_autocommit is False + assert writer_status == TransactionStatus.INTRANS + + +def test_postgres_production_metadata_reader_keeps_catalogs_and_responses( + published_engine: Engine, +) -> None: + # The production combination: autocommit read sessions, the process's + # selected catalogs and its whole-version response cache. + from policyengine_api.data.v2.database import build_v2_read_session_factory + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) + + read_factory = build_v2_read_session_factory(published_engine) + selected_catalogs = SelectedCatalogStore() + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_metadata_reader_factory=lambda: V2MetadataService( + MetadataDatabaseSession(read_factory()), + running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, + ), + v2_metadata_response_cache=MetadataResponseCache(), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + query = {"country_id": "us", "policyengine_version": POLICYENGINE_VERSION} + + event.listen(published_engine, "before_cursor_execute", record_statement) + try: + first_variables = client.get( + "/v2/model-catalog/variables", params={"country_id": "us"} + ) + statements_for_first = len(statements) + repeated_variables = client.get("/v2/model-catalog/variables", params=query) + statements_for_repeat = len(statements) - statements_for_first + parameters = client.get( + "/v2/parameters", params={"country_id": "us", "name": "gov.example.rate"} + ) + statements_for_parameters = ( + len(statements) - statements_for_first - statements_for_repeat + ) + finally: + event.remove(published_engine, "before_cursor_execute", record_statement) + + assert first_variables.status_code == 200 + assert repeated_variables.content == first_variables.content + assert repeated_variables.headers["etag"] == first_variables.headers["etag"] + # The running version's selection and the variables read (the fixture's + # variables name no parameter-defined parts, so no value read); then + # nothing for the explicit request, which shares the kept response; then + # only the parameter read, the running version's selection being kept. + assert statements_for_first == 2 + assert statements_for_repeat == 0 + assert statements_for_parameters == 1 + assert [item["name"] for item in parameters.json()["result"]["items"]] == [ + "gov.example.rate" + ] + assert selected_catalogs.get(("us", None)) is not None @pytest.mark.parametrize( From 67785b5a8dd007b777b830f47a451ca9145bf4a5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:56 +0400 Subject: [PATCH 27/33] Bring the 1.16 changelog fragment in line with the caching behavior The fragment did not mention Vary, the readable ETag or how requests waiting on a whole-version read are answered, and described the current-value lifetime loosely. It now states each as the code does. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index a80f72ae6..09a26ee81 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,5 +1,5 @@ -Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the next UTC midnight for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. -Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. From 1e93eaaddd9243dc690b07fd14221def3b999700 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:06:40 +0400 Subject: [PATCH 28/33] Import the connector modules in a test instead of suppressing line length Two imports in the typed-rows test named the connector functions by long dotted paths and suppressed E501. The test now imports the two connector modules and calls the functions through them, so it needs no suppression. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index fbe09f72c..798c6ce02 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1090,20 +1090,18 @@ def test_each_country_and_version_keeps_its_own_selection( def test_value_and_node_label_reads_return_typed_rows( catalog_session: Session, ) -> None: - from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 - read_parameter_node_labels, - ) - from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( # noqa: E501 - read_current_law_values_by_parameter_name, + from policyengine_api.services.v2.metadata.database_connectors import ( + reads_parameter_nodes, + reads_parameters, ) model_version = _us_model_version(catalog_session) - values = read_current_law_values_by_parameter_name( + values = reads_parameters.read_current_law_values_by_parameter_name( catalog_session, model_version_id=model_version.id, parameter_names={"gov.example.rate"}, ) - labels = read_parameter_node_labels( + labels = reads_parameter_nodes.read_parameter_node_labels( catalog_session, model_version_id=model_version.id, node_names={"gov.example", "gov.absent"}, From 02e120a0f509cca2fbe7c9b9d5651da6b8f76379 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:44:06 +0400 Subject: [PATCH 29/33] Serve each variable's unit and definition period The app's v3 household builder and breakdown need each variable's unit (currency formatting) and definition period (per-month or per-year labels and household validation), which catalog format 2 now stores. Every variable response, in /v2/model-catalog/variables and in /v2/variables pages, now carries unit and definition_period, declared in OpenAPI and null for versions published before catalog format 2. The 6.10.0 US variables body grows from 3,910,063 to 4,190,824 bytes uncompressed (0.589 MB compressed, from 0.583 MB), and the documented bound of the response store is updated to match. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 10 +-- .../v2/metadata/response_cache.py | 6 +- .../services/v2/metadata/transformations.py | 2 + .../services/v2/metadata/types.py | 11 ++- tests/integration/test_v2_metadata_routes.py | 2 + tests/unit/v2/test_metadata_routes.py | 12 ++++ tests/unit/v2/test_metadata_service.py | 67 +++++++++++++++++++ 8 files changed, 101 insertions(+), 11 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 09a26ee81..bbfc742cb 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,5 +1,5 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. -GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, enum labels, unit and definition period), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8f7a8431a..3785f5fbc 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -45,8 +45,10 @@ parameters are rejected. `GET /v2/model-catalog/variables` (same query) returns every variable of the version in name order, in one response, with the catalog format 2 fields: `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, -`subtracts_parameter_path` and `enum_values` (each value's name and its label, -in the type the country package declares). Where a variable's `adds` or +`subtracts_parameter_path`, `enum_values` (each value's name and its label, +in the type the country package declares), `unit` (for example +`currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which +the household builder uses for formatting and per-period labels. Where a variable's `adds` or `subtracts` list comes from a parameter, `part_parameter_values` maps that parameter's path to its current-law values in start-date order (`start_date`, `end_date`, `value`), so a client picks the list for the year @@ -132,8 +134,8 @@ without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, for up to eight route, country and version combinations (least recently used -dropped first; the 6.10.0 US variables body is 3,910,063 bytes uncompressed, -so eight entries hold about 11 MB for both countries' two routes at two +dropped first; the 6.10.0 US variables body is 4,190,824 bytes uncompressed, +so eight entries hold about 12 MB for both countries' two routes at two versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 039c9dd03..1a1ad0b2d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -77,10 +77,10 @@ class MetadataResponseCache: are kept under the exact version they were read from: a request without a version (the running version, fixed for the process) and one naming that version share one response. The most recently used ``capacity`` responses - are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes - uncompressed, so the default of eight holds about 11 MB for both + are kept: the US variables body of the 6.10.0 catalog is 4,190,824 bytes + uncompressed, so the default of eight holds about 12 MB for both countries' two whole-version routes at two versions, and at most about - 31 MB if every entry were that large. Requests that arrive while a read + 34 MB if every entry were that large. Requests that arrive while a read for the same key is in progress wait for it, and all receive its outcome: the kept response, or a copy of its error response. A failed read is not kept, so the first request after it reads again. diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 8234fb8db..00c197cac 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -126,6 +126,8 @@ def metadata_variable(variable: Variable) -> MetadataVariable: for enum_value in variable.enum_values ] ), + unit=variable.unit, + definition_period=variable.definition_period, ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 812f73638..52e8be0a0 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -119,9 +119,12 @@ class MetadataVariable(StrictResponseModel): subtracted from this one. When the model takes that list from a parameter, adds_parameter_path or subtracts_parameter_path names the parameter, whose dated values give the list for each year (adds and - subtracts hold its value on 2025-01-01). The catalog format 2 fields - (is_input_variable, hidden_input, module_name, the two parameter paths - and enum_values) are null in versions published before that format. + subtracts hold its value on 2025-01-01). unit is the country package's + unit (for example ``currency-USD``) and definition_period the period a + value covers (``year``, ``month`` or ``eternity``). The catalog format 2 + fields (is_input_variable, hidden_input, module_name, the two parameter + paths, enum_values, unit and definition_period) are null in versions + published before that format. """ id: UUID @@ -140,6 +143,8 @@ class MetadataVariable(StrictResponseModel): adds_parameter_path: str | None = None subtracts_parameter_path: str | None = None enum_values: list[MetadataEnumValue] | None = None + unit: str | None = None + definition_period: str | None = None class MetadataParameterSummary(StrictResponseModel): diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index e72cbd17f..7f479d183 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -488,6 +488,8 @@ def test_postgres_catalog_variables_carry_part_parameter_values_by_date( assert result["variables"][0]["module_name"] == ( "household.income.employment_income" ) + assert result["variables"][0]["unit"] == "currency-USD" + assert result["variables"][0]["definition_period"] == "year" assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" assert result["part_parameter_values"] == { "gov.example.parts": [ diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index bc5eafb7a..176fcc568 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1420,3 +1420,15 @@ def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( assert _vary_value_counts(small) == expected assert _vary_value_counts(compressed) == expected assert _vary_value_counts(not_modified) == expected + + +def test_openapi_declares_each_variables_unit_and_definition_period() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + variable_properties = schema["components"]["schemas"]["MetadataVariable"][ + "properties" + ] + for field in ("unit", "definition_period"): + assert {"type": "string"} in variable_properties[field]["anyOf"] + assert {"type": "null"} in variable_properties[field]["anyOf"] diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 798c6ce02..454d90aad 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -124,6 +124,8 @@ def _insert_country( for enum_value in record.enum_values ] ), + unit=record.unit, + definition_period=record.definition_period, ) for record in country.variables ) @@ -1135,6 +1137,71 @@ def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) +def _uk_model_version(session: Session) -> TaxBenefitModelVersion: + return session.exec( + select(TaxBenefitModelVersion) + .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) + .where( + TaxBenefitModel.name == "policyengine-uk", + TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + ) + ).one() + + +def test_variables_carry_their_unit_and_definition_period( + catalog_session: Session, +) -> None: + uk_version = _uk_model_version(catalog_session) + catalog_session.add_all( + [ + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ), + # A variable published before catalog format 2 stored neither. + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="age", + entity="person", + data_type="int", + default_value=0, + ), + ] + ) + catalog_session.commit() + service = _service(catalog_session) + + us_variables = service.get_catalog_variables("us").variables + uk_variables = { + variable.name: variable + for variable in service.get_catalog_variables("uk").variables + } + us_page = service.list_variables("us").items + + assert [(item.unit, item.definition_period) for item in us_variables] == [ + ("currency-USD", "year") + ] + assert (us_page[0].unit, us_page[0].definition_period) == ( + "currency-USD", + "year", + ) + assert ( + uk_variables["universal_credit"].unit, + uk_variables["universal_credit"].definition_period, + ) == ("currency-GBP", "month") + assert (uk_variables["age"].unit, uk_variables["age"].definition_period) == ( + None, + None, + ) + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 32eb33615b9b8fde26a7ca47beb5b8c387814e8f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:47:13 +0400 Subject: [PATCH 30/33] Require the catalog format 2 fields in the metadata response schemas The format 2 fields on MetadataVariable (input and hidden-input flags, module, the two parameter paths, enum values, unit and definition period) and on MetadataParameterSummary (economy and household) defaulted to None, so the OpenAPI schema left them out of required although every response includes them, and a generated client would type them as possibly missing. They are now required and nullable; every constructor already passes them, and the route test fixtures now do too. A test checks the OpenAPI required lists. Co-Authored-By: Claude Opus 5.5 --- .../services/v2/metadata/types.py | 20 ++++----- tests/unit/v2/test_metadata_routes.py | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 52e8be0a0..63a77944e 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -137,14 +137,14 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None - is_input_variable: bool | None = None - hidden_input: bool | None = None - module_name: str | None = None - adds_parameter_path: str | None = None - subtracts_parameter_path: str | None = None - enum_values: list[MetadataEnumValue] | None = None - unit: str | None = None - definition_period: str | None = None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: list[MetadataEnumValue] | None + unit: str | None + definition_period: str | None class MetadataParameterSummary(StrictResponseModel): @@ -161,8 +161,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None - economy: bool | None = None - household: bool | None = None + economy: bool | None + household: bool | None class MetadataCanonicalParameterValue(StrictResponseModel): diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 176fcc568..037b2952f 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -126,6 +126,14 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: default_value=0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter = MetadataParameterSummary( id=parameter_id, @@ -134,6 +142,8 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: description=None, data_type="float", unit="/1", + economy=True, + household=False, ) parameter_value = MetadataCanonicalParameterValue( id=parameter_value_id, @@ -1432,3 +1442,34 @@ def test_openapi_declares_each_variables_unit_and_definition_period() -> None: for field in ("unit", "definition_period"): assert {"type": "string"} in variable_properties[field]["anyOf"] assert {"type": "null"} in variable_properties[field]["anyOf"] + + +def test_openapi_requires_every_catalog_format_2_field_as_nullable() -> None: + # Every response includes these fields (null for versions published before + # catalog format 2), so a generated client must not type them as missing. + results, _ids = _resource_results() + schemas = ( + _client(lambda: ResourceReader(results)) + .get("/v2/openapi.json") + .json()["components"]["schemas"] + ) + format_2_fields = { + "MetadataVariable": { + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + }, + "MetadataParameterSummary": {"economy", "household"}, + "MetadataCatalogParameter": {"economy", "household"}, + } + + for schema_name, fields in format_2_fields.items(): + schema = schemas[schema_name] + assert set(schema["properties"]) <= set(schema["required"]), schema_name + for field in fields: + assert {"type": "null"} in schema["properties"][field]["anyOf"], field From 504f57e5d28c2de666e64a5f2d64953a428f3015 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:47:35 +0400 Subject: [PATCH 31/33] Look up test model versions through one helper taking the model name _uk_model_version duplicated _us_model_version except for the model name. Both are replaced by _model_version(session, model_name, version), which the tests call with the model they need. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 41 ++++++++++---------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 454d90aad..5772b8187 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -235,13 +235,15 @@ def _add_country_version( session.commit() -def _us_model_version(session: Session) -> TaxBenefitModelVersion: +def _model_version( + session: Session, model_name: str, version: str = POLICYENGINE_VERSION +) -> TaxBenefitModelVersion: return session.exec( select(TaxBenefitModelVersion) .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) .where( - TaxBenefitModel.name == "policyengine-us", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + TaxBenefitModel.name == model_name, + TaxBenefitModelVersion.version == version, ) ).one() @@ -259,7 +261,7 @@ def _service( def test_resource_collection_uses_bounded_pagination_without_counting( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") catalog_session.add( Variable( id=uuid4(), @@ -351,7 +353,7 @@ def record_statement(_connection, _cursor, statement, *_args) -> None: def test_parameter_values_are_separate_canonical_resources( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -412,7 +414,7 @@ def test_current_parameter_value_uses_inclusive_effective_dates( def test_parameter_children_are_loaded_one_level_at_a_time( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") nested_node = ParameterNode( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -547,7 +549,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( def test_dataset_collection_excludes_outputs_and_storage_references( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") output_id = uuid4() catalog_session.add( Dataset( @@ -661,7 +663,7 @@ def test_model_catalog_without_regions_offers_no_economy_calculation( ) -> None: # A household-only country publishes no regions or datasets; its catalog # still serves entities and basic inputs, with empty economy choices. - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") for region in catalog_session.exec( select(Region).where(Region.tax_benefit_model_version_id == model_version.id) ).all(): @@ -686,7 +688,7 @@ def test_model_catalog_without_regions_offers_no_economy_calculation( def test_model_catalog_published_before_format_2_has_empty_new_fields( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") model_version.catalog_format = None model_version.entities = None model_version.basic_inputs = None @@ -714,7 +716,7 @@ def test_model_catalog_with_regions_requires_the_national_region( def _add_variable_with_parameter_named_parts(session: Session) -> None: - model_version = _us_model_version(session) + model_version = _model_version(session, "policyengine-us") parts = Parameter( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -858,7 +860,7 @@ def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> N def _add_bracket_schedule(session: Session) -> None: - model_version = _us_model_version(session) + model_version = _model_version(session, "policyengine-us") session.add_all( [ ParameterNode( @@ -967,7 +969,7 @@ def test_named_parameters_list_names_the_version_lacks( def test_named_parameters_exclude_policy_and_dynamic_values( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -1097,7 +1099,7 @@ def test_value_and_node_label_reads_return_typed_rows( reads_parameters, ) - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") values = reads_parameters.read_current_law_values_by_parameter_name( catalog_session, model_version_id=model_version.id, @@ -1137,21 +1139,10 @@ def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) -def _uk_model_version(session: Session) -> TaxBenefitModelVersion: - return session.exec( - select(TaxBenefitModelVersion) - .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) - .where( - TaxBenefitModel.name == "policyengine-uk", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, - ) - ).one() - - def test_variables_carry_their_unit_and_definition_period( catalog_session: Session, ) -> None: - uk_version = _uk_model_version(catalog_session) + uk_version = _model_version(catalog_session, "policyengine-uk") catalog_session.add_all( [ Variable( From fc8f61f9bc1449d42be60c80a0b8c2820255b4e3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:48:13 +0400 Subject: [PATCH 32/33] Test that both variables routes serve unit and definition period Through the routes, with a response store: /v2/model-catalog/variables (read, then served from the stored body) and /v2/variables pages carry unit and definition_period for the US, for a UK variable with its own values, and as null keys for a version whose catalog_format is null. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 88 ++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 5772b8187..b2f35d7be 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1193,6 +1193,94 @@ def test_variables_carry_their_unit_and_definition_period( ) +@pytest.mark.parametrize( + ("country_id", "version", "expected_catalog_format", "expected_fields"), + [ + ("us", None, 2, {"employment_income": ("currency-USD", "year")}), + ( + "uk", + None, + 2, + { + "employment_income": ("currency-USD", "year"), + "universal_credit": ("currency-GBP", "month"), + }, + ), + # A version published before catalog format 2 stored neither field. + ("us", "9.9.9", None, {"employment_income": (None, None)}), + ], +) +def test_both_variables_routes_serve_unit_and_definition_period( + catalog_session: Session, + country_id: str, + version: str | None, + expected_catalog_format: int | None, + expected_fields: dict[str, tuple[str | None, str | None]], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + + from .test_metadata_routes import _client + + catalog_session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=_model_version( + catalog_session, "policyengine-uk" + ).id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ) + ) + catalog_session.commit() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + earlier_version = _model_version(catalog_session, "policyengine-us", "9.9.9") + earlier_version.catalog_format = None + for variable in catalog_session.exec( + select(Variable).where( + Variable.tax_benefit_model_version_id == earlier_version.id + ) + ).all(): + variable.unit = None + variable.definition_period = None + catalog_session.commit() + client = _client( + lambda: _service(catalog_session), response_cache=MetadataResponseCache() + ) + query = f"country_id={country_id}" + ( + "" if version is None else f"&policyengine_version={version}" + ) + + whole_version_bodies = [ + client.get(f"/v2/model-catalog/variables?{query}").json()["result"] + for _attempt in ("read", "stored") + ] + page = client.get(f"/v2/variables?{query}").json()["result"] + + for result in whole_version_bodies: + assert result["catalog_format"] == expected_catalog_format + served = { + variable["name"]: (variable["unit"], variable["definition_period"]) + for variable in result["variables"] + } + assert served == expected_fields + paged = { + item["name"]: (item["unit"], item["definition_period"]) + for item in page["items"] + } + assert paged == expected_fields + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 3885872ae2a9eb525b1581d6e78aff8c15f5d935 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:48:24 +0400 Subject: [PATCH 33/33] Say the same in the Stage 9 guide about who reads unit and definition period One paragraph said the household builder uses the two fields for formatting and per-period labels, while the guide's table says the builder and the breakdown read them. The paragraph now says the builder and breakdown read them, for currency formatting, per-period labels and household validation. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3785f5fbc..8e9ea0123 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -48,8 +48,9 @@ version in name order, in one response, with the catalog format 2 fields: `subtracts_parameter_path`, `enum_values` (each value's name and its label, in the type the country package declares), `unit` (for example `currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which -the household builder uses for formatting and per-period labels. Where a variable's `adds` or -`subtracts` list comes from a parameter, `part_parameter_values` maps that +the household builder and breakdown read (currency formatting, per-period +labels and household validation). Where a variable's `adds` or `subtracts` list +comes from a parameter, `part_parameter_values` maps that parameter's path to its current-law values in start-date order (`start_date`, `end_date`, `value`), so a client picks the list for the year it shows instead of the list resolved on 2025-01-01; a path whose parameter