diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md new file mode 100644 index 000000000..bbfc742cb --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -0,0 +1,5 @@ +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. +GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, enum labels, unit and definition period), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md new file mode 100644 index 000000000..553c9f234 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -0,0 +1 @@ +Every v2 metadata route reads its query through the shared query models, so unknown or repeated parameters, a missing or unsupported country_id and a malformed policyengine_version return 422 before any read (unsupported countries and malformed versions returned 400 before, and unknown parameters were ignored). /v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md new file mode 100644 index 000000000..4c325f3a5 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -0,0 +1 @@ +v2 metadata reads run in autocommit sessions on the shared pool, and each process keeps the catalog versions it has selected. A request that makes one read for a version the process has already selected, such as a page of parameters or one parameter's values, now sends two statements to the database (the pool's liveness check and the read) instead of five; each further read in a request adds one statement. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..411656a62 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 89 | +| request count | 92 | | db entity count | 6 | | sim flow count | 3 | @@ -18,7 +18,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | @@ -215,6 +215,9 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/regions/by-code/state/ca?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.item` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | +| `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | +| `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | ### `simulation_submit_poll` diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index bbe997b65..b32fe2af1 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -21,9 +21,18 @@ user_policies/ metadata/ response_models.py common.py + query_dependencies.py + response_cache.py *_routes.py ``` +`common.py` also holds the route class that gives metadata responses their +caching headers. `query_dependencies.py` defines the typed FastAPI +dependencies for the shared query models once, for every metadata route +module to import. `response_cache.py` keeps serialized whole-version +responses (bodies and ETags) for the process; it is HTTP-level, so it lives +with the routes. + `errors.py` defines the strict error response and serialization function shared by every API v2 resource. Application-wide exception handling distinguishes API v2 from non-v2 requests but must not select error behavior by matching @@ -52,6 +61,13 @@ services/v2// deletes.py ``` +A resource may also add a module for process-wide state its service reuses +across requests, named for what it keeps, such as the metadata package's +`selected_catalogs.py` (each catalog version a request has selected, which +never changes once published). Such a module holds plain data, constructs no +SQL and owns no session; the composition code creates one instance per process +and injects it into the service. + Only add CRUD connector modules for operations the resource supports. Append a specific descriptor when one operation file would become too broad, such as `reads_variables.py` or `reads_parameter_tree.py`. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..9232097e8 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 92, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -33,6 +33,7 @@ "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", @@ -1614,6 +1615,55 @@ "result.time_period", "result.datasets" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/variables?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0ad3eb455..8e9ea0123 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,127 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Loading a catalog version on demand + +`GET /v2/model-catalog?country_id=[&policyengine_version=]` +returns only the small parts of one catalog version: the model and model +version (with the current-law policy id and the years offered), the entities +(`key`, `plural`, `is_person`), the basic inputs, every region and logical +input dataset, and the economy choices. A catalog without regions, such as a +household-only country's, has empty region, dataset and economy-choice lists +rather than an error. A version published before catalog format 2 has a null +`catalog_format` and empty entities and basic inputs. The result names the +selected PolicyEngine.py version, so a client that starts without a version +learns the running one and requests every other part for that exact version. +The query uses the shared `CatalogQuery` model: unknown and repeated +parameters are rejected. + +`GET /v2/model-catalog/variables` (same query) returns every variable of the +version in name order, in one response, with the catalog format 2 fields: +`is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, +`subtracts_parameter_path`, `enum_values` (each value's name and its label, +in the type the country package declares), `unit` (for example +`currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which +the household builder and breakdown read (currency formatting, per-period +labels and household validation). Where a variable's `adds` or `subtracts` list +comes from a parameter, `part_parameter_values` maps that +parameter's path to its current-law values in start-date order +(`start_date`, `end_date`, `value`), so a client picks the list for the year +it shows instead of the list resolved on 2025-01-01; a path whose parameter +the version lacks maps to an empty list. `GET /v2/variables` pages carry the +same variable fields. + +`GET /v2/model-catalog/parameters` takes the same query plus one or more +`name` keys (at most 100, each an exact parameter path) and returns those +parameters in name order, each with its `economy` and `household` flags and its +current-law values in start-date order, in one response: a client showing or +editing a reform loads all of its parameters at once. `node_labels` maps each +ancestor of a returned parameter that the version stores as a parameter node +to its stored label (null when it has none), for breadcrumbs; a bracket index +such as `rates[0]` counts as a child of its schedule `rates`. `missing_names` +lists requested names the version has no parameter for, so a reform saved +against an older version still opens. `GET /v2/parameters` pages and details +carry the same `economy` and `household` flags. + +`GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page +only those exact parameters, and `GET /v2/parameter-values` repeated +`parameter_name` keys to page only those parameters' values. + +Every v2 metadata route reads its query through the shared query models in +`policyengine_api.query_parameters` (`CatalogQuery`, `CatalogPageQuery` and the +resource-specific models built on them): unknown parameters, repeated scalar +parameters, a missing or unsupported `country_id`, a malformed +`policyengine_version` and out-of-range `offset` or `limit` values are +rejected with HTTP 422 before any read. Before, the routes other than the +`/v2/model-catalog` ones ignored unknown parameters, and an unsupported country +or malformed version returned HTTP 400 from the read. A country given in the +path (`/v2/tax-benefit-models/by-country/{country_id}`) that is not supported +still returns HTTP 400. + +## Database round trips per request + +Metadata reads use sessions in autocommit mode on the shared v2 pool +(`get_v2_read_session_factory`), which only the metadata reader opens: each +read is one statement on rows a published catalog version never changes, so no +transaction is needed, and the database receives no `BEGIN` before the reads +and no `ROLLBACK` when the session closes. The sessions refuse to flush added +or changed rows (`ReadOnlySessionError`); a statement passed directly to +`execute` is not checked, so the metadata connectors keep to `SELECT`. Each process also keeps every catalog version a request has +selected (`SelectedCatalogStore`), since a published version never changes, so +later requests for the same country and version skip the selection query; +failed selections are not kept. Measured with statement logging in a local +database, a page of 500 parameters or one parameter's values took five +statements per request before (the pool's liveness check, `BEGIN`, the catalog +selection, the read and `ROLLBACK`) and two after, once the process has +selected that version (the liveness check and the read). A route that makes +several reads, such as `/v2/model-catalog` (regions and datasets), sends one +statement per read after the liveness check. The pool's liveness check +(`pool_pre_ping`) is unchanged. + +## Response caching + +Every successful metadata response carries a strong `ETag`, a hash of its +exact body, and `Cache-Control: public`. A request that names its catalog with +`policyengine_version` may be reused for a day (`max-age=86400`): a published +catalog version never changes, so such a response changes only with the +route's response format or with the day it is read on, and the `ETag` lets a +client revalidate after the day. A request without it selects the running +version, which changes on deploy, and may be reused for five minutes +(`max-age=300`). A response that depends on the request's UTC day, which is +`GET /v2/parameter-values?current=true` (the values in effect today), takes +the request's instant once: the read selects that day's values, and the +response may be reused only until the UTC midnight after that instant, +whichever lifetime would otherwise apply (`max-age=0` if its headers are +written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or +weak form, alone or in a list) gets HTTP 304 with no body. Error responses +carry neither header. Every cacheable response, and every 304, carries +`Vary: Origin, Accept-Encoding`, each value once: the API reflects every +origin in `Access-Control-Allow-Origin` and a request without `Origin` gets no +such header, so a shared cache must keep responses apart by `Origin`, and gzip +compresses bodies for clients that accept it. A 304 carries the same `Vary` as +its 200. The CORS and gzip middleware append to `Vary` without checking for +values already present, so the application's outermost middleware +(`UniqueVaryMiddleware`) lists each value once. `ETag` is +listed in `Access-Control-Expose-Headers`, so +browser scripts on other origins can read it (`Cache-Control` is readable +without that). + +Each API process also keeps the serialized bodies of `/v2/model-catalog` and +`/v2/model-catalog/variables` once built, each with its `ETag` computed once, +for up to eight route, country and version combinations (least recently used +dropped first; the 6.10.0 US variables body is 4,190,824 bytes uncompressed, +so eight entries hold about 12 MB for both countries' two routes at two +versions). A response is kept under the exact version it was read from, +so a request without `policyengine_version` (the process's running version, +which never changes while it runs) and one naming that version share it. Only +the first successful request per process reads the database. Requests that +arrive while that read is in progress wait for it and all receive its outcome, +the kept response or a copy of its error response, so one failing read is not +repeated by each of them; a failed read is not kept, so the first request after +it reads again. Measured locally against the full 6.10.0 US catalog, the +variables route takes about 160 ms when read and about 25 ms when kept, most of +it compression. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and @@ -263,13 +384,24 @@ A successful response has HTTP 200, `status: "ok"`, `message: null`, and a typed `result`. Collection results contain `policyengine_version`, `items`, `offset`, `limit`, and `has_more`; detail results contain `policyengine_version` and `item`. A malformed or noncanonical explicit -version returns a typed HTTP 400 error. A valid explicit version that has not +version returns a typed HTTP 422 error. A valid explicit version that has not been published for the country returns a typed HTTP 404 error. An absent catalog for the candidate's installed default version returns a typed HTTP 503 service error. None of these outcomes reads or repairs v1. Unsupported countries and methods return typed client errors. +The same test loads each country's catalog in parts as a client starting +cold would: `GET /v2/model-catalog` for the running version, then +`/v2/model-catalog/variables` for that exact version twice (the second request +may be served by another instance, so its time is not necessarily a kept +response's) and once more with `If-None-Match`, which must return 304, then a +page of 500 parameters, one parameter's values (the two routes measured at +0.8–2.6 s and 0.5–0.65 s in production on 2026-10-21) and that parameter by +name. It appends each request's time and body size to the GitHub job summary, +even when an assertion fails, so every staging and production release records +these route times against its own catalog. + Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for success and error responses. diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 1b986188e..8f74b1bb2 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -53,6 +53,7 @@ current_observability_id, generate_request_id, ) +from policyengine_api.vary_headers import UniqueVaryMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterable @@ -471,7 +472,9 @@ def create_asgi_app( app.mount("/", WSGIMiddleware(wsgi_app)) # The public API already permits every web origin. Use the standard ASGI # implementation while preserving the existing reflected-origin response. - return CORSMiddleware( + # UniqueVaryMiddleware wraps it, because CORS and gzip each append to Vary + # values a route may already have named. + cors_app = CORSMiddleware( app=app, allow_origin_regex=".*", allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], @@ -480,7 +483,10 @@ def create_asgi_app( REQUEST_ID_HEADER, OBSERVABILITY_ID_HEADER, "WWW-Authenticate", + # v2 metadata responses carry an ETag a client may revalidate with. + "ETag", ], allow_credentials=False, max_age=600, ) + return UniqueVaryMiddleware(cors_app) diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 07b88ad1a..f9df94461 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -8,6 +8,7 @@ from threading import Lock from typing import TYPE_CHECKING +from sqlalchemy import event from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -28,7 +29,7 @@ @dataclass class _V2EngineState: - """The process-owned v2 engine and Session factory. + """The process-owned v2 engine and Session factories. ``engine_pid`` is the ID of the process that created the engine, and ``engine_url_fingerprint`` identifies its target. ``_state_lock`` guards @@ -39,6 +40,7 @@ class _V2EngineState: engine_pid: int | None = None engine_url_fingerprint: tuple[object, ...] | None = None session_factory: sessionmaker[Session] | None = None + read_session_factory: sessionmaker[Session] | None = None _state_lock = Lock() @@ -82,6 +84,36 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +class ReadOnlySessionError(RuntimeError): + """Raised when code tries to write rows through a v2 read session.""" + + +def _refuse_flush(*_flush_arguments: object) -> None: + msg = "v2 read sessions do not write rows" + raise ReadOnlySessionError(msg) + + +def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: + """Build a factory of Sessions for v2 metadata reads, run without a transaction. + + The sessions share ``engine``'s pool in autocommit mode, so psycopg sends + each statement on its own: no BEGIN before the first read and no ROLLBACK + when the session closes. Only the metadata reader uses them: each of its + reads is one statement on a published catalog version, which never + changes, so no common snapshot is needed. The sessions refuse to flush + added or changed rows; statements passed directly to ``execute`` are not + checked, so they must stay reads. The pool restores the ordinary mode + when a connection returns. + """ + factory = sessionmaker( + bind=engine.execution_options(isolation_level="AUTOCOMMIT"), + class_=Session, + expire_on_commit=False, + ) + event.listen(factory, "before_flush", _refuse_flush) + return factory + + def _discard_inherited_state(current_pid: int) -> None: """Drop pool state inherited through a process fork without touching parent FDs.""" if _engine_state.engine is not None: @@ -90,6 +122,7 @@ def _discard_inherited_state(current_pid: int) -> None: _engine_state.engine_pid = current_pid _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None def get_v2_engine(settings: V2DatabaseSettings | None = None) -> Engine: @@ -130,6 +163,20 @@ def get_v2_session_factory( return _engine_state.session_factory +def get_v2_read_session_factory( + settings: V2DatabaseSettings | None = None, +) -> sessionmaker[Session]: + """Return the process-owned read-only Session factory without opening one. + + See ``build_v2_read_session_factory``. + """ + engine = get_v2_engine(settings) + with _state_lock: + if _engine_state.read_session_factory is None: + _engine_state.read_session_factory = build_v2_read_session_factory(engine) + return _engine_state.read_session_factory + + def close_v2_database() -> None: """Dispose this process's v2 pool and forget its lazy configuration.""" with _state_lock: @@ -139,6 +186,7 @@ def close_v2_database() -> None: _engine_state.engine_pid = None _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None atexit.register(close_v2_database) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..0be11f8fc 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -10,12 +10,18 @@ from fastapi import Depends from fastapi.security import APIKeyHeader, HTTPAuthorizationCredentials, HTTPBearer +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from uuid import UUID from policyengine_api.json_types import JSONObject @@ -34,10 +40,13 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -45,6 +54,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.policies.types import ( @@ -190,11 +200,12 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: - """Return parameters in name order, optionally filtered by ``search``. + """Return parameters in name order, optionally filtered. - ``search`` is a case-insensitive substring of the name, label or description. + The filter's ``search`` is a case-insensitive substring of the name, + label or description; its ``names`` keep only those exact names. """ ... @@ -231,9 +242,10 @@ def list_parameter_values( offset: int = 0, limit: int = 100, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - """Return canonical values, optionally for one parameter. + """Return canonical values, optionally for one or several parameters. - With the filter's ``current``, only values in effect on the UTC day of its + The filter's ``parameter_id`` or ``parameter_names`` select parameters. + With its ``current``, only values in effect on the UTC day of its ``now`` (default: the present) are returned. """ ... @@ -311,6 +323,46 @@ def get_economy_options( """ ... + def get_model_catalog( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + These are the model and model version, entities, basic inputs, regions, + logical input datasets and economy choices, which are empty for a + catalog without regions. Raise ``MetadataCatalogUnavailableError`` when + a catalog with regions lacks what ``get_economy_options`` needs. + """ + ... + + def get_catalog_variables( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also carries the current-law values of each parameter a variable's + adds or subtracts names. + """ + ... + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values; the result also carries the + stored labels of their ancestor nodes and the names the version lacks. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" @@ -727,9 +779,21 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_selected_catalogs() -> SelectedCatalogStore: + """Return the catalog versions this process's metadata reads have selected.""" + return SelectedCatalogStore() + + +@lru_cache(maxsize=1) +def _process_metadata_response_cache() -> MetadataResponseCache: + """Return the whole catalog-version responses this process keeps.""" + return MetadataResponseCache() + + def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( - get_v2_session_factory, + get_v2_read_session_factory, ) from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, @@ -737,8 +801,9 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.services.v2.metadata.services import V2MetadataService return V2MetadataService( - MetadataDatabaseSession(get_v2_session_factory()()), + MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), + selected_catalogs=_process_selected_catalogs(), ) @@ -932,6 +997,9 @@ class NativeRouteDependencies: None ) v2_account_service_factory: Callable[[], V2AccountResourceService] | None = None + # Whole catalog-version responses (/v2/model-catalog and its variables) are + # kept here once built; None reads them on every request. + v2_metadata_response_cache: MetadataResponseCache | None = None # User-owned routes attach this at router level. The default validates the # Auth0 bearer token, or else accepts the unauthenticated-user header, and # fails closed (401/403/503); tests replace it with a callable returning a @@ -947,6 +1015,7 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_metadata_response_cache=_process_metadata_response_cache(), v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py new file mode 100644 index 000000000..968c802c4 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -0,0 +1,124 @@ +"""API v2 routes that serve a catalog version in parts loaded on demand.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from starlette.responses import JSONResponse, Response + +from policyengine_api.fastapi_routes.v2.metadata.common import ( + METADATA_RESPONSES, + CacheableMetadataRoute, + read_resource, + read_whole_version_resource, +) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogParameterQueryDependency, + CatalogQueryDependency, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogParametersResponse, + MetadataCatalogVariablesResponse, + MetadataModelCatalogResponse, + MetadataResourceSuccessResponse, +) +from policyengine_api.services.v2.metadata.types import ( + MetadataCatalogVariablesResult, + MetadataModelCatalogResult, +) + +if TYPE_CHECKING: + from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies + + +def build_v2_model_catalog_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the routes that serve one catalog version in parts. + + Each request opens and closes its own metadata reader, so building the + router touches no database. + """ + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) + + @router.get( + "/model-catalog", + response_model=MetadataModelCatalogResponse, + summary="Get the small parts of one PolicyEngine.py catalog version", + description=( + "Returns the selected catalog version's model, model version, " + "entities, basic inputs, regions, logical input datasets and " + "economy choices, which are empty lists for a catalog without " + "regions. The result names the selected PolicyEngine.py version, " + "so a client can request the version's other parts by that exact " + "version; a response for a named version may be reused for a day." + ), + ) + def get_model_catalog( + query: CatalogQueryDependency, + ) -> MetadataResourceSuccessResponse[MetadataModelCatalogResult] | Response: + return read_whole_version_resource( + dependencies, + "model-catalog", + query, + MetadataModelCatalogResponse, + lambda reader: reader.get_model_catalog( + query.country_id, query.policyengine_version + ), + ) + + @router.get( + "/model-catalog/variables", + response_model=MetadataCatalogVariablesResponse, + summary="Get every variable of one PolicyEngine.py catalog version", + description=( + "Returns all of the selected catalog version's variables in name " + "order, in one response. Where a variable's adds or subtracts list " + "comes from a parameter, part_parameter_values carries that " + "parameter's current-law values by date, so a client can pick the " + "list for any year." + ), + ) + def get_catalog_variables( + query: CatalogQueryDependency, + ) -> MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] | Response: + return read_whole_version_resource( + dependencies, + "model-catalog/variables", + query, + MetadataCatalogVariablesResponse, + lambda reader: reader.get_catalog_variables( + query.country_id, query.policyengine_version + ), + ) + + @router.get( + "/model-catalog/parameters", + response_model=MetadataCatalogParametersResponse, + summary="Get several parameters of one catalog version by name", + description=( + "Returns the named parameters (repeat name for each, at most 100) " + "in name order, each with its economy and household flags and its " + "current-law values by date, plus the stored labels of their " + "ancestor nodes for breadcrumbs and the requested names the " + "version has no parameter for. A client showing or editing a " + "reform loads all of its parameters in this one request." + ), + ) + def get_catalog_parameters( + query: CatalogParameterQueryDependency, + ) -> MetadataCatalogParametersResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogParametersResponse, + lambda reader: reader.get_catalog_parameters( + query.country_id, query.policyengine_version, names=query.name + ), + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index fd4eaab5d..a2dab29ee 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,13 @@ from __future__ import annotations import contextlib -from typing import TYPE_CHECKING, Any, TypeVar +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any, Protocol, TypeVar +from fastapi.routing import APIRoute from pydantic import BaseModel +from starlette.responses import Response +from starlette.status import HTTP_200_OK from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, @@ -18,26 +22,45 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + body_entity_tag, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataResourceSuccessResponse, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, ) +from policyengine_api.vary_headers import unique_vary_values if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Coroutine + from fastapi import Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, V2MetadataResourceReader, ) + from policyengine_api.query_parameters import CatalogQuery -ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { +PATH_COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, - "description": "The resource request or PolicyEngine.py version is invalid.", + "description": ( + "The country in the path is not supported. Query parameters are " + "checked first: an invalid one, including the PolicyEngine.py " + "version, returns 422." + ), }, +} +"""The extra response of the one metadata route that takes the country in its path.""" + +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 404: { "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", @@ -48,7 +71,10 @@ }, 422: { "model": V2ErrorResponse, - "description": "The request parameters do not match the resource schema.", + "description": ( + "The query parameters do not match the resource schema, for example " + "an unknown parameter or a malformed PolicyEngine.py version." + ), }, 500: { "model": V2ErrorResponse, @@ -61,7 +87,180 @@ } +CACHE_HEADER_DESCRIPTIONS: dict[str, dict[str, Any]] = { + "ETag": { + "description": "Strong validator: a hash of the exact response body.", + "schema": {"type": "string"}, + }, + "Cache-Control": { + "description": ( + "public, with max-age 86400 when the request names " + "policyengine_version and 300 otherwise; current parameter values " + "expire at the next UTC midnight." + ), + "schema": {"type": "string"}, + }, + "Vary": { + "description": ( + "Origin, Accept-Encoding: caches keep responses apart by origin " + "and by compression; a 304 carries the same value." + ), + "schema": {"type": "string"}, + }, +} + +METADATA_RESPONSES: dict[int | str, dict[str, Any]] = { + **ERROR_RESPONSES, + 200: {"headers": CACHE_HEADER_DESCRIPTIONS}, + 304: { + "description": ( + "Not modified: If-None-Match names the current ETag. No body is sent." + ), + "headers": CACHE_HEADER_DESCRIPTIONS, + }, +} +"""Every metadata route's documented responses, with its caching headers.""" + + +class CatalogVersionResult(Protocol): + """A whole-version result, which names the catalog version it was read from.""" + + @property + def policyengine_version(self) -> str: + """The exact PolicyEngine.py version of the catalog read.""" + ... + + ResponseT = TypeVar("ResponseT", bound=BaseModel) +ReadT = TypeVar("ReadT") +CatalogResultT = TypeVar("CatalogResultT", bound=CatalogVersionResult) + +EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 +"""How long a response for a named catalog version may be reused (a day). + +A published catalog version never changes, so such a response stays valid +until its code's response format changes; the ETag lets a client revalidate +it cheaply after that. +""" +RUNNING_VERSION_MAX_AGE_SECONDS = 300 +"""How long a response for the running catalog version may be reused. + +Without ``policyengine_version`` the running PolicyEngine.py version answers, +and it changes whenever the API deploys a new one. +""" + + +UTC_DAY_READ = "utc_day_read" +"""The ``request.state`` attribute holding the instant a day-dependent read used. + +A route whose result depends on the UTC day, such as current parameter values, +sets it with ``take_utc_instant_for_read`` and reads with that same instant; +its response may then be reused only until the next UTC midnight after it. +""" + + +def utc_now() -> datetime: + """Return the current time in UTC; tests replace it to fix the clock.""" + return datetime.now(UTC) + + +def take_utc_instant_for_read(request: Request) -> datetime: + """Take the instant a day-dependent read uses, and record it on the request. + + The read and its response's cache lifetime then use the same UTC day. + """ + instant = utc_now() + setattr(request.state, UTC_DAY_READ, instant) + return instant + + +def _seconds_left_in_utc_day(read_instant: datetime, now: datetime) -> int: + """Return the seconds from ``now`` to the UTC midnight after ``read_instant``. + + Zero when that midnight has passed, so values read for one day are never + reused into the next. + """ + next_midnight = (read_instant.astimezone(UTC) + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + return max(int((next_midnight - now).total_seconds()), 0) + + +class CacheableMetadataRoute(APIRoute): + """A metadata route whose successful responses carry caching headers. + + See ``with_cache_headers``. Failures are served unchanged, without them. + """ + + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + """Wrap FastAPI's handler so a 200 response gains its caching headers.""" + handle = super().get_route_handler() + + async def handle_with_cache_headers(request: Request) -> Response: + response = await handle(request) + if response.status_code != HTTP_200_OK: + return response + return with_cache_headers(request, response) + + return handle_with_cache_headers + + +def with_cache_headers(request: Request, response: Response) -> Response: + """Give a JSON response a strong ETag and a public cache lifetime. + + The ETag is a hash of the exact body, so it changes with any change to the + content, including the catalog format. A request naming its catalog with + ``policyengine_version`` may be reused for a day, one that takes the + running version for five minutes, and one whose route read for a UTC day + (``UTC_DAY_READ``) no later than the midnight after that day. Every such + response, and the 304 a request whose ``If-None-Match`` names the ETag + gets instead, varies by ``Origin`` and ``Accept-Encoding``. + """ + entity_tag = response.headers.get("etag") or body_entity_tag(response.body) + max_age = ( + EXPLICIT_VERSION_MAX_AGE_SECONDS + if "policyengine_version" in request.query_params + else RUNNING_VERSION_MAX_AGE_SECONDS + ) + read_instant = getattr(request.state, UTC_DAY_READ, None) + if read_instant is not None: + max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) + headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} + if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): + not_modified = Response(status_code=304, headers=headers) + not_modified.headers["Vary"] = _cacheable_vary(None) + return not_modified + response.headers.update(headers) + response.headers["Vary"] = _cacheable_vary(response.headers.get("vary")) + return response + + +CACHEABLE_VARY_VALUES = ("Origin", "Accept-Encoding") +"""What every cacheable metadata response, and its 304, varies by. + +The API reflects every origin in Access-Control-Allow-Origin, and a request +without Origin gets no such header, so a shared cache must keep responses +apart by Origin; gzip compresses the body for clients that accept it. A 304 +carries the Vary its 200 would. +""" + + +def _cacheable_vary(existing: str | None) -> str: + """Return ``existing`` with Origin and Accept-Encoding added if absent. + + The CORS and gzip middleware may append them again; the application's + outermost middleware (UniqueVaryMiddleware) then lists each value once. + """ + return unique_vary_values(", ".join([existing or "", *CACHEABLE_VARY_VALUES])) + + +def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: + if if_none_match is None: + return False + candidates = {candidate.strip() for candidate in if_none_match.split(",")} + return "*" in candidates or bool( + {entity_tag, f"W/{entity_tag}"}.intersection(candidates) + ) def read_resource( @@ -75,6 +274,60 @@ def read_resource( (400, 404 or 503, and 500 for anything unexpected), and the reader is closed whatever happens. """ + return run_metadata_read( + dependencies, lambda reader: response_type(result=operation(reader)) + ) + + +def read_whole_version_resource( + dependencies: NativeRouteDependencies, + route: str, + query: CatalogQuery, + response_type: type[MetadataResourceSuccessResponse[CatalogResultT]], + operation: Callable[[V2MetadataResourceReader], CatalogResultT], +) -> MetadataResourceSuccessResponse[CatalogResultT] | Response: + """Serve a whole catalog-version response, built once per process when kept. + + With the dependencies' response cache, the serialized body and its ETag + for ``route``, the query's country and the version it selects are read + and built only on the first successful request; failures are served as + ``read_resource`` serves them and never kept. Without a cache this is + ``read_resource``. + """ + cache = dependencies.v2_metadata_response_cache + if cache is None: + return read_resource(dependencies, response_type, operation) + + def build() -> BuiltResponse | Response: + result = read_resource(dependencies, response_type, operation) + if isinstance(result, MetadataResourceSuccessResponse): + return BuiltResponse( + body=result.model_dump_json().encode(), + policyengine_version=result.result.policyengine_version, + ) + return result + + kept = cache.get_or_build( + route, query.country_id, query.policyengine_version, build + ) + if isinstance(kept, KeptResponse): + return Response( + content=kept.body, + media_type="application/json", + headers={"ETag": kept.entity_tag}, + ) + return kept + + +def run_metadata_read( + dependencies: NativeRouteDependencies, + operation: Callable[[V2MetadataResourceReader], ReadT], +) -> ReadT | JSONResponse: + """Run one metadata read and return its result unchanged. + + Failures become the typed v2 error responses described by read_resource, + and the reader is closed whatever happens. + """ reader = None try: factory = dependencies.v2_metadata_reader_factory @@ -85,7 +338,7 @@ def read_resource( factory = _default_v2_metadata_reader_factory reader = factory() - return response_type(result=operation(reader)) + return operation(reader) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 17ff62e27..448826c5d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -2,16 +2,22 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + RegionCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, @@ -19,14 +25,10 @@ MetadataRegionDetailResponse, MetadataRegionPageResponse, ) -from policyengine_api.services.v2.metadata.types import MetadataRegionType if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_geography_router( dependencies: NativeRouteDependencies, @@ -36,7 +38,11 @@ def build_v2_metadata_geography_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/datasets", @@ -44,19 +50,16 @@ def build_v2_metadata_geography_router( summary="List logical inputs from a selected PolicyEngine.py catalog", ) def list_datasets( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataDatasetPageResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetPageResponse, lambda reader: reader.list_datasets( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -67,16 +70,15 @@ def list_datasets( ) def get_dataset( dataset_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataDatasetDetailResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetDetailResponse, lambda reader: reader.get_dataset( - country_id, + query.country_id, dataset_id, - policyengine_version, + query.policyengine_version, ), ) @@ -86,21 +88,19 @@ def get_dataset( summary="List regions from a selected PolicyEngine.py catalog", ) def list_regions( - country_id: str, - policyengine_version: str | None = None, - region_type: MetadataRegionType | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: RegionCollectionQueryDependency, ) -> MetadataRegionPageResponse | JSONResponse: return read_resource( dependencies, MetadataRegionPageResponse, lambda reader: reader.list_regions( - country_id, - policyengine_version, - region_type=region_type.value if region_type is not None else None, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + region_type=( + None if query.region_type is None else query.region_type.value + ), + offset=query.offset, + limit=query.limit, ), ) @@ -111,16 +111,15 @@ def list_regions( ) def get_region_by_code( region_code: str, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region_by_code( - country_id, + query.country_id, region_code, - policyengine_version, + query.policyengine_version, ), ) @@ -131,16 +130,15 @@ def get_region_by_code( ) def get_region( region_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region( - country_id, + query.country_id, region_id, - policyengine_version, + query.policyengine_version, ), ) @@ -150,15 +148,14 @@ def get_region( summary="Get compact economy-selection options from a selected catalog", ) def get_economy_options( - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataEconomyOptionsResponse | JSONResponse: return read_resource( dependencies, MetadataEconomyOptionsResponse, lambda reader: reader.get_economy_options( - country_id, - policyengine_version, + query.country_id, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 927b52536..ae3127725 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -2,16 +2,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + PATH_COUNTRY_ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + CatalogVersionQueryDependency, + VariableCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, @@ -25,9 +33,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_model_router( dependencies: NativeRouteDependencies, @@ -37,7 +42,11 @@ def build_v2_metadata_model_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/tax-benefit-models", @@ -45,37 +54,35 @@ def build_v2_metadata_model_router( summary="List models for one PolicyEngine.py catalog", ) def list_models( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelPageResponse, lambda reader: reader.list_models( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @router.get( "/tax-benefit-models/by-country/{country_id}", response_model=MetadataModelSelectionResponse, + responses=PATH_COUNTRY_ERROR_RESPONSES, summary="Get a country model and selected PolicyEngine.py version", ) def get_model_by_country( country_id: str, - policyengine_version: str | None = None, + query: CatalogVersionQueryDependency, ) -> MetadataModelSelectionResponse | JSONResponse: return read_resource( dependencies, MetadataModelSelectionResponse, lambda reader: reader.get_model_by_country( country_id, - policyengine_version, + query.policyengine_version, ), ) @@ -86,16 +93,15 @@ def get_model_by_country( ) def get_model( model_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelDetailResponse, lambda reader: reader.get_model( - country_id, + query.country_id, model_id, - policyengine_version, + query.policyengine_version, ), ) @@ -105,19 +111,16 @@ def get_model( summary="List selected PolicyEngine.py model versions", ) def list_model_versions( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelVersionPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionPageResponse, lambda reader: reader.list_model_versions( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -128,16 +131,15 @@ def list_model_versions( ) def get_model_version( version_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelVersionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionDetailResponse, lambda reader: reader.get_model_version( - country_id, + query.country_id, version_id, - policyengine_version, + query.policyengine_version, ), ) @@ -147,21 +149,17 @@ def get_model_version( summary="List variables from a selected PolicyEngine.py catalog", ) def list_variables( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: VariableCollectionQueryDependency, ) -> MetadataVariablePageResponse | JSONResponse: return read_resource( dependencies, MetadataVariablePageResponse, lambda reader: reader.list_variables( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + search=query.search, ), ) @@ -172,16 +170,15 @@ def list_variables( ) def get_variable( variable_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataVariableDetailResponse | JSONResponse: return read_resource( dependencies, MetadataVariableDetailResponse, lambda reader: reader.get_variable( - country_id, + query.country_id, variable_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 0f4714e5c..51383ccc0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,16 +2,23 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + CacheableMetadataRoute, read_resource, + take_utc_instant_for_read, +) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogQueryDependency, + ParameterChildrenQueryDependency, + ParameterCollectionQueryDependency, + ParameterValueCollectionQueryDependency, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, @@ -20,26 +27,14 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueFilter, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - - -@dataclass(frozen=True) -class ParameterValueListingQuery: - """The query parameters of the canonical parameter-value listing route.""" - - country_id: str - policyengine_version: str | None = None - parameter_id: UUID | None = None - current: bool = False - offset: Offset = 0 - limit: Limit = 100 - def build_v2_metadata_parameter_router( dependencies: NativeRouteDependencies, @@ -49,7 +44,11 @@ def build_v2_metadata_parameter_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/parameters", @@ -57,21 +56,20 @@ def build_v2_metadata_parameter_router( summary="List parameters from a selected PolicyEngine.py catalog", ) def list_parameters( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: ParameterCollectionQueryDependency, ) -> MetadataParameterPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterPageResponse, lambda reader: reader.list_parameters( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + parameter_filter=ParameterFilter( + search=query.search, + names=None if query.name is None else tuple(query.name), + ), ), ) @@ -81,21 +79,17 @@ def list_parameters( summary="List direct children of one parameter path", ) def list_parameter_children( - country_id: str, - parent_path: str = "", - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: ParameterChildrenQueryDependency, ) -> MetadataParameterChildPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterChildPageResponse, lambda reader: reader.list_parameter_children( - country_id, - policyengine_version, - parent_path=parent_path, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + parent_path=query.parent_path, + offset=query.offset, + limit=query.limit, ), ) @@ -106,16 +100,15 @@ def list_parameter_children( ) def get_parameter( parameter_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterDetailResponse, lambda reader: reader.get_parameter( - country_id, + query.country_id, parameter_id, - policyengine_version, + query.policyengine_version, ), ) @@ -125,8 +118,12 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( - query: Annotated[ParameterValueListingQuery, Depends()], + request: Request, + query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: + # Current values are those in effect on the request's UTC day; the + # read and the response's lifetime use the same instant. + now = take_utc_instant_for_read(request) if query.current else None return read_resource( dependencies, MetadataParameterValuePageResponse, @@ -136,6 +133,12 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + now=now, + parameter_names=( + None + if query.parameter_name is None + else tuple(query.parameter_name) + ), ), offset=query.offset, limit=query.limit, @@ -149,16 +152,15 @@ def list_parameter_values( ) def get_parameter_value( value_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterValueDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterValueDetailResponse, lambda reader: reader.get_parameter_value( - country_id, + query.country_id, value_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py new file mode 100644 index 000000000..b52d18d1e --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py @@ -0,0 +1,53 @@ +"""Typed FastAPI dependencies for the shared v2 metadata query contracts. + +Route modules import these aliases into their own globals, because FastAPI +resolves an endpoint's string annotations against its module only. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CatalogPageQuery, + CatalogParameterQuery, + CatalogQuery, + CatalogVersionQuery, + ParameterChildrenQuery, + ParameterCollectionQuery, + ParameterValueCollectionQuery, + RegionCollectionQuery, + VariableCollectionQuery, +) + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] +CatalogVersionQueryDependency = Annotated[ + CatalogVersionQuery, Depends(query_dependency(CatalogVersionQuery)) +] +CatalogPageQueryDependency = Annotated[ + CatalogPageQuery, Depends(query_dependency(CatalogPageQuery)) +] +VariableCollectionQueryDependency = Annotated[ + VariableCollectionQuery, Depends(query_dependency(VariableCollectionQuery)) +] +RegionCollectionQueryDependency = Annotated[ + RegionCollectionQuery, Depends(query_dependency(RegionCollectionQuery)) +] +ParameterChildrenQueryDependency = Annotated[ + ParameterChildrenQuery, Depends(query_dependency(ParameterChildrenQuery)) +] +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py new file mode 100644 index 000000000..1a1ad0b2d --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -0,0 +1,192 @@ +"""A bounded, process-wide store of whole catalog-version responses.""" + +from __future__ import annotations + +import hashlib +import threading +from collections import OrderedDict +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from starlette.responses import Response + +if TYPE_CHECKING: + from collections.abc import Callable + +DEFAULT_RESPONSE_CAPACITY = 8 + +KeptResponseKey = tuple[str, str, str] +"""A route name, a country ID and the exact PolicyEngine.py version served.""" + + +def body_entity_tag(body: bytes) -> str: + """Return the strong ETag of a response body: a hash of its exact bytes.""" + return f'"{hashlib.sha256(body).hexdigest()[:32]}"' + + +@dataclass(frozen=True) +class BuiltResponse: + """A serialized success body and the catalog version it was read from.""" + + body: bytes + policyengine_version: str + + +@dataclass(frozen=True) +class KeptResponse: + """A kept success body with its ETag, computed once when it was kept.""" + + body: bytes + entity_tag: str + + +@dataclass(frozen=True) +class FailedRead: + """A failed whole-version read, given to the requests that waited on it.""" + + status_code: int + body: bytes + + def response(self) -> Response: + """Return a new error response for one waiting request. + + Each request gets its own response object, because middleware adds + request-specific headers to the response it sends. + """ + return Response( + content=self.body, + status_code=self.status_code, + media_type="application/json", + ) + + +class _InFlightRead: + """One read in progress for a key, and how many requests wait on it.""" + + def __init__(self) -> None: + self.done = threading.Event() + self.outcome: KeptResponse | FailedRead | None = None + self.waiters = 0 + + +class MetadataResponseCache: + """Keep whole-version responses, built once each per process. + + A published catalog version never changes, so a body built for a route, + country and version stays valid for the life of the process. Responses + are kept under the exact version they were read from: a request without a + version (the running version, fixed for the process) and one naming that + version share one response. The most recently used ``capacity`` responses + are kept: the US variables body of the 6.10.0 catalog is 4,190,824 bytes + uncompressed, so the default of eight holds about 12 MB for both + countries' two whole-version routes at two versions, and at most about + 34 MB if every entry were that large. Requests that arrive while a read + for the same key is in progress wait for it, and all receive its outcome: + the kept response, or a copy of its error response. A failed read is not + kept, so the first request after it reads again. + """ + + def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: + """Keep at most ``capacity`` responses (at least one).""" + if capacity < 1: + msg = "capacity must be positive" + raise ValueError(msg) + self._capacity = capacity + self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() + self._running_versions: dict[tuple[str, str], str] = {} + self._lock = threading.Lock() + self._builds: dict[tuple[str, str, str | None], _InFlightRead] = {} + + def _stored( + self, route: str, country_id: str, requested_version: str | None + ) -> KeptResponse | None: + with self._lock: + version = ( + requested_version + if requested_version is not None + else self._running_versions.get((route, country_id)) + ) + if version is None: + return None + key = (route, country_id, version) + kept = self._responses.get(key) + if kept is not None: + self._responses.move_to_end(key) + return kept + + def get_or_build( + self, + route: str, + country_id: str, + requested_version: str | None, + build: Callable[[], BuiltResponse | Response], + ) -> KeptResponse | Response: + """Return the kept response, calling ``build`` only when none is kept. + + ``requested_version`` None means the running version. Requests that + arrive while a read for the same key is in progress wait for it and + receive its outcome: the kept response, or a copy of its error + response. A failed read is not kept, so the next request after it + reads again. + """ + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + key = (route, country_id, requested_version) + with self._lock: + in_flight = self._builds.get(key) + if in_flight is None: + in_flight = _InFlightRead() + self._builds[key] = in_flight + reads_itself = True + else: + in_flight.waiters += 1 + reads_itself = False + if not reads_itself: + in_flight.done.wait() + outcome = in_flight.outcome + if outcome is None: + # The read raised instead of returning a response: try again. + return self.get_or_build(route, country_id, requested_version, build) + return outcome.response() if isinstance(outcome, FailedRead) else outcome + try: + kept = self._stored(route, country_id, requested_version) + if kept is None: + built = build() + if not isinstance(built, BuiltResponse): + in_flight.outcome = FailedRead( + status_code=built.status_code, body=bytes(built.body) + ) + return built + kept = self._keep(route, country_id, requested_version, built) + in_flight.outcome = kept + return kept + finally: + with self._lock: + # Only this read's own entry is removed; a later read for the + # same key keeps its entry. + if self._builds.get(key) is in_flight: + del self._builds[key] + in_flight.done.set() + + def _keep( + self, + route: str, + country_id: str, + requested_version: str | None, + built: BuiltResponse, + ) -> KeptResponse: + key = (route, country_id, built.policyengine_version) + with self._lock: + if requested_version is None: + self._running_versions[(route, country_id)] = built.policyengine_version + kept = self._responses.get(key) + if kept is None: + kept = KeptResponse( + body=built.body, entity_tag=body_entity_tag(built.body) + ) + self._responses[key] = kept + self._responses.move_to_end(key) + while len(self._responses) > self._capacity: + self._responses.popitem(last=False) + return kept diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index d53660834..697ab6f76 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,10 +6,13 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -139,3 +142,21 @@ class MetadataEconomyOptionsResponse( MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] ): """Success response holding the options for configuring an economy comparison.""" + + +class MetadataModelCatalogResponse( + MetadataResourceSuccessResponse[MetadataModelCatalogResult] +): + """Success response holding the small parts of one catalog version.""" + + +class MetadataCatalogVariablesResponse( + MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] +): + """Success response holding every variable of one catalog version.""" + + +class MetadataCatalogParametersResponse( + MetadataResourceSuccessResponse[MetadataCatalogParametersResult] +): + """Success response holding parameters fetched by name, with their values.""" diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..b55b57436 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -12,6 +12,9 @@ build_v2_household_router, ) from policyengine_api.fastapi_routes.v2.me.routes import build_v2_me_router +from policyengine_api.fastapi_routes.v2.metadata.catalog_routes import ( + build_v2_model_catalog_router, +) from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) @@ -56,6 +59,7 @@ def build_v2_router( router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) + router.include_router(build_v2_model_catalog_router(dependencies)) @router.get( "/v2/openapi.json", diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..25fd541f6 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -39,6 +39,7 @@ class RouteGroupConfig: "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index e06e9cdcb..0d94c8484 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -30,6 +30,7 @@ validate_policyengine_version, ) from policyengine_api.services.economic_impact_request import LocalAreaBreakdown +from policyengine_api.services.v2.metadata.types import MetadataRegionType from policyengine_api.spm import SPMSelection from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, @@ -232,6 +233,88 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CatalogVersionQuery(StrictQueryParameters): + """An optional exact catalog version, for routes with the country in the path.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class CatalogPageQuery(CatalogQuery, PaginationQuery): + """Query contract for one page of a catalog's resources.""" + + +class VariableCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's variables.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + + +class RegionCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's regions.""" + + region_type: MetadataRegionType | None = Field( + default=None, description="Keep only regions of this type" + ) + + +class ParameterChildrenQuery(CatalogPageQuery): + """Query contract for a page of one parameter path's direct children.""" + + parent_path: str = Field( + default="", + description="Dotted path whose direct children are listed; empty for the root", + ) + + +MAXIMUM_PARAMETER_NAMES = 100 +ParameterName = Annotated[ + str, + Field(min_length=1, max_length=512, description="Exact parameter path"), +] + + +class CatalogParameterQuery(CatalogQuery): + """Query contract for fetching several parameters of a catalog by name.""" + + name: list[ParameterName] = Field( + min_length=1, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Exact parameter paths; repeat the key for each one", + ) + + +class ParameterCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's parameters.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths; repeat the key", + ) + + +class ParameterValueCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's current-law parameter values.""" + + parameter_id: ResourceId | None = None + parameter_name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths' values; repeat the key", + ) + current: bool = Field( + default=False, + description="Keep only the values in effect today (UTC)", + ) + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py index cd5dbc276..b6d79b6fa 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -48,3 +48,17 @@ def read_dataset( ), ) return rows[0] if rows else None + + +def read_input_datasets(session: Session, *, model_version_id: UUID) -> list[Dataset]: + """Return every logical input dataset of a model version in name order.""" + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py new file mode 100644 index 000000000..24968399a --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -0,0 +1,36 @@ +"""Database selections for parameter-node labels.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import ParameterNodeLabelRow + +if TYPE_CHECKING: + from collections.abc import Collection + from uuid import UUID + + +def read_parameter_node_labels( + session: Session, *, model_version_id: UUID, node_names: Collection[str] +) -> list[ParameterNodeLabelRow]: + """Return the name and stored label of each named node of a model version. + + Names the version has no node for are absent. Rows are in name order. + """ + if not node_names: + return [] + rows = read_rows( + session, + select(col(ParameterNode.name), col(ParameterNode.label)) + .where( + col(ParameterNode.tax_benefit_model_version_id) == model_version_id, + col(ParameterNode.name).in_(sorted(node_names)), + ) + .order_by(col(ParameterNode.name)), + ) + return [ParameterNodeLabelRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index cd945768e..49f232a8e 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -114,11 +114,7 @@ def _parameter_children_statement( ), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), - parameter_id.label("parameter_id"), - col(Parameter.label).label("parameter_label"), - col(Parameter.description).label("parameter_description"), - col(Parameter.data_type).label("parameter_data_type"), - col(Parameter.unit).label("parameter_unit"), + Parameter, ) .select_from( paths.outerjoin( @@ -152,7 +148,7 @@ def read_parameter_children( """Return a page of the direct children of parent_path in a model version's tree. Each row has the child's path, label, type (node or parameter), child count - and, for parameters, the parameter's summary columns. The session's dialect + and the stored Parameter (None for a node). The session's dialect selects SQLite or PostgreSQL string functions; limit + 1 rows are fetched so the caller can tell whether another page exists. """ diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 02062798a..eadd87be9 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -9,11 +9,16 @@ from policyengine_api.data.v2.models import Parameter, ParameterValue from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import CurrentLawValueRow if TYPE_CHECKING: + from collections.abc import Collection from uuid import UUID - from policyengine_api.services.v2.metadata.types import ParameterValueSelection + from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueSelection, + ) def _escape_like(value: str) -> str: @@ -26,17 +31,23 @@ def read_parameters( model_version_id: UUID, offset: int, limit: int, - search: str | None, + parameter_filter: ParameterFilter, ) -> list[Parameter]: """Return a page of a model version's parameters in name order. - search matches name, label or description as a literal, case-insensitive - substring. Fetch limit + 1 rows so the caller can tell whether another page - exists. + The filter's search matches name, label or description as a literal, + case-insensitive substring, and its names keep only parameters with those + exact names. Fetch limit + 1 rows so the caller can tell whether another + page exists. """ statement = select(Parameter).where( col(Parameter.tax_benefit_model_version_id) == model_version_id ) + if parameter_filter.names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(parameter_filter.names)) + ) + search = parameter_filter.search if search: pattern = f"%{_escape_like(search)}%" statement = statement.where( @@ -52,6 +63,26 @@ def read_parameters( ) +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Parameter]: + """Return the named parameters of a model version in name order. + + Names the version has no parameter for are absent. + """ + if not parameter_names: + return [] + return read_rows( + session, + select(Parameter) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + ) + .order_by(col(Parameter.name)), + ) + + def read_parameter( session: Session, *, model_version_id: UUID, parameter_id: UUID ) -> Parameter | None: @@ -77,8 +108,9 @@ def read_parameter_values( """Return a page of a model version's current-law parameter values. Values tied to a policy or dynamic are excluded. The selection's - parameter_id keeps one parameter's values and its selected_day keeps those - in effect on that day. Rows are ordered by parameter name, then newest + parameter_id keeps one parameter's values, its parameter_names the values + of the parameters with those names, and its selected_day those in effect + on that day. Rows are ordered by parameter name, then newest first, and limit + 1 are fetched so the caller can tell whether another page exists. """ @@ -95,6 +127,10 @@ def read_parameter_values( ) if parameter_id is not None: statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selection.parameter_names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(selection.parameter_names)) + ) if selected_day is not None: statement = statement.where( col(ParameterValue.start_date) <= selected_day, @@ -131,3 +167,33 @@ def read_parameter_value( ), ) return rows[0] if rows else None + + +def read_current_law_values_by_parameter_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[CurrentLawValueRow]: + """Return the current-law values of the named parameters of a model version. + + Values tied to a policy or dynamic are excluded. Rows are ordered by + parameter name, then start date. + """ + if not parameter_names: + return [] + rows = read_rows( + session, + select( + col(Parameter.name), + col(ParameterValue.value_json), + col(ParameterValue.start_date), + col(ParameterValue.end_date), + ) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + .order_by(col(Parameter.name), col(ParameterValue.start_date)), + ) + return [CurrentLawValueRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py index 4d84be396..1378f6528 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -62,3 +62,13 @@ def read_variable( ), ) return rows[0] if rows else None + + +def read_all_variables(session: Session, *, model_version_id: UUID) -> list[Variable]: + """Return every variable of a model version in name order.""" + return read_rows( + session, + select(Variable) + .where(col(Variable.tax_benefit_model_version_id) == model_version_id) + .order_by(col(Variable.name)), + ) diff --git a/policyengine_api/services/v2/metadata/selected_catalogs.py b/policyengine_api/services/v2/metadata/selected_catalogs.py new file mode 100644 index 000000000..832705432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/selected_catalogs.py @@ -0,0 +1,41 @@ +"""A process-wide store of the catalog versions metadata reads have selected.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + +CatalogSelectionKey = tuple[str, str | None] +"""A country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + + +class SelectedCatalogStore: + """Keep each catalog version a request has selected, for the process's life. + + A published catalog version never changes, so once a request has found it, + later requests for the same country and version need not query for it + again. Only successful selections are kept: an unsupported country, an + invalid version or an absent catalog is looked up, and fails, each time. + The kept records are detached, fully loaded rows that are only read. + """ + + def __init__(self) -> None: + """Start with no selections kept.""" + self._selections: dict[CatalogSelectionKey, SelectedCatalog] = {} + self._lock = threading.Lock() + + def get(self, key: CatalogSelectionKey) -> SelectedCatalog | None: + """Return the selection kept for ``key``, or None.""" + with self._lock: + return self._selections.get(key) + + def keep(self, key: CatalogSelectionKey, selected: SelectedCatalog) -> None: + """Keep ``selected`` as the selection for ``key``.""" + with self._lock: + self._selections[key] = selected diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 938eb660e..4496e3b30 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -14,15 +14,21 @@ from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( read_dataset, read_datasets, + read_input_datasets, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 # a dotted module path cannot be wrapped + read_parameter_node_labels, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_current_law_values_by_parameter_name, read_parameter, read_parameter_value, read_parameter_values, read_parameters, + read_parameters_by_name, ) from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( read_input_dataset, @@ -30,13 +36,19 @@ read_regions, ) from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_all_variables, read_variable, read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + ancestor_paths, + dated_values_by_parameter_name, + metadata_catalog_parameters, + metadata_catalog_variables, metadata_dataset, metadata_economy_options, metadata_model, + metadata_model_catalog, metadata_model_selection, metadata_model_version, metadata_parameter, @@ -44,15 +56,20 @@ metadata_parameter_value, metadata_region, metadata_variable, + national_region, page_result, parameter_value_selection, + part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -60,6 +77,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -69,11 +87,16 @@ ) if TYPE_CHECKING: + from collections.abc import Callable, Sequence from uuid import UUID + from policyengine_api.data.v2.models import Dataset, Region from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) class V2MetadataService: @@ -90,12 +113,18 @@ def __init__( database_session: MetadataDatabaseSession, *, running_policyengine_version: str, + selected_catalogs: SelectedCatalogStore | None = None, ) -> None: - """Bind to a request's session and the running PolicyEngine.py version.""" + """Bind to a request's session and the running PolicyEngine.py version. + + With ``selected_catalogs``, a catalog version found once is taken from + that store instead of being queried again. + """ self._database_session = database_session self._running_policyengine_version = validate_policyengine_version( running_policyengine_version ) + self._selected_catalogs = selected_catalogs def close(self) -> None: """Close the request's database session.""" @@ -104,12 +133,20 @@ def close(self) -> None: def _select_catalog( self, country_id: str, policyengine_version: str | None ) -> SelectedCatalog: - return read_metadata_catalog( + key = (country_id, policyengine_version) + if self._selected_catalogs is not None: + kept = self._selected_catalogs.get(key) + if kept is not None: + return kept + selected = read_metadata_catalog( self._database_session.session, country_id=country_id, running_policyengine_version=self._running_policyengine_version, policyengine_version=policyengine_version, ) + if self._selected_catalogs is not None: + self._selected_catalogs.keep(key, selected) + return selected def _select_paginated_catalog( self, @@ -255,12 +292,13 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: """Return a page of the catalog's parameters in name order. - search keeps parameters whose name, label or description contains it, - ignoring case. + The filter's search keeps parameters whose name, label or description + contains it, ignoring case, and its names keep only parameters with + those exact names. No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -270,7 +308,9 @@ def list_parameters( model_version_id=selected.model_version.id, offset=offset, limit=limit, - search=search, + parameter_filter=( + parameter_filter if parameter_filter is not None else ParameterFilter() + ), ) return page_result( selected, @@ -343,8 +383,9 @@ def list_parameter_values( """Return a page of current-law parameter values, newest first per parameter. Values belonging to a reform or dynamic are never listed. The filter's - parameter_id keeps one parameter's values, and current=True keeps those in - effect on the UTC day of now (today by default). No filter lists them all. + parameter_id keeps one parameter's values, parameter_names the values of + the parameters with those exact names, and current=True those in effect + on the UTC day of now (today by default). No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -516,28 +557,147 @@ def get_economy_options( region, its default input dataset, the current-law id or the years. """ selected = self._select_catalog(country_id, policyengine_version) - regions = read_regions( - self._database_session.session, - model_version_id=selected.model_version.id, + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + dataset = _validated_national_dataset( + selected, + country_id=country_id, + regions=regions, + find_dataset=lambda dataset_id: read_input_dataset( + session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), ) - national_region = next( - (region for region in regions if region.code == country_id), None + return metadata_economy_options( + selected, regions=regions, national_dataset=dataset + ) + + def get_model_catalog( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + They are the model and model version, entities, basic inputs, every + region and logical input dataset, and the economy choices. A catalog + without regions offers no economy calculation, so its choices are + empty; one with regions raises MetadataCatalogUnavailableError when it + lacks the national region, its default dataset or the model-version + options, as get_economy_options does. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + datasets = read_input_datasets( + session, model_version_id=selected.model_version.id ) national_dataset = ( - read_input_dataset( - self._database_session.session, - model_version_id=selected.model_version.id, - dataset_id=national_region.default_dataset_id, + _validated_national_dataset( + selected, + country_id=country_id, + regions=regions, + find_dataset=lambda dataset_id: next( + (dataset for dataset in datasets if dataset.id == dataset_id), + None, + ), ) - if national_region is not None + if regions else None ) - dataset = validate_economy_options( + return metadata_model_catalog( selected, - country_id=country_id, regions=regions, + datasets=datasets, national_dataset=national_dataset, ) - return metadata_economy_options( - selected, regions=regions, national_dataset=dataset + + def get_catalog_variables( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also returns the current-law values of each parameter a variable's + adds or subtracts names, so a client can pick the list for any year. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + variables = read_all_variables( + session, model_version_id=selected.model_version.id + ) + part_paths = part_parameter_paths(variables) + part_values = read_current_law_values_by_parameter_name( + session, + model_version_id=selected.model_version.id, + parameter_names=part_paths, + ) + return metadata_catalog_variables( + selected, + variables=variables, + part_paths=part_paths, + part_values=part_values, + ) + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values, and the result carries the stored + labels of their ancestor nodes and the requested names the version + lacks. Repeated names count once. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + model_version_id = selected.model_version.id + requested_names = set(names) + parameters = read_parameters_by_name( + session, model_version_id=model_version_id, parameter_names=requested_names + ) + found_names = {parameter.name for parameter in parameters} + value_rows = read_current_law_values_by_parameter_name( + session, model_version_id=model_version_id, parameter_names=found_names + ) + node_rows = read_parameter_node_labels( + session, + model_version_id=model_version_id, + node_names={ + ancestor for name in found_names for ancestor in ancestor_paths(name) + }, + ) + return metadata_catalog_parameters( + selected, + parameters=parameters, + values_by_name=dated_values_by_parameter_name( + value_rows, parameter_names=found_names + ), + node_rows=node_rows, + missing_names=requested_names - found_names, ) + + +def _validated_national_dataset( + selected: SelectedCatalog, + *, + country_id: str, + regions: list[Region], + find_dataset: Callable[[UUID], Dataset | None], +) -> Dataset: + """Return the national region's default dataset, validated for economy choices. + + find_dataset loads a dataset of the selected version by id. Raise + MetadataCatalogUnavailableError when the national region, its dataset or + the model-version options are missing. + """ + region = national_region(regions, country_id) + return validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + None if region is None else find_dataset(region.default_dataset_id) + ), + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 68ebe21fe..00c197cac 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -7,11 +7,20 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, + MetadataEnumValue, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -22,6 +31,7 @@ MetadataRegionType, MetadataTimePeriodOption, MetadataVariable, + ParameterNodeLabelRow, ParameterValueSelection, ) @@ -103,9 +113,79 @@ def metadata_variable(variable: Variable) -> MetadataVariable: default_value=variable.default_value, adds=variable.adds, subtracts=variable.subtracts, + is_input_variable=variable.is_input_variable, + hidden_input=variable.hidden_input, + module_name=variable.module_name, + adds_parameter_path=variable.adds_parameter_path, + subtracts_parameter_path=variable.subtracts_parameter_path, + enum_values=( + None + if variable.enum_values is None + else [ + MetadataEnumValue.model_validate(enum_value) + for enum_value in variable.enum_values + ] + ), + unit=variable.unit, + definition_period=variable.definition_period, ) +def part_parameter_paths(variables: list[Variable]) -> set[str]: + """Return every parameter path the variables' adds or subtracts names.""" + return { + path + for variable in variables + for path in (variable.adds_parameter_path, variable.subtracts_parameter_path) + if path is not None + } + + +def metadata_catalog_variables( + selected: SelectedCatalog, + *, + variables: list[Variable], + part_paths: set[str], + part_values: list[CurrentLawValueRow], +) -> MetadataCatalogVariablesResult: + """Build every variable of the selected version and its part parameters' values. + + part_paths holds part_parameter_paths of the variables, and part_values the + current-law values of those parameters in name and start-date order. Each + path gets an entry, empty when no value carries its name. + """ + values_by_path = dated_values_by_parameter_name( + part_values, parameter_names=part_paths + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, + ) + + +def dated_values_by_parameter_name( + value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] +) -> dict[str, list[MetadataDatedValue]]: + """Group value rows by parameter name, with an empty list for each other name. + + value_rows are in parameter-name and start-date order. + """ + values_by_name: dict[str, list[MetadataDatedValue]] = { + name: [] for name in sorted(parameter_names) + } + for row in value_rows: + values_by_name.setdefault(row.name, []).append( + MetadataDatedValue( + start_date=as_utc(row.start_date), + end_date=None if row.end_date is None else as_utc(row.end_date), + value=row.value_json, + ) + ) + return values_by_name + + def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: """Return the API representation of a stored parameter.""" return MetadataParameterSummary( @@ -115,6 +195,51 @@ def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: description=parameter.description, data_type=parameter.data_type, unit=parameter.unit, + economy=parameter.economy, + household=parameter.household, + ) + + +def ancestor_paths(parameter_name: str) -> list[str]: + """Return the paths of every node above parameter_name, from the root down. + + A dot separates a child, and so does the ``[`` of a bracket index: + ``gov.tax.rates[0].rate`` has the ancestors ``gov``, ``gov.tax``, + ``gov.tax.rates`` and ``gov.tax.rates[0]``. + """ + return [ + parameter_name[:position] + for position, character in enumerate(parameter_name) + if character in ".[" and position > 0 + ] + + +def metadata_catalog_parameters( + selected: SelectedCatalog, + *, + parameters: list[Parameter], + values_by_name: dict[str, list[MetadataDatedValue]], + node_rows: list[ParameterNodeLabelRow], + missing_names: set[str], +) -> MetadataCatalogParametersResult: + """Build the requested parameters with their values and ancestor labels. + + values_by_name maps each parameter's name to its current-law values (see + dated_values_by_parameter_name), node_rows are the stored labels of their + ancestor nodes, and missing_names the requested names the version lacks. + """ + return MetadataCatalogParametersResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + parameters=[ + MetadataCatalogParameter( + **metadata_parameter(parameter).model_dump(), + values=values_by_name[parameter.name], + ) + for parameter in parameters + ], + node_labels={row.name: row.label for row in node_rows}, + missing_names=sorted(missing_names), ) @@ -129,6 +254,18 @@ def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParamete ) +def as_utc(moment: datetime) -> datetime: + """Return moment in UTC, treating a naive time as UTC. + + The database may return times in its session's time zone (or, in SQLite, + without one); serving them in UTC keeps a response's bytes, and so its + ETag, the same whatever that setting is. + """ + if moment.tzinfo is None: + return moment.replace(tzinfo=UTC) + return moment.astimezone(UTC) + + def utc_day_start(selected_time: datetime) -> datetime: """Return midnight UTC on the day of selected_time, treating a naive time as UTC.""" if selected_time.tzinfo is None: @@ -153,6 +290,7 @@ def parameter_value_selection( return ParameterValueSelection( parameter_id=value_filter.parameter_id, selected_day=selected_day, + parameter_names=value_filter.parameter_names, ) @@ -187,31 +325,26 @@ def metadata_region(region: Region) -> MetadataRegion: def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: """Build parameter-tree children from read_parameter_children rows. - Parameter leaves carry the parameter's summary, and a child with no label is - labelled with the last segment of its path. + Parameter leaves carry the same summary as the parameter routes, and a child + with no label is labelled with the last segment of its path. """ - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) + return [ + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=( + metadata_parameter(row.Parameter) if row.type == "parameter" else None + ), ) - return items + for row in rows + ] + + +def national_region(regions: list[Region], country_id: str) -> Region | None: + """Return the country's national region (its code is the country ID), or None.""" + return next((region for region in regions if region.code == country_id), None) def metadata_economy_options( @@ -221,9 +354,22 @@ def metadata_economy_options( The national dataset is the only dataset choice, and it is the default. """ + options = _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) return MetadataEconomyOptionsResult( policyengine_version=selected.policyengine_version, current_law_id=selected.model_version.current_law_id, + region=options.region, + time_period=options.time_period, + datasets=options.datasets, + ) + + +def _economy_option_lists( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptions: + return MetadataEconomyOptions( region=[ MetadataRegionOption( name=region.code, @@ -243,3 +389,39 @@ def metadata_economy_options( ) ], ) + + +def metadata_model_catalog( + selected: SelectedCatalog, + *, + regions: list[Region], + datasets: list[Dataset], + national_dataset: Dataset | None, +) -> MetadataModelCatalogResult: + """Build the small parts of the selected catalog version. + + Without a national dataset (a catalog without regions) the economy choices + are empty. Entities and basic inputs a version published before catalog + format 2 lacks are empty lists. + """ + model_version = selected.model_version + return MetadataModelCatalogResult( + policyengine_version=selected.policyengine_version, + catalog_format=model_version.catalog_format, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + entities=[ + MetadataEntity.model_validate(entity) + for entity in model_version.entities or [] + ], + basic_inputs=list(model_version.basic_inputs or []), + regions=[metadata_region(region) for region in regions], + datasets=[metadata_dataset(dataset) for dataset in datasets], + economy_options=( + _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) + if national_dataset is not None + else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) + ), + ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index a7336d7e9..63a77944e 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,34 +5,65 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import Generic, Literal, TypeVar +from typing import Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue +@dataclass(frozen=True) +class ParameterFilter: + """Which parameters a listing keeps. + + ``search`` keeps parameters whose name, label or description contains it, + ignoring case, and ``names`` the parameters with those exact names. + """ + + search: str | None = None + names: tuple[str, ...] | None = None + + @dataclass(frozen=True) class ParameterValueFilter: """Which current-law parameter values a listing keeps. - ``parameter_id`` keeps one parameter's values, and ``current`` keeps the - values in effect on the UTC day of ``now`` (today when ``now`` is None). + ``parameter_id`` keeps one parameter's values, ``parameter_names`` the + values of the parameters with those exact names, and ``current`` the values + in effect on the UTC day of ``now`` (today when ``now`` is None). """ parameter_id: UUID | None = None current: bool = False now: datetime | None = None + parameter_names: tuple[str, ...] | None = None @dataclass(frozen=True) class ParameterValueSelection: - """The parameter and effective day a parameter-value read is narrowed to. + """The parameters and effective day a parameter-value read is narrowed to. - None for either leaves the read unnarrowed by it. + None for any of them leaves the read unnarrowed by it. """ parameter_id: UUID | None selected_day: datetime | None + parameter_names: tuple[str, ...] | None = None + + +class CurrentLawValueRow(NamedTuple): + """One current-law value of a parameter, read with the parameter's name.""" + + name: str + value_json: Any + start_date: datetime + end_date: datetime | None + + +class ParameterNodeLabelRow(NamedTuple): + """A parameter node's name and its stored label (None when it has none).""" + + name: str + label: str | None class StrictResponseModel(BaseModel): @@ -71,11 +102,29 @@ class MetadataModelVersion(StrictResponseModel): description: str | None +class MetadataEnumValue(StrictResponseModel): + """One value of an enum variable: its stored name and its display label. + + The label keeps the type the country package declares it with. + """ + + value: str + label: str | int | float | bool | None + + class MetadataVariable(StrictResponseModel): """A variable the tax-benefit model computes or accepts as input. adds and subtracts, when present, list the variables summed into or - subtracted from this one. + subtracted from this one. When the model takes that list from a + parameter, adds_parameter_path or subtracts_parameter_path names the + parameter, whose dated values give the list for each year (adds and + subtracts hold its value on 2025-01-01). unit is the country package's + unit (for example ``currency-USD``) and definition_period the period a + value covers (``year``, ``month`` or ``eternity``). The catalog format 2 + fields (is_input_variable, hidden_input, module_name, the two parameter + paths, enum_values, unit and definition_period) are null in versions + published before that format. """ id: UUID @@ -88,12 +137,22 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: list[MetadataEnumValue] | None + unit: str | None + definition_period: str | None class MetadataParameterSummary(StrictResponseModel): """A policy parameter, described without its values. - Its current-law values are listed by GET /v2/parameter-values. + Its current-law values are listed by GET /v2/parameter-values. economy and + household say whether a reform of an economy or of a household may change + it; they are null in versions published before catalog format 2. """ id: UUID @@ -102,6 +161,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None + economy: bool | None + household: bool | None class MetadataCanonicalParameterValue(StrictResponseModel): @@ -239,3 +300,98 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] + + +class MetadataEntity(StrictResponseModel): + """A kind of member a household holds, such as a person or a tax unit. + + plural is the key the household's members of this kind are listed under, + and is_person marks the entity every household input about a person uses. + """ + + key: str + plural: str + is_person: bool + + +class MetadataEconomyOptions(StrictResponseModel): + """The region, year and dataset choices for an economy calculation. + + Every list is empty for a catalog without regions, such as a + household-only country's, which offers no economy calculation. + """ + + region: list[MetadataRegionOption] + time_period: list[MetadataTimePeriodOption] + datasets: list[MetadataDatasetOption] + + +class MetadataModelCatalogResult(StrictResponseModel): + """The small parts of one catalog version, read when a client starts. + + policyengine_version names the selected catalog version, so a client can + request its other parts for that exact version. catalog_format is null, + and entities and basic_inputs are empty, for a version published before + catalog format 2. Variables, parameters and their values are served by + their own routes. + """ + + policyengine_version: str + catalog_format: int | None + model: MetadataModel + model_version: MetadataModelVersionDetail + entities: list[MetadataEntity] + basic_inputs: list[str] + regions: list[MetadataRegion] + datasets: list[MetadataDataset] + economy_options: MetadataEconomyOptions + + +class MetadataDatedValue(StrictResponseModel): + """A current-law parameter value from start_date until end_date. + + A null end_date means the value applies until a later value starts, or + indefinitely. + """ + + start_date: datetime + end_date: datetime | None + value: JsonValue + + +class MetadataCatalogVariablesResult(StrictResponseModel): + """Every variable of one catalog version, in name order. + + part_parameter_values has an entry for each parameter path a variable's + adds_parameter_path or subtracts_parameter_path names: that parameter's + current-law values in start-date order, or an empty list when the + catalog has no such parameter. + """ + + policyengine_version: str + catalog_format: int | None + variables: list[MetadataVariable] + part_parameter_values: dict[str, list[MetadataDatedValue]] + + +class MetadataCatalogParameter(MetadataParameterSummary): + """A parameter with its current-law values in start-date order.""" + + values: list[MetadataDatedValue] + + +class MetadataCatalogParametersResult(StrictResponseModel): + """Requested parameters of one catalog version, in name order. + + node_labels maps each ancestor of a requested parameter that the version + stores as a parameter node to the node's stored label (null when it has + none), for breadcrumbs; a bracket index such as ``rates[0]`` counts as a + child of its schedule ``rates``. missing_names lists requested names the + version has no parameter for. + """ + + policyengine_version: str + catalog_format: int | None + parameters: list[MetadataCatalogParameter] + node_labels: dict[str, str | None] + missing_names: list[str] diff --git a/policyengine_api/vary_headers.py b/policyengine_api/vary_headers.py new file mode 100644 index 000000000..98d834086 --- /dev/null +++ b/policyengine_api/vary_headers.py @@ -0,0 +1,56 @@ +"""ASGI middleware that lists each Vary value of a response once.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from starlette.datastructures import MutableHeaders + +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +def unique_vary_values(vary: str) -> str: + """Return ``vary`` with each value listed once, in first-seen order. + + Values are compared without regard to case, as HTTP field names are. + """ + values: list[str] = [] + seen: set[str] = set() + for value in vary.split(","): + name = value.strip() + if name and name.lower() not in seen: + seen.add(name.lower()) + values.append(name) + return ", ".join(values) + + +class UniqueVaryMiddleware: + """List each Vary value of every HTTP response once. + + Starlette's CORS and gzip middleware append Origin and Accept-Encoding to + Vary without checking whether a route already named them. This wraps the + outermost middleware, so the response that leaves the application names + each value once. + """ + + def __init__(self, app: ASGIApp) -> None: + """Wrap ``app``.""" + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Pass the request through, rewriting a duplicated Vary header.""" + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_unique_vary(message: Message) -> None: + if message["type"] == "http.response.start": + message.setdefault("headers", []) + headers = MutableHeaders(scope=message) + vary = headers.get("vary") + if vary is not None: + headers["vary"] = unique_vary_values(vary) + await send(message) + + await self.app(scope, receive, send_with_unique_vary) diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..b84980a1c 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -671,6 +671,55 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options", + ), + route_group="metadata", + ), + ContractRequest( + method="GET", + path="/v2/model-catalog/variables?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values", + ), + route_group="metadata", + ), + ContractRequest( + method="GET", + path=( + "/v2/model-catalog/parameters?country_id=us" + "&name=gov.irs.income.bracket.rates.1" + ), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 235304156..00ff143e2 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -1,8 +1,11 @@ """Read-only checks for deployed Cloud Run v2 metadata preview routes.""" +import os +import time import tomllib from pathlib import Path +import httpx import pytest REPO = Path(__file__).parents[2] @@ -44,7 +47,15 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: assert response.status_code == 200, response.text[:500] document = response.json() expected_paths = {f"/v2/{resource}" for resource in PAGED_RESOURCES} - expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) + expected_paths.update( + { + "/v2/parameters/children", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", + } + ) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) assert "V2ErrorResponse" in document["components"]["schemas"] @@ -99,8 +110,129 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: params={"country_id": "us", "policyengine_version": "not a version"}, ) - assert response.status_code == 400, response.text[:500] + assert response.status_code == 422, response.text[:500] payload = response.json() assert payload["status"] == "error" assert payload["message"] assert "result" not in payload or payload["result"] is None + + +def _timed_get( + api_client: httpx.Client, path: str, **request_options: object +) -> tuple[httpx.Response, float]: + started = time.perf_counter() + response = api_client.get(path, **request_options) + return response, time.perf_counter() - started + + +def _record_route_times( + country_id: str, route_times: list[tuple[str, float, httpx.Response]] +) -> None: + """Append each route's time and size to the GitHub job summary, when there is one. + + The release workflow runs this test against each Cloud Run candidate, so + the summary keeps every release's staging and production route times. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + lines = [ + f"\n#### v2 catalog route times ({country_id})\n", + "| Request | Time (ms) | Body (bytes, uncompressed) |", + "| --- | ---: | ---: |", + ] + lines.extend( + f"| {label} | {seconds * 1000:.0f} | {len(response.content)} |" + for label, seconds, response in route_times + ) + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write("\n".join(lines) + "\n") + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> None: + expected_version = _installed_policyengine_version() + route_times: list[tuple[str, float, httpx.Response]] = [] + + def timed_get(label: str, path: str, **request_options: object) -> httpx.Response: + response, seconds = _timed_get(api_client, path, **request_options) + route_times.append((label, seconds, response)) + return response + + # The times are recorded even when an assertion fails, so every release + # keeps the measurements it reached. + try: + _check_catalog_parts(timed_get, country_id, expected_version) + finally: + _record_route_times(country_id, route_times) + + +def _check_catalog_parts(timed_get, country_id: str, expected_version: str) -> None: + catalog = _assert_ok( + timed_get( + "/v2/model-catalog (running version)", + "/v2/model-catalog", + params={"country_id": country_id}, + ), + expected_version, + ) + assert catalog["regions"] + assert catalog["economy_options"]["region"] + assert "variables" not in catalog + pinned = {"country_id": country_id, "policyengine_version": expected_version} + + variables_response = timed_get( + "/v2/model-catalog/variables", "/v2/model-catalog/variables", params=pinned + ) + second_variables = timed_get( + "/v2/model-catalog/variables (second request; another instance may serve it)", + "/v2/model-catalog/variables", + params=pinned, + ) + variables = _assert_ok(variables_response, expected_version) + assert variables["variables"] + assert variables_response.headers["cache-control"] == "public, max-age=86400" + assert second_variables.headers["etag"] == variables_response.headers["etag"] + revalidated = timed_get( + "/v2/model-catalog/variables (revalidated)", + "/v2/model-catalog/variables", + params=pinned, + headers={"If-None-Match": variables_response.headers["etag"]}, + ) + assert revalidated.status_code == 304 + + # The two existing routes measured slow in production on 2026-10-21. + page = _assert_ok( + timed_get( + "/v2/parameters (page of 500)", + "/v2/parameters", + params={**pinned, "limit": 500}, + ), + expected_version, + ) + first_parameter = page["items"][0] + one_parameter_values = _assert_ok( + timed_get( + "/v2/parameter-values (one parameter)", + "/v2/parameter-values", + params={**pinned, "parameter_id": first_parameter["id"]}, + ), + expected_version, + ) + assert all( + value["parameter_id"] == first_parameter["id"] + for value in one_parameter_values["items"] + ) + + parameters = _assert_ok( + timed_get( + "/v2/model-catalog/parameters (1 name)", + "/v2/model-catalog/parameters", + params={**pinned, "name": first_parameter["name"]}, + ), + expected_version, + ) + assert [parameter["name"] for parameter in parameters["parameters"]] == [ + first_parameter["name"] + ] + assert parameters["missing_names"] == [] diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 326eb78d9..7f479d183 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 @@ -11,7 +12,7 @@ from alembic.config import Config from fastapi.testclient import TestClient from flask import Flask, jsonify -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import Engine, make_url from sqlalchemy.pool import NullPool from sqlmodel import Session, select @@ -20,8 +21,11 @@ from policyengine_api.data.v2.catalog.publication import publish_catalog from policyengine_api.data.v2.models import ( Dataset, + Parameter, + ParameterValue, TaxBenefitModel, TaxBenefitModelVersion, + Variable, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -324,7 +328,7 @@ def test_postgres_resources_distinguish_invalid_and_absent_versions( params={"country_id": "us", "policyengine_version": "4.99.0"}, ) - assert invalid.status_code == 400 + assert invalid.status_code == 422 assert invalid.json()["status"] == "error" assert invalid.json()["message"] assert absent.status_code == 404 @@ -360,3 +364,362 @@ def test_postgres_parameter_collection_remains_available_without_values( assert parameters.status_code == values.status_code == 200 assert parameters.json()["result"]["items"] assert values.json()["result"]["items"] == [] + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_postgres_model_catalog_serves_the_small_parts_of_a_version( + published_engine: Engine, + country_id: str, +) -> None: + client = _client(published_engine) + country = normalized_catalog().country(country_id) + + running = client.get("/v2/model-catalog", params={"country_id": country_id}) + named = client.get( + "/v2/model-catalog", + params={"country_id": country_id, "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert running.status_code == 200 + assert running.headers["cache-control"] == "public, max-age=300" + assert named.headers["cache-control"] == "public, max-age=86400" + assert running.headers["etag"] == named.headers["etag"] + result = running.json()["result"] + assert result["policyengine_version"] == POLICYENGINE_VERSION + assert result["catalog_format"] == country.model_version.catalog_format + assert result["entities"] == [ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ] + assert result["basic_inputs"] == list(country.model_version.basic_inputs) + assert [region["code"] for region in result["regions"]] == sorted( + region.code for region in country.regions + ) + assert [dataset["name"] for dataset in result["datasets"]] == sorted( + dataset.name for dataset in country.datasets + ) + assert [option["name"] for option in result["economy_options"]["region"]] == [ + region["code"] for region in result["regions"] + ] + assert "variables" not in result + assert "parameters" not in result + + revalidated = client.get( + "/v2/model-catalog", + params={"country_id": country_id}, + headers={"If-None-Match": running.headers["etag"]}, + ) + assert revalidated.status_code == 304 + + +def _us_model_version_id(engine: Engine): + with Session(engine) as session: + return session.exec( + select(TaxBenefitModelVersion.id) + .join( + TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id + ) + .where(TaxBenefitModel.name == "policyengine-us") + ).one() + + +def test_postgres_catalog_variables_carry_part_parameter_values_by_date( + published_engine: Engine, +) -> None: + model_version_id = _us_model_version_id(published_engine) + parts_id = uuid4() + with Session(published_engine) as session: + session.add( + Parameter( + id=parts_id, + tax_benefit_model_version_id=model_version_id, + name="gov.example.parts", + label="Income parts", + economy=False, + household=True, + ) + ) + session.flush() + session.add_all( + [ + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + ), + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + ), + Variable( + tax_benefit_model_version_id=model_version_id, + name="total_income", + entity="person", + data_type="float", + default_value=0, + adds=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + ), + ] + ) + session.commit() + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income", + "total_income", + ] + assert result["variables"][0]["module_name"] == ( + "household.income.employment_income" + ) + assert result["variables"][0]["unit"] == "currency-USD" + assert result["variables"][0]["definition_period"] == "year" + assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2024-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + }, + { + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income", "pension_income"], + }, + ] + } + uk = client.get("/v2/model-catalog/variables", params={"country_id": "uk"}) + assert [variable["name"] for variable in uk.json()["result"]["variables"]] == [ + "employment_income" + ] + assert uk.json()["result"]["part_parameter_values"] == {} + + +def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( + published_engine: Engine, +) -> None: + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/parameters", + params=[ + ("country_id", "us"), + ("policyengine_version", POLICYENGINE_VERSION), + ("name", "gov.example.rate"), + ("name", "gov.removed.amount"), + ], + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [parameter["name"] for parameter in result["parameters"]] == [ + "gov.example.rate" + ] + parameter = result["parameters"][0] + assert (parameter["economy"], parameter["household"]) == (True, False) + assert parameter["values"] == [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": "2025-12-31T00:00:00Z", + "value": 0.1, + }, + {"start_date": "2026-01-01T00:00:00Z", "end_date": None, "value": 0.2}, + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed.amount"] + + +def test_postgres_metadata_reads_send_no_transaction_statements( + published_engine: Engine, +) -> None: + # Each metadata read is one statement on rows a published catalog never + # changes, so the read session runs in autocommit mode: psycopg then sends + # no BEGIN before the reads and no ROLLBACK when the session closes. The + # engine has one pooled connection, so the writer below must get the same + # connection back in the ordinary, transactional mode. + from psycopg.pq import TransactionStatus + + from policyengine_api.data.v2.database import build_v2_read_session_factory + + del published_engine # publishes the catalog this pooled engine reads + pooled_engine = create_engine(_disposable_url(), pool_size=1, max_overflow=0) + read_connections: list[object] = [] + autocommit_modes: list[bool] = [] + + def record_mode(connection, _cursor, _statement, *_args) -> None: + dbapi_connection = connection.connection.dbapi_connection + read_connections.append(dbapi_connection) + autocommit_modes.append(dbapi_connection.autocommit) + + try: + event.listen(pooled_engine, "before_cursor_execute", record_mode) + service = V2MetadataService( + MetadataDatabaseSession(build_v2_read_session_factory(pooled_engine)()), + running_policyengine_version=POLICYENGINE_VERSION, + ) + result = service.list_parameters("us") + service.close() + event.remove(pooled_engine, "before_cursor_execute", record_mode) + + with Session(pooled_engine) as writer: + writer_connection = writer.connection().connection.dbapi_connection + writer.execute(text("SELECT 1")) + writer_status = writer_connection.info.transaction_status + writer_autocommit = writer_connection.autocommit + finally: + pooled_engine.dispose() + + assert [item.name for item in result.items] == ["gov.example.rate"] + assert autocommit_modes + assert all(autocommit_modes) + assert {id(connection) for connection in read_connections} == { + id(writer_connection) + } + assert writer_autocommit is False + assert writer_status == TransactionStatus.INTRANS + + +def test_postgres_production_metadata_reader_keeps_catalogs_and_responses( + published_engine: Engine, +) -> None: + # The production combination: autocommit read sessions, the process's + # selected catalogs and its whole-version response cache. + from policyengine_api.data.v2.database import build_v2_read_session_factory + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) + + read_factory = build_v2_read_session_factory(published_engine) + selected_catalogs = SelectedCatalogStore() + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_metadata_reader_factory=lambda: V2MetadataService( + MetadataDatabaseSession(read_factory()), + running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, + ), + v2_metadata_response_cache=MetadataResponseCache(), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + query = {"country_id": "us", "policyengine_version": POLICYENGINE_VERSION} + + event.listen(published_engine, "before_cursor_execute", record_statement) + try: + first_variables = client.get( + "/v2/model-catalog/variables", params={"country_id": "us"} + ) + statements_for_first = len(statements) + repeated_variables = client.get("/v2/model-catalog/variables", params=query) + statements_for_repeat = len(statements) - statements_for_first + parameters = client.get( + "/v2/parameters", params={"country_id": "us", "name": "gov.example.rate"} + ) + statements_for_parameters = ( + len(statements) - statements_for_first - statements_for_repeat + ) + finally: + event.remove(published_engine, "before_cursor_execute", record_statement) + + assert first_variables.status_code == 200 + assert repeated_variables.content == first_variables.content + assert repeated_variables.headers["etag"] == first_variables.headers["etag"] + # The running version's selection and the variables read (the fixture's + # variables name no parameter-defined parts, so no value read); then + # nothing for the explicit request, which shares the kept response; then + # only the parameter read, the running version's selection being kept. + assert statements_for_first == 2 + assert statements_for_repeat == 0 + assert statements_for_parameters == 1 + assert [item["name"] for item in parameters.json()["result"]["items"]] == [ + "gov.example.rate" + ] + assert selected_catalogs.get(("us", None)) is not None + + +@pytest.mark.parametrize( + "case", + [ + # Read for 31 December, headers written after midnight: not reusable. + ( + datetime(2025, 12, 31, 23, 59, 59, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, 1, tzinfo=UTC), + [0.1], + "public, max-age=0", + ), + ( + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + [0.1], + "public, max-age=43200", + ), + ( + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + [0.2], + "public, max-age=86400", + ), + ], +) +def test_postgres_current_values_and_their_lifetime_use_one_instant( + published_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + case: tuple[datetime, datetime, list[float], str], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant, header_instant, expected_values, expected_cache_control = case + instants = iter([read_instant, header_instant]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + + response = _client(published_engine).get( + "/v2/parameter-values", + params={ + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "current": "true", + }, + ) + + assert response.status_code == 200 + assert [item["value"] for item in response.json()["result"]["items"]] == ( + expected_values + ) + assert response.headers["cache-control"] == expected_cache_control diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..e35785e12 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 89, + "request_count": 92, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index a00667a49..fbfa92c72 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -1,11 +1,14 @@ """Tests for lazy process-owned v2 engine and SQLModel Session factories.""" from collections.abc import Iterator +from pathlib import Path import pytest -from sqlmodel import Session +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select from policyengine_api.data.v2 import database +from policyengine_api.data.v2.models import V2_METADATA, TaxBenefitModel from policyengine_api.data.v2.settings import ( V2_MIGRATION_DATABASE_URL, V2_RUNTIME_DATABASE_URL, @@ -125,3 +128,66 @@ def test_forked_process_state_builds_a_new_engine( assert child_engine is not parent_engine assert database._engine_state.engine_pid == parent_pid + 1 + + +def test_read_sessions_share_the_runtime_pool_without_transactions() -> None: + settings = load_v2_runtime_database_settings(_environment()) + + read_factory = database.get_v2_read_session_factory(settings) + runtime_engine = database.get_v2_engine(settings) + + with read_factory() as session: + bind = session.get_bind() + assert bind.get_execution_options()["isolation_level"] == "AUTOCOMMIT" + assert bind.pool is runtime_engine.pool + assert database.get_v2_read_session_factory(settings) is read_factory + assert runtime_engine.get_execution_options().get("isolation_level") is None + + +def test_read_session_factory_is_rebuilt_with_the_engine() -> None: + settings = load_v2_runtime_database_settings(_environment()) + first = database.get_v2_read_session_factory(settings) + + database.close_v2_database() + + assert database.get_v2_read_session_factory(settings) is not first + + +def test_read_sessions_refuse_to_write_rows() -> None: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + V2_METADATA.create_all(engine) + read_factory = database.build_v2_read_session_factory(engine) + + with read_factory() as session: + session.add(TaxBenefitModel(name="policyengine-test")) + with pytest.raises(database.ReadOnlySessionError): + session.flush() + + with Session(engine) as session: + assert session.exec(select(TaxBenefitModel)).all() == [] + engine.dispose() + + +def test_only_the_metadata_reader_opens_read_sessions() -> None: + package = Path(database.__file__).parents[2] + users = sorted( + str(module.relative_to(package)) + for module in package.rglob("*.py") + if module.name != "database.py" + and ( + "get_v2_read_session_factory" in module.read_text(encoding="utf-8") + or "build_v2_read_session_factory" in module.read_text(encoding="utf-8") + ) + ) + + assert users == ["fastapi_routes/dependencies.py"] + dependencies_source = (package / "fastapi_routes/dependencies.py").read_text( + encoding="utf-8" + ) + reader_factory = dependencies_source.split( + "def _default_v2_metadata_reader_factory" + )[1].split("\ndef ")[0] + assert "get_v2_read_session_factory" in reader_factory + assert dependencies_source.count("get_v2_read_session_factory") == 2 diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py new file mode 100644 index 000000000..a0a788c1c --- /dev/null +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -0,0 +1,268 @@ +"""The process-wide store of whole catalog-version responses.""" + +from __future__ import annotations + +import threading +import time + +import pytest +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + MetadataResponseCache, + body_entity_tag, +) + +RUNNING_VERSION = "6.10.0" + + +def _counting_build( + builds: list[str], *, body: bytes = b'{"a":1}', version: str = RUNNING_VERSION +): + def build() -> BuiltResponse: + builds.append("built") + return BuiltResponse(body=body, policyengine_version=version) + + return build + + +def test_a_response_is_built_once_and_kept_with_its_entity_tag() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + first = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds, body=b"{}") + ) + + assert first == KeptResponse( + body=b'{"a":1}', entity_tag=body_entity_tag(b'{"a":1}') + ) + assert second is first + assert builds == ["built"] + + +@pytest.mark.parametrize("requested_first", [None, RUNNING_VERSION]) +def test_the_running_version_and_its_number_share_one_kept_response( + requested_first: str | None, +) -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + requested_second = RUNNING_VERSION if requested_first is None else None + + first = cache.get_or_build( + "variables", "us", requested_first, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + third = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + + assert second is first + assert third is first + assert builds == (["built"] if requested_first is None else ["built", "built"]) + + +def test_different_routes_countries_and_versions_are_kept_apart() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + for route, country_id, version in ( + ("variables", "us", "6.10.0"), + ("variables", "uk", "6.10.0"), + ("variables", "us", "6.9.0"), + ("model-catalog", "us", "6.10.0"), + ): + cache.get_or_build( + route, country_id, version, _counting_build(builds, version=version) + ) + + assert len(builds) == 4 + + +def test_a_failed_read_is_returned_but_not_kept() -> None: + cache = MetadataResponseCache() + failure = JSONResponse({"status": "error"}, status_code=503) + builds: list[str] = [] + + def failing_build() -> JSONResponse: + builds.append("failed") + return failure + + first = cache.get_or_build("variables", "us", None, failing_build) + second = cache.get_or_build("variables", "us", None, _counting_build(builds)) + + assert first is failure + assert isinstance(second, KeptResponse) + assert builds == ["failed", "built"] + + +def test_the_least_recently_used_response_is_dropped_beyond_capacity() -> None: + cache = MetadataResponseCache(capacity=2) + builds: list[str] = [] + + def get(country_id: str) -> None: + cache.get_or_build("variables", country_id, "6.10.0", _counting_build(builds)) + + for country_id in ("a", "b", "a", "c", "a", "b"): + get(country_id) + + assert builds == ["built", "built", "built", "built"] + + +def test_concurrent_requests_for_one_response_build_it_once() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + building = threading.Event() + release = threading.Event() + + def blocked_build() -> BuiltResponse: + builds.append("built") + building.set() + release.wait() + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + def request() -> None: + cache.get_or_build("variables", "us", None, blocked_build) + + first = threading.Thread(target=request) + first.start() + building.wait() + others = [threading.Thread(target=request) for _ in range(3)] + for thread in others: + thread.start() + release.set() + for thread in [first, *others]: + thread.join() + + # Whether the others waited on the build or arrived after it, none read. + assert builds == ["built"] + + +def test_a_request_arriving_while_a_response_is_stored_waits_for_it() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + second_results: list[object] = [] + second_requests: list[threading.Thread] = [] + original_keep = cache._keep + + def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: + second_request = threading.Thread( + target=lambda: second_results.append( + cache.get_or_build("variables", "us", None, _counting_build(builds)) + ) + ) + second_requests.append(second_request) + second_request.start() + # The second request may not build while this one stores its result: + # it waits on the build, whatever the timing. + second_request.join(timeout=0.2) + return original_keep(*keep_arguments) + + cache._keep = keep_after_another_request_arrives # type: ignore[method-assign] + + first = cache.get_or_build("variables", "us", None, _counting_build(builds)) + second_requests[0].join() + + assert builds == ["built"] + assert second_results == [first] + + +def _wait_for_waiters(cache: MetadataResponseCache, key: tuple, count: int) -> None: + # Polls until every request has registered as waiting on the read in + # progress, so the result does not depend on thread timing. + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + with cache._lock: + in_flight = cache._builds.get(key) + if in_flight is not None and in_flight.waiters == count: + return + time.sleep(0.001) + pytest.fail(f"{count} requests did not wait on the read in progress") + + +@pytest.mark.parametrize("read_outcome", ["failure", "success"]) +def test_waiting_requests_receive_the_outcome_of_the_one_read( + read_outcome: str, +) -> None: + read_fails = read_outcome == "failure" + cache = MetadataResponseCache() + builds: list[str] = [] + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> BuiltResponse | JSONResponse: + builds.append("built") + release.wait() + if read_fails: + return JSONResponse({"status": "error"}, status_code=503) + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + results: list[object] = [] + + def request() -> None: + results.append(cache.get_or_build(*key, blocked_build)) + + owner = threading.Thread(target=request, daemon=True) + waiters = [threading.Thread(target=request, daemon=True) for _ in range(9)] + try: + owner.start() + while not builds: + time.sleep(0.001) + for thread in waiters: + thread.start() + _wait_for_waiters(cache, key, 9) + finally: + release.set() + for thread in [owner, *waiters]: + thread.join(timeout=10) + + assert builds == ["built"] + assert len(results) == 10 + if read_fails: + assert all(result.status_code == 503 for result in results) + assert {bytes(result.body) for result in results} == {b'{"status":"error"}'} + # Each request gets its own response object, so middleware headers + # added for one request cannot reach another. + assert len({id(result) for result in results}) == 10 + else: + assert all(result is results[0] for result in results) + assert key not in cache._builds + + +def test_a_finished_read_leaves_a_newer_reads_entry_in_place() -> None: + cache = MetadataResponseCache() + building = threading.Event() + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> JSONResponse: + building.set() + release.wait() + return JSONResponse({"status": "error"}, status_code=503) + + owner = threading.Thread( + target=lambda: cache.get_or_build(*key, blocked_build), daemon=True + ) + newer_read = object() + try: + owner.start() + building.wait(timeout=10) + with cache._lock: + cache._builds[key] = newer_read + finally: + release.set() + owner.join(timeout=10) + + assert cache._builds[key] is newer_read + + +def test_capacity_must_be_positive() -> None: + with pytest.raises(ValueError, match="capacity"): + MetadataResponseCache(capacity=0) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..037b2952f 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -20,17 +20,30 @@ from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataDetailResult, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -40,6 +53,7 @@ MetadataRegionOption, MetadataTimePeriodOption, MetadataVariable, + ParameterFilter, ) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, @@ -112,6 +126,14 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: default_value=0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter = MetadataParameterSummary( id=parameter_id, @@ -120,6 +142,8 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: description=None, data_type="float", unit="/1", + economy=True, + household=False, ) parameter_value = MetadataCanonicalParameterValue( id=parameter_value_id, @@ -193,6 +217,63 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_parameters": MetadataCatalogParametersResult( + policyengine_version=version, + catalog_format=2, + parameters=[ + MetadataCatalogParameter( + **parameter.model_dump(), + values=[ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=0.1, + ) + ], + ) + ], + node_labels={"gov.example": "Example policy"}, + missing_names=["gov.removed"], + ), + "get_catalog_variables": MetadataCatalogVariablesResult( + policyengine_version=version, + catalog_format=2, + variables=[variable], + part_parameter_values={ + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income"], + ) + ] + }, + ), + "get_model_catalog": MetadataModelCatalogResult( + policyengine_version=version, + catalog_format=2, + model=model, + model_version=model_version, + entities=[ + MetadataEntity(key="person", plural="people", is_person=True) + ], + basic_inputs=["employment_income"], + regions=[region], + datasets=[dataset], + economy_options=MetadataEconomyOptions( + region=[ + MetadataRegionOption( + name="us", label="United States", type="national" + ) + ], + time_period=[MetadataTimePeriodOption(name=2026, label="2026")], + datasets=[ + MetadataDatasetOption( + name="populace_us_2024", label="Microcosm" + ) + ], + ), + ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, current_law_id=2, @@ -221,7 +302,7 @@ def detail(item: object) -> MetadataDetailResult: ) -def _client(factory) -> TestClient: +def _client(factory, *, response_cache=None) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -240,6 +321,7 @@ def v1_metadata(country_id: str): metadata_reader_factory=lambda: None, specification_provider=dict, v2_metadata_reader_factory=factory, + v2_metadata_response_cache=response_cache, ) settings = RouteImplementationSettings( health=RouteImplementation.FLASK_FALLBACK, @@ -443,12 +525,17 @@ def test_default_reader_uses_the_installed_policyengine_version( captured = {} reader = object() - def query_service(candidate_session, *, running_policyengine_version): + def query_service( + candidate_session, *, running_policyengine_version, selected_catalogs + ): captured["session"] = candidate_session.session captured["version"] = running_policyengine_version + captured["selected_catalogs"] = selected_catalogs return reader - monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) + monkeypatch.setattr( + database, "get_v2_read_session_factory", lambda: lambda: session + ) monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, @@ -462,7 +549,12 @@ def query_service(candidate_session, *, running_policyengine_version): route_dependencies._running_policyengine_version.cache_clear() assert result is reader - assert captured == {"session": session, "version": "5.2.0"} + assert captured["session"] is session + assert captured["version"] == "5.2.0" + assert isinstance(captured["selected_catalogs"], SelectedCatalogStore) + assert ( + route_dependencies._process_selected_catalogs() is captured["selected_catalogs"] + ) def test_default_factory_and_close_failures_preserve_success( @@ -521,6 +613,9 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -565,21 +660,816 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: for path in metadata_paths: operation = schema["paths"][path]["get"] - assert set(operation["responses"]) >= { - "200", - "400", - "404", - "405", - "422", - "500", - "503", - } + error_statuses = ["404", "405", "422", "500", "503"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + error_statuses.append("400") + assert set(operation["responses"]) >= {"200", *error_statuses} success_schema = operation["responses"]["200"]["content"]["application/json"][ "schema" ] assert success_schema["$ref"].startswith("#/components/schemas/Metadata") - for status in ("400", "404", "405", "422", "500", "503"): + for status in error_statuses: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" + + +def _every_resource_path(ids: dict[str, object]) -> list[str]: + return [ + "/v2/tax-benefit-models?country_id=us", + "/v2/tax-benefit-models/by-country/us", + f"/v2/tax-benefit-models/{ids['model_id']}?country_id=us", + "/v2/tax-benefit-model-versions?country_id=us", + f"/v2/tax-benefit-model-versions/{ids['model_version_id']}?country_id=us", + "/v2/variables?country_id=us", + f"/v2/variables/{ids['variable_id']}?country_id=us", + "/v2/parameters?country_id=us", + "/v2/parameters/children?country_id=us&parent_path=gov.example", + f"/v2/parameters/{ids['parameter_id']}?country_id=us", + "/v2/parameter-values?country_id=us", + f"/v2/parameter-values/{ids['parameter_value_id']}?country_id=us", + "/v2/datasets?country_id=us", + f"/v2/datasets/{ids['dataset_id']}?country_id=us", + "/v2/regions?country_id=us", + "/v2/regions/by-code/us?country_id=us", + f"/v2/regions/{ids['region_id']}?country_id=us", + "/v2/economy-options?country_id=us", + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/parameters?country_id=us&name=gov.example.rate", + ] + + +def test_every_resource_route_serves_an_entity_tag_and_cache_lifetime() -> None: + results, ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + + for path in _every_resource_path(ids): + response = client.get(path) + assert response.status_code == 200, path + assert response.headers["cache-control"] == "public, max-age=300", path + assert response.headers["etag"].startswith('"'), path + assert response.headers["etag"].endswith('"'), path + + +def test_a_named_catalog_version_is_reusable_for_a_day() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + query = {"country_id": "us", "policyengine_version": "4.20.3"} + + first = client.get("/v2/variables", params=query) + second = client.get("/v2/variables", params=query) + + assert first.headers["cache-control"] == "public, max-age=86400" + assert first.headers["etag"] == second.headers["etag"] + + +def test_a_matching_entity_tag_gets_a_not_modified_response_without_a_body() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + path = "/v2/variables?country_id=us&policyengine_version=4.20.3" + entity_tag = client.get(path).headers["etag"] + + revalidated = client.get(path, headers={"If-None-Match": entity_tag}) + weakly_revalidated = client.get(path, headers={"If-None-Match": f"W/{entity_tag}"}) + listed = client.get(path, headers={"If-None-Match": f'"other", {entity_tag}'}) + stale = client.get(path, headers={"If-None-Match": '"other"'}) + + for not_modified in (revalidated, weakly_revalidated, listed): + assert not_modified.status_code == 304 + assert not_modified.content == b"" + assert not_modified.headers["etag"] == entity_tag + assert not_modified.headers["cache-control"] == "public, max-age=86400" + assert stale.status_code == 200 + assert stale.json()["status"] == "ok" + + +def test_different_content_gets_a_different_entity_tag() -> None: + results, _ids = _resource_results() + changed_results, _changed_ids = _resource_results() + + first = _client(lambda: ResourceReader(results)).get("/v2/variables?country_id=us") + changed = _client(lambda: ResourceReader(changed_results)).get( + "/v2/variables?country_id=us" + ) + + assert first.headers["etag"] != changed.headers["etag"] + + +def test_failed_reads_carry_no_cache_headers() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get( + "/v2/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert response.status_code == 404 + assert "etag" not in response.headers + assert "cache-control" not in response.headers + + +def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", params={"country_id": "US"} + ) + + assert response.status_code == 200 + result = response.json()["result"] + assert result["policyengine_version"] == "4.20.3" + assert result["catalog_format"] == 2 + assert result["entities"] == [ + {"key": "person", "plural": "people", "is_person": True} + ] + assert result["basic_inputs"] == ["employment_income"] + assert [region["code"] for region in result["regions"]] == ["us"] + assert result["economy_options"]["datasets"][0]["default"] is True + assert reader.calls == [("get_model_catalog", ("us", None), {})] + assert reader.closed + + +def test_model_catalog_route_names_an_explicit_version_for_a_day() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", + params={"country_id": "uk", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + assert reader.calls == [("get_model_catalog", ("uk", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + [ + "", + "country_id=ca", + "country_id=us&country_id=uk", + "country_id=us&limit=10", + "country_id=us&policyengine_version=latest", + ], +) +def test_model_catalog_route_rejects_invalid_queries(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog?{query}") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert calls == [] + + +def test_model_catalog_route_declares_its_query_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version"} + assert parameters["country_id"]["required"] is True + assert parameters["policyengine_version"]["required"] is False + success = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} + + +def test_catalog_variables_route_serves_every_variable_and_part_values() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["catalog_format"] == 2 + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income" + ] + assert result["variables"][0]["module_name"] is None + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + } + ] + } + assert reader.calls == [("get_catalog_variables", ("us", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + ["", "country_id=us&offset=0", "country_id=us&search=income"], +) +def test_catalog_variables_route_rejects_paging_and_unknown_queries( + query: str, +) -> None: + results, _ids = _resource_results() + response = _client(lambda: ResourceReader(results)).get( + f"/v2/model-catalog/variables?{query}" + ) + + assert response.status_code == 422 + + +def test_catalog_parameters_route_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/parameters" + "?country_id=us&policyengine_version=4.20.3" + "&name=gov.example.rate&name=gov.removed" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["parameters"][0]["name"] == "gov.example.rate" + assert result["parameters"][0]["values"] == [ + {"start_date": "2025-01-01T00:00:00Z", "end_date": None, "value": 0.1} + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed"] + assert reader.calls == [ + ( + "get_catalog_parameters", + ("us", "4.20.3"), + {"names": ["gov.example.rate", "gov.removed"]}, + ) + ] + + +@pytest.mark.parametrize( + "query", + [ + "country_id=us", + "country_id=us&name=", + "country_id=us&name=gov.example.rate&offset=0", + "country_id=us&" + "&".join(f"name=gov.p{index}" for index in range(101)), + ], +) +def test_catalog_parameters_route_rejects_invalid_name_lists(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog/parameters?{query}") + + assert response.status_code == 422 + assert calls == [] + + +def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog/parameters"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version", "name"} + assert parameters["name"]["required"] is True + name_schema = parameters["name"]["schema"] + assert name_schema["type"] == "array" + assert name_schema["minItems"] == 1 + assert name_schema["maxItems"] == 100 + + +def test_parameter_collection_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + client = _client(lambda: reader) + + named = client.get( + "/v2/parameters?country_id=US&name=gov.example.rate&name=gov.other.rate" + ) + paged = client.get("/v2/parameters?country_id=us&offset=5&limit=50&search=rate") + + assert named.status_code == 200 + assert paged.status_code == 200 + assert reader.calls == [ + ( + "list_parameters", + ("us", None), + { + "offset": 0, + "limit": 100, + "parameter_filter": ParameterFilter( + search=None, names=("gov.example.rate", "gov.other.rate") + ), + }, + ), + ( + "list_parameters", + ("us", None), + { + "offset": 5, + "limit": 50, + "parameter_filter": ParameterFilter(search="rate", names=None), + }, + ), + ] + + +def test_parameter_values_fetch_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3" + "¶meter_name=gov.example.rate¶meter_name=gov.other.rate¤t=true" + ) + + assert response.status_code == 200 + name, args, kwargs = reader.calls[0] + assert (name, args) == ("list_parameter_values", ("us", "4.20.3")) + assert kwargs["value_filter"].parameter_names == ( + "gov.example.rate", + "gov.other.rate", + ) + assert kwargs["value_filter"].parameter_id is None + assert kwargs["value_filter"].current is True + assert (kwargs["offset"], kwargs["limit"]) == (0, 100) + + +@pytest.mark.parametrize( + "path", + [ + "/v2/parameters?country_id=us¶meter_id=1", + "/v2/parameters?country_id=us&limit=5&limit=6", + "/v2/parameters?country_id=us&name=", + "/v2/parameters?country_id=ca", + "/v2/parameters?country_id=us&" + + "&".join(f"name=gov.p{index}" for index in range(101)), + "/v2/parameter-values?country_id=us&name=gov.example.rate", + "/v2/parameter-values?country_id=us¤t=true¤t=false", + "/v2/parameter-values?country_id=us¶meter_id=not-a-uuid", + ], +) +def test_parameter_collections_reject_unknown_repeated_and_invalid_queries( + path: str, +) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(path) + + assert response.status_code == 422 + assert calls == [] + + +def test_parameter_collections_declare_their_name_filters_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + def query_parameters(path: str) -> dict[str, dict]: + return { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + } + + parameters = query_parameters("/v2/parameters") + values = query_parameters("/v2/parameter-values") + assert set(parameters) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "search", + "name", + } + assert parameters["name"]["required"] is False + assert set(values) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "parameter_id", + "parameter_name", + "current", + } + assert values["parameter_name"]["required"] is False + + +def test_whole_version_responses_are_read_once_per_process_and_version() -> None: + results, _ids = _resource_results() + readers: list[ResourceReader] = [] + + def factory() -> ResourceReader: + readers.append(ResourceReader(results)) + return readers[-1] + + client = _client(factory, response_cache=MetadataResponseCache()) + paths = [ + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3", + "/v2/model-catalog/variables?country_id=uk", + ] + + first_responses = [client.get(path) for path in paths] + repeated_responses = [client.get(path) for path in paths] + + # The explicit request names the running version (4.20.3), so it shares + # the response the request without a version read: three reads in all. + assert len(readers) == len(paths) - 1 + assert first_responses[2].content == first_responses[1].content + for first, repeated in zip(first_responses, repeated_responses, strict=True): + assert repeated.status_code == 200 + assert repeated.content == first.content + assert repeated.headers["etag"] == first.headers["etag"] + assert repeated.headers["cache-control"] == first.headers["cache-control"] + assert all(reader.closed for reader in readers) + + +def test_failed_whole_version_reads_are_retried() -> None: + results, _ids = _resource_results() + readers = [ + ResourceReader(results, error=MetadataCatalogUnavailableError("down")), + ResourceReader(results), + ] + client = _client(lambda: readers.pop(0), response_cache=MetadataResponseCache()) + + failed = client.get("/v2/model-catalog/variables?country_id=us") + recovered = client.get("/v2/model-catalog/variables?country_id=us") + + assert failed.status_code == 503 + assert "etag" not in failed.headers + assert recovered.status_code == 200 + assert readers == [] + + +def test_default_dependencies_share_one_response_cache() -> None: + first = route_dependencies.NativeRouteDependencies.defaults() + second = route_dependencies.NativeRouteDependencies.defaults() + + assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) + assert first.v2_metadata_response_cache is second.v2_metadata_response_cache + + +@pytest.mark.parametrize( + ("now", "query", "expected_cache_control"), + [ + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=true", + "public, max-age=3600", + ), + ( + datetime(2026, 12, 31, 23, 58, tzinfo=UTC), + "¤t=true", + "public, max-age=120", + ), + ( + datetime(2026, 6, 1, 12, 0, tzinfo=UTC), + "¤t=true", + "public, max-age=300", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=false", + "public, max-age=86400", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3", + "public, max-age=86400", + ), + ], +) +def test_current_parameter_values_are_reused_only_until_the_next_utc_day( + monkeypatch: pytest.MonkeyPatch, + now: datetime, + query: str, + expected_cache_control: str, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + monkeypatch.setattr(common, "utc_now", lambda: now) + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + f"/v2/parameter-values?country_id=us{query}" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == expected_cache_control + + +def test_a_kept_response_is_served_without_hashing_its_body_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common, response_cache + + hashed_bodies: list[int] = [] + original_entity_tag = response_cache.body_entity_tag + + def counting_entity_tag(body: bytes) -> str: + hashed_bodies.append(len(body)) + return original_entity_tag(body) + + monkeypatch.setattr(response_cache, "body_entity_tag", counting_entity_tag) + monkeypatch.setattr(common, "body_entity_tag", counting_entity_tag) + results, _ids = _resource_results() + client = _client( + lambda: ResourceReader(results), response_cache=MetadataResponseCache() + ) + + first = client.get("/v2/model-catalog/variables?country_id=us") + hashed_after_first = len(hashed_bodies) + second = client.get( + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert hashed_after_first == 1 + assert len(hashed_bodies) == 1 + assert second.headers["etag"] == first.headers["etag"] + assert second.content == first.content + + +def _query_rejections(path: str) -> list[str]: + separator = "&" if "?" in path else "?" + rejections = [ + f"{path}{separator}policyengine_version=not%20a%20version", + f"{path}{separator}unexpected=1", + f"{path}{separator}policyengine_version=4.20.3&policyengine_version=4.20.3", + ] + if "country_id=us" in path: + rejections.append(path.replace("country_id=us", "country_id=ca")) + rejections.append( + path.replace("country_id=us&", "").replace("?country_id=us", "") + ) + return rejections + + +def test_every_metadata_route_rejects_invalid_queries_before_reading() -> None: + _results, ids = _resource_results() + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + client = _client(factory) + for path in _every_resource_path(ids): + for rejected in _query_rejections(path): + response = client.get(rejected) + assert response.status_code == 422, (rejected, response.text) + assert response.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert calls == [] + + +def test_every_metadata_route_declares_the_shared_catalog_query() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + parameters = { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + if parameter["in"] == "query" + } + version = parameters["policyengine_version"] + assert version["required"] is False, path + assert version["schema"]["anyOf"][0]["maxLength"] == 128, path + if path != "/v2/tax-benefit-models/by-country/{country_id}": + country = parameters["country_id"] + assert country["required"] is True, path + assert country["schema"]["enum"] == ["us", "uk"], path + + +def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + "/v2/model-catalog?country_id=us", + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert "etag" in exposed_headers + + +def test_current_values_read_just_before_midnight_expire_at_that_midnight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The read and the response headers must use the same day: values read for + # 31 December may not be reused into 1 January, even when the headers are + # written after midnight. + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant = datetime(2026, 12, 31, 23, 59, 59, tzinfo=UTC) + instants = iter([read_instant, datetime(2027, 1, 1, 0, 0, 1, tzinfo=UTC)]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3¤t=true" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=0" + _name, _args, kwargs = reader.calls[0] + assert kwargs["value_filter"].now == read_instant + + +def _vary_values(response) -> set[str]: + return { + value.strip().lower() + for value in response.headers.get("vary", "").split(",") + if value.strip() + } + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_metadata_responses_vary_by_origin(origin: str | None) -> None: + # The API reflects any origin, so a shared cache must not reuse a response + # stored for a request without Origin (no Access-Control-Allow-Origin) for + # a cross-origin one, or the reverse. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {} if origin is None else {"Origin": origin} + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + large = client.get("/v2/model-catalog?country_id=us", headers=headers) + revalidated = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": large.headers["etag"]}, + ) + + assert revalidated.status_code == 304 + for response in (small, large, revalidated): + assert "origin" in _vary_values(response) + assert "accept-encoding" in _vary_values(large) + + +def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get("/v2/variables?country_id=us") + + assert response.status_code == 404 + assert "origin" not in _vary_values(response) + + +def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + responses = schema["paths"][path]["get"]["responses"] + assert set(responses["200"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "$ref" in responses["200"]["content"]["application/json"]["schema"] + assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "content" not in responses["304"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + assert "path" in responses["400"]["description"] + else: + # Query parameters are validated first (422), so only a country + # given in the path can return 400. + assert "400" not in responses, path + + +def _vary_value_counts(response) -> dict[str, int]: + counts: dict[str, int] = {} + for value in response.headers.get("vary", "").split(","): + if value.strip(): + counts[value.strip().lower()] = counts.get(value.strip().lower(), 0) + 1 + return counts + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( + origin: str | None, +) -> None: + # Through the whole middleware stack (gzip, then CORS): a 304 must carry + # the Vary its 200 would, and each value appears once. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {"Accept-Encoding": "gzip"} + if origin is not None: + headers["Origin"] = origin + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + compressed = client.get("/v2/model-catalog?country_id=us", headers=headers) + not_modified = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": compressed.headers["etag"]}, + ) + + assert compressed.headers.get("content-encoding") == "gzip" + assert not_modified.status_code == 304 + expected = {"origin": 1, "accept-encoding": 1} + assert _vary_value_counts(small) == expected + assert _vary_value_counts(compressed) == expected + assert _vary_value_counts(not_modified) == expected + + +def test_openapi_declares_each_variables_unit_and_definition_period() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + variable_properties = schema["components"]["schemas"]["MetadataVariable"][ + "properties" + ] + for field in ("unit", "definition_period"): + assert {"type": "string"} in variable_properties[field]["anyOf"] + assert {"type": "null"} in variable_properties[field]["anyOf"] + + +def test_openapi_requires_every_catalog_format_2_field_as_nullable() -> None: + # Every response includes these fields (null for versions published before + # catalog format 2), so a generated client must not type them as missing. + results, _ids = _resource_results() + schemas = ( + _client(lambda: ResourceReader(results)) + .get("/v2/openapi.json") + .json()["components"]["schemas"] + ) + format_2_fields = { + "MetadataVariable": { + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + }, + "MetadataParameterSummary": {"economy", "household"}, + "MetadataCatalogParameter": {"economy", "household"}, + } + + for schema_name, fields in format_2_fields.items(): + schema = schemas[schema_name] + assert set(schema["properties"]) <= set(schema["required"]), schema_name + for field in fields: + assert {"type": "null"} in schema["properties"][field]["anyOf"], field diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 1f80d1b3f..b2f35d7be 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -33,8 +33,20 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.services import V2MetadataService -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.transformations import ancestor_paths +from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, + MetadataDatedValue, + MetadataEntity, + MetadataEnumValue, + ParameterFilter, + ParameterNodeLabelRow, + ParameterValueFilter, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -74,6 +86,16 @@ def _insert_country( if time_periods is None else time_periods ), + catalog_format=country.model_version.catalog_format, + entities=[ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ], + basic_inputs=list(country.model_version.basic_inputs), ) ) session.add_all( @@ -89,6 +111,21 @@ def _insert_country( default_value=record.default_value, adds=record.adds, subtracts=record.subtracts, + is_input_variable=record.is_input_variable, + hidden_input=record.hidden_input, + module_name=record.module_name, + adds_parameter_path=record.adds_parameter_path, + subtracts_parameter_path=record.subtracts_parameter_path, + enum_values=( + None + if record.enum_values is None + else [ + {"value": enum_value.value, "label": enum_value.label} + for enum_value in record.enum_values + ] + ), + unit=record.unit, + definition_period=record.definition_period, ) for record in country.variables ) @@ -111,6 +148,8 @@ def _insert_country( description=record.description, data_type=record.data_type, unit=record.unit, + economy=record.economy, + household=record.household, ) for record in country.parameters ) @@ -196,28 +235,33 @@ def _add_country_version( session.commit() -def _us_model_version(session: Session) -> TaxBenefitModelVersion: +def _model_version( + session: Session, model_name: str, version: str = POLICYENGINE_VERSION +) -> TaxBenefitModelVersion: return session.exec( select(TaxBenefitModelVersion) .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) .where( - TaxBenefitModel.name == "policyengine-us", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + TaxBenefitModel.name == model_name, + TaxBenefitModelVersion.version == version, ) ).one() -def _service(session: Session) -> V2MetadataService: +def _service( + session: Session, *, selected_catalogs: SelectedCatalogStore | None = None +) -> V2MetadataService: return V2MetadataService( MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, ) def test_resource_collection_uses_bounded_pagination_without_counting( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") catalog_session.add( Variable( id=uuid4(), @@ -309,7 +353,7 @@ def record_statement(_connection, _cursor, statement, *_args) -> None: def test_parameter_values_are_separate_canonical_resources( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -370,7 +414,7 @@ def test_current_parameter_value_uses_inclusive_effective_dates( def test_parameter_children_are_loaded_one_level_at_a_time( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") nested_node = ParameterNode( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -421,7 +465,9 @@ def test_resource_filters_and_details_remain_inside_selected_catalog( ) -> None: service = _service(catalog_session) variables = service.list_variables("us", search="employment") - parameters = service.list_parameters("us", search="example rate") + parameters = service.list_parameters( + "us", parameter_filter=ParameterFilter(search="example rate") + ) states = service.list_regions("us", region_type="state") region = service.get_region_by_code("us", "state/ca") @@ -503,7 +549,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( def test_dataset_collection_excludes_outputs_and_storage_references( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") output_id = uuid4() catalog_session.add( Dataset( @@ -560,6 +606,681 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") +def _recorded_statements(session: Session, read): + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement.lower()) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record_statement) + try: + result = read() + finally: + event.remove(bind, "before_cursor_execute", record_statement) + return result, statements + + +def test_model_catalog_holds_only_the_small_parts_of_a_catalog_version( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, lambda: _service(catalog_session).get_model_catalog("us") + ) + us_catalog = normalized_catalog().country("us") + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert result.model.name == "policyengine-us" + assert ( + result.model_version.current_law_id == us_catalog.model_version.current_law_id + ) + assert result.entities == [ + MetadataEntity(key="person", plural="people", is_person=True), + MetadataEntity(key="household", plural="households", is_person=False), + ] + assert result.basic_inputs == ["employment_income"] + assert [region.code for region in result.regions] == sorted( + region.code for region in us_catalog.regions + ) + assert [dataset.name for dataset in result.datasets] == sorted( + dataset.name for dataset in us_catalog.datasets + ) + assert [option.name for option in result.economy_options.region] == [ + region.code for region in result.regions + ] + assert [option.name for option in result.economy_options.time_period] == list( + us_catalog.model_version.metadata_time_periods + ) + assert [option.label for option in result.economy_options.datasets] == ["Microcosm"] + assert len(statements) == 3 + for table in ("variables", "parameters", "parameter_values", "parameter_nodes"): + assert all(f"from {table}" not in statement for statement in statements) + + +def test_model_catalog_without_regions_offers_no_economy_calculation( + catalog_session: Session, +) -> None: + # A household-only country publishes no regions or datasets; its catalog + # still serves entities and basic inputs, with empty economy choices. + model_version = _model_version(catalog_session, "policyengine-us") + for region in catalog_session.exec( + select(Region).where(Region.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(region) + catalog_session.flush() + for dataset in catalog_session.exec( + select(Dataset).where(Dataset.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(dataset) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.regions == [] + assert result.datasets == [] + assert result.economy_options.region == [] + assert result.economy_options.time_period == [] + assert result.economy_options.datasets == [] + assert result.basic_inputs == ["employment_income"] + + +def test_model_catalog_published_before_format_2_has_empty_new_fields( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + model_version.catalog_format = None + model_version.entities = None + model_version.basic_inputs = None + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.catalog_format is None + assert result.entities == [] + assert result.basic_inputs == [] + + +def test_model_catalog_with_regions_requires_the_national_region( + catalog_session: Session, +) -> None: + national_region = catalog_session.exec( + select(Region).where(Region.code == "us") + ).one() + catalog_session.delete(national_region) + catalog_session.commit() + + with pytest.raises(MetadataCatalogUnavailableError, match="national v2 region"): + _service(catalog_session).get_model_catalog("us") + + +def _add_variable_with_parameter_named_parts(session: Session) -> None: + model_version = _model_version(session, "policyengine-us") + parts = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.parts", + label="Income parts", + description=None, + data_type="list", + unit=None, + economy=False, + household=True, + ) + session.add(parts) + session.add_all( + [ + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["reform_only_income"], + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ), + ] + ) + session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="total_income", + label="Total income", + entity="person", + description=None, + data_type="float", + possible_values=["LOW", "HIGH"], + default_value=0, + adds=["employment_income"], + subtracts=None, + is_input_variable=False, + hidden_input=True, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + subtracts_parameter_path="gov.example.unpublished_parts", + enum_values=[ + {"value": "LOW", "label": "Low"}, + {"value": "HIGH", "label": 2}, + ], + ) + ) + session.commit() + + +def test_catalog_variables_hold_every_variable_with_its_format_2_fields( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("us"), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [variable.name for variable in result.variables] == [ + "employment_income", + "total_income", + ] + employment_income, total_income = result.variables + assert employment_income.is_input_variable is True + assert employment_income.hidden_input is False + assert employment_income.module_name == "household.income.employment_income" + assert employment_income.adds_parameter_path is None + assert total_income.hidden_input is True + assert total_income.adds_parameter_path == "gov.example.parts" + assert total_income.subtracts_parameter_path == "gov.example.unpublished_parts" + assert total_income.enum_values == [ + MetadataEnumValue(value="LOW", label="Low"), + MetadataEnumValue(value="HIGH", label=2), + ] + assert len(statements) == 3 + + +def test_catalog_variables_carry_the_dated_values_of_parameter_named_parts( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result = _service(catalog_session).get_catalog_variables("us") + + assert result.part_parameter_values == { + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + value=["employment_income"], + ), + MetadataDatedValue( + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income", "pension_income"], + ), + ], + "gov.example.unpublished_parts": [], + } + + +def test_catalog_variables_without_parameter_named_parts_read_no_values( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("uk"), + ) + + assert [variable.name for variable in result.variables] == ["employment_income"] + assert result.part_parameter_values == {} + assert len(statements) == 2 + assert all("parameter_values" not in statement for statement in statements) + + +def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> None: + variable = _service(catalog_session).list_variables("us").items[0] + + assert variable.is_input_variable is True + assert variable.module_name == "household.income.employment_income" + assert variable.enum_values is None + + +def _add_bracket_schedule(session: Session) -> None: + model_version = _model_version(session, "policyengine-us") + session.add_all( + [ + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates", + label="Example rates", + description=None, + ), + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0]", + label=None, + description=None, + ), + ] + ) + bracket_rate = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0].rate", + label="Rate", + description=None, + data_type="float", + unit="/1", + economy=True, + household=True, + ) + session.add(bracket_rate) + session.add( + ParameterValue( + id=uuid4(), + parameter_id=bracket_rate.id, + value_json=0.2, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ) + session.commit() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("gov", []), + ("gov.example.rate", ["gov", "gov.example"]), + ( + "gov.example.rates[0].rate", + ["gov", "gov.example", "gov.example.rates", "gov.example.rates[0]"], + ), + ("gov.example.rates[10]", ["gov", "gov.example", "gov.example.rates"]), + ], +) +def test_ancestor_paths_treat_a_bracket_index_as_a_child( + name: str, expected: list[str] +) -> None: + assert ancestor_paths(name) == expected + + +def test_named_parameters_load_with_values_and_ancestor_labels_in_one_read( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_parameters( + "us", + names=["gov.example.rates[0].rate", "gov.example.rate", "gov.example.rate"], + ), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [parameter.name for parameter in result.parameters] == [ + "gov.example.rate", + "gov.example.rates[0].rate", + ] + example_rate, bracket_rate = result.parameters + assert (example_rate.economy, example_rate.household) == (True, False) + assert [value.value for value in example_rate.values] == [0.1, 0.2] + assert example_rate.values[0].start_date == datetime(2025, 1, 1, tzinfo=UTC) + assert example_rate.values[0].end_date == datetime(2025, 12, 31, tzinfo=UTC) + assert [value.value for value in bracket_rate.values] == [0.2] + assert result.node_labels == { + "gov.example": "Example policy", + "gov.example.rates": "Example rates", + "gov.example.rates[0]": None, + } + assert result.missing_names == [] + assert len(statements) == 4 + + +def test_named_parameters_list_names_the_version_lacks( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_catalog_parameters( + "uk", names=["gov.example.rate", "gov.removed.amount"] + ) + + assert [parameter.name for parameter in result.parameters] == ["gov.example.rate"] + assert result.missing_names == ["gov.removed.amount"] + + +def test_named_parameters_exclude_policy_and_dynamic_values( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + parameter = catalog_session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version.id, + Parameter.name == "gov.example.rate", + ) + ).one() + catalog_session.add( + ParameterValue( + id=uuid4(), + parameter_id=parameter.id, + value_json=0.9, + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ) + ) + catalog_session.commit() + + result = _service(catalog_session).get_catalog_parameters( + "us", names=["gov.example.rate"] + ) + + assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] + + +def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + service = _service(catalog_session) + + parameters = service.list_parameters( + "us", + parameter_filter=ParameterFilter( + names=("gov.example.rates[0].rate", "gov.removed.amount") + ), + ) + values = service.list_parameter_values( + "us", + value_filter=ParameterValueFilter( + parameter_names=("gov.example.rate", "gov.example.rates[0].rate") + ), + ) + + assert [parameter.name for parameter in parameters.items] == [ + "gov.example.rates[0].rate" + ] + assert parameters.items[0].economy is True + assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] + + +def test_a_kept_catalog_selection_saves_its_query_on_later_reads( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + + first, first_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + second, second_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + + assert second == first + assert len(first_statements) == 2 + assert len(second_statements) == 1 + assert "tax_benefit_model_versions" not in second_statements[0] + + +def test_failed_catalog_selections_are_not_kept(catalog_session: Session) -> None: + store = SelectedCatalogStore() + service = _service(catalog_session, selected_catalogs=store) + + for _attempt in range(2): + with pytest.raises(MetadataCatalogVersionNotFoundError): + service.list_parameters("us", "9.9.9") + with pytest.raises(UnsupportedPreviewCountryError): + service.list_parameters("ca") + with pytest.raises(InvalidPolicyEngineVersionError): + service.list_parameters("us", "not a version") + + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + assert service.list_parameters("us", "9.9.9").policyengine_version == "9.9.9" + + +def test_each_country_and_version_keeps_its_own_selection( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + service = _service(catalog_session, selected_catalogs=store) + + running_us = service.get_model_by_country("us") + named_us = service.get_model_by_country("us", "9.9.9") + running_uk = service.get_model_by_country("uk") + + assert (running_us.policyengine_version, named_us.policyengine_version) == ( + POLICYENGINE_VERSION, + "9.9.9", + ) + assert named_us.model_version.current_law_id == 7 + assert running_uk.model.name == "policyengine-uk" + assert service.get_model_by_country("us", "9.9.9") == named_us + + +def test_value_and_node_label_reads_return_typed_rows( + catalog_session: Session, +) -> None: + from policyengine_api.services.v2.metadata.database_connectors import ( + reads_parameter_nodes, + reads_parameters, + ) + + model_version = _model_version(catalog_session, "policyengine-us") + values = reads_parameters.read_current_law_values_by_parameter_name( + catalog_session, + model_version_id=model_version.id, + parameter_names={"gov.example.rate"}, + ) + labels = reads_parameter_nodes.read_parameter_node_labels( + catalog_session, + model_version_id=model_version.id, + node_names={"gov.example", "gov.absent"}, + ) + + assert [type(row) for row in values] == [CurrentLawValueRow, CurrentLawValueRow] + assert [(row.name, row.value_json) for row in values] == [ + ("gov.example.rate", 0.1), + ("gov.example.rate", 0.2), + ] + assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] + + +def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( + catalog_session: Session, +) -> None: + service = _service(catalog_session) + + leaf = next( + child + for child in service.list_parameter_children( + "us", parent_path="gov.example" + ).items + if child.type == "parameter" + ) + page_item = service.list_parameters( + "us", parameter_filter=ParameterFilter(names=("gov.example.rate",)) + ).items[0] + + assert leaf.parameter == page_item + assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) + + +def test_variables_carry_their_unit_and_definition_period( + catalog_session: Session, +) -> None: + uk_version = _model_version(catalog_session, "policyengine-uk") + catalog_session.add_all( + [ + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ), + # A variable published before catalog format 2 stored neither. + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="age", + entity="person", + data_type="int", + default_value=0, + ), + ] + ) + catalog_session.commit() + service = _service(catalog_session) + + us_variables = service.get_catalog_variables("us").variables + uk_variables = { + variable.name: variable + for variable in service.get_catalog_variables("uk").variables + } + us_page = service.list_variables("us").items + + assert [(item.unit, item.definition_period) for item in us_variables] == [ + ("currency-USD", "year") + ] + assert (us_page[0].unit, us_page[0].definition_period) == ( + "currency-USD", + "year", + ) + assert ( + uk_variables["universal_credit"].unit, + uk_variables["universal_credit"].definition_period, + ) == ("currency-GBP", "month") + assert (uk_variables["age"].unit, uk_variables["age"].definition_period) == ( + None, + None, + ) + + +@pytest.mark.parametrize( + ("country_id", "version", "expected_catalog_format", "expected_fields"), + [ + ("us", None, 2, {"employment_income": ("currency-USD", "year")}), + ( + "uk", + None, + 2, + { + "employment_income": ("currency-USD", "year"), + "universal_credit": ("currency-GBP", "month"), + }, + ), + # A version published before catalog format 2 stored neither field. + ("us", "9.9.9", None, {"employment_income": (None, None)}), + ], +) +def test_both_variables_routes_serve_unit_and_definition_period( + catalog_session: Session, + country_id: str, + version: str | None, + expected_catalog_format: int | None, + expected_fields: dict[str, tuple[str | None, str | None]], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + + from .test_metadata_routes import _client + + catalog_session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=_model_version( + catalog_session, "policyengine-uk" + ).id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ) + ) + catalog_session.commit() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + earlier_version = _model_version(catalog_session, "policyengine-us", "9.9.9") + earlier_version.catalog_format = None + for variable in catalog_session.exec( + select(Variable).where( + Variable.tax_benefit_model_version_id == earlier_version.id + ) + ).all(): + variable.unit = None + variable.definition_period = None + catalog_session.commit() + client = _client( + lambda: _service(catalog_session), response_cache=MetadataResponseCache() + ) + query = f"country_id={country_id}" + ( + "" if version is None else f"&policyengine_version={version}" + ) + + whole_version_bodies = [ + client.get(f"/v2/model-catalog/variables?{query}").json()["result"] + for _attempt in ("read", "stored") + ] + page = client.get(f"/v2/variables?{query}").json()["result"] + + for result in whole_version_bodies: + assert result["catalog_format"] == expected_catalog_format + served = { + variable["name"]: (variable["unit"], variable["definition_period"]) + for variable in result["variables"] + } + assert served == expected_fields + paged = { + item["name"]: (item["unit"], item["definition_period"]) + for item in page["items"] + } + assert paged == expected_fields + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -569,6 +1290,7 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package / "data" / "v2" / "catalog" / "catalog_selection.py", connector_directory / "reads.py", connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_nodes.py", connector_directory / "reads_parameter_tree.py", connector_directory / "reads_parameters.py", connector_directory / "reads_regions.py", @@ -624,6 +1346,9 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region", "get_region_by_code", "get_economy_options", + "get_model_catalog", + "get_catalog_variables", + "get_catalog_parameters", } for method_name in method_names: diff --git a/tests/unit/v2/test_parameter_value_listing_route.py b/tests/unit/v2/test_parameter_value_listing_route.py index 39ac27b72..a07aa4ca5 100644 --- a/tests/unit/v2/test_parameter_value_listing_route.py +++ b/tests/unit/v2/test_parameter_value_listing_route.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import UTC, datetime + import pytest from policyengine_api.services.v2.metadata.types import ParameterValueFilter @@ -9,7 +11,13 @@ from .test_metadata_routes import ResourceReader, _client, _resource_results -def test_listing_forwards_every_query_parameter() -> None: +def test_listing_forwards_every_query_parameter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + request_instant = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) + monkeypatch.setattr(common, "utc_now", lambda: request_instant) results, ids = _resource_results() reader = ResourceReader(results) @@ -32,7 +40,7 @@ def test_listing_forwards_every_query_parameter() -> None: ("uk", "4.19.0"), { "value_filter": ParameterValueFilter( - parameter_id=ids["parameter_id"], current=True + parameter_id=ids["parameter_id"], current=True, now=request_instant ), "offset": 5, "limit": 7,