diff --git a/changelog.d/apiv2-1e-catalog-fields.added.md b/changelog.d/apiv2-1e-catalog-fields.added.md new file mode 100644 index 000000000..7c6359369 --- /dev/null +++ b/changelog.d/apiv2-1e-catalog-fields.added.md @@ -0,0 +1 @@ +The v2 metadata catalog (format 2) stores the presentation metadata the app shows and PolicyEngine.py's public records omit: model entities and basic inputs; variable value types, definition periods, units, modules, input and hidden-input flags and enum labels; parameter metadata labels, periods and economy and household flags; and bracket schedules as parameter nodes. The routes return them, with null values for catalog versions published before this change, which publication never rewrites. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index a0d5a7842..ce9da0287 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -95,6 +95,26 @@ metadata time-period options are stored on the model-version snapshot. PolicyEngine Core and country-package versions are derived diagnostic evidence and are not separately persisted. +Catalog format 2 adds the presentation metadata the app's v3 workspace shows, +which PolicyEngine.py's public records omit: model entities (key, plural, +label, description, whether it is a person, and roles) and basic inputs on the +model version; each variable's value type, definition period, unit, module, +input and hidden-input flags and enum value labels; each parameter's own +metadata label, period, and economy and household flags; and the bracket +schedules (Core `ParameterScale` nodes) as parameter nodes. The extractor reads +them from the country system that PolicyEngine.py built its model from, +through the model version's `_load_system` hook, by attribute and without +importing PolicyEngine Core or a country package. That hook is not part of +PolicyEngine.py's public interface, so a release that removes it stops +extraction with an error; publishing these fields in PolicyEngine.py's public +records would remove the dependency. + +Publication never rewrites a stored version. A version published before +format 2 has `catalog_format` null and null format 2 fields, and later +publications of the same version compare only the fields it was published +with. The first PolicyEngine.py version published after format 2 deploys +carries the new fields; until then the routes return them as null. + Parameter extraction requires the PolicyEngine.py public history contract from 5.0.4 and later: values are ordered from oldest to newest, bounded `end_date` values are inclusive and fall one day before the following `start_date`, and @@ -207,9 +227,10 @@ workspace that load the whole model at start-up; the paged routes would need about 2,550 requests of 500 rows for the US catalog (100,276 `gov` parameters and 1,174,586 values). Each API instance builds the serialized document once per country and catalog version and keeps the four most recently used. Measured -locally against the 6.7.5 catalog, the US document takes 5.8 seconds to build -and is 60.7 MB (12.1 MB gzipped); the UK document takes 1.0 second and is -1.3 MB. The response carries an ETag naming the country, catalog version and +locally against the 6.7.5 catalog in format 2, the US document takes 6.8 +seconds to build and is 70.1 MB (12.5 MB gzipped), with the building process +peaking at 544 MB resident; the UK document takes 1.0 second and is 1.7 MB. +The v1 `GET /us/metadata` document for the same models is 74.7 MB. The response carries an ETag naming the country, catalog version and document format, and `Cache-Control: public, max-age=86400` when the request names `policyengine_version` (`max-age=300` otherwise, because the running version changes on deploy). The release workflow checks the UK document, diff --git a/migrations/v2/versions/72fab86fa97d_store_catalog_presentation_metadata.py b/migrations/v2/versions/72fab86fa97d_store_catalog_presentation_metadata.py new file mode 100644 index 000000000..6e208b9b0 --- /dev/null +++ b/migrations/v2/versions/72fab86fa97d_store_catalog_presentation_metadata.py @@ -0,0 +1,99 @@ +"""store catalog presentation metadata + +Revision ID: 72fab86fa97d +Revises: fbb7488ebe8a +Create Date: 2026-10-20 02:03:28.633166 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "72fab86fa97d" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "parameters", + sa.Column( + "metadata_label", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + ) + op.add_column( + "parameters", + sa.Column("period", sqlmodel.sql.sqltypes.AutoString(length=32), nullable=True), + ) + op.add_column("parameters", sa.Column("economy", sa.Boolean(), nullable=True)) + op.add_column("parameters", sa.Column("household", sa.Boolean(), nullable=True)) + op.add_column( + "tax_benefit_model_versions", + sa.Column("catalog_format", sa.Integer(), nullable=True), + ) + op.add_column( + "tax_benefit_model_versions", sa.Column("entities", sa.JSON(), nullable=True) + ) + op.add_column( + "tax_benefit_model_versions", + sa.Column("basic_inputs", sa.JSON(), nullable=True), + ) + op.add_column( + "variables", + sa.Column( + "value_type", sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True + ), + ) + op.add_column( + "variables", + sa.Column( + "definition_period", + sqlmodel.sql.sqltypes.AutoString(length=32), + nullable=True, + ), + ) + op.add_column( + "variables", + sa.Column("unit", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + ) + op.add_column( + "variables", + sa.Column( + "module_name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True + ), + ) + op.add_column( + "variables", sa.Column("is_input_variable", sa.Boolean(), nullable=True) + ) + op.add_column("variables", sa.Column("hidden_input", sa.Boolean(), nullable=True)) + op.add_column( + "variables", sa.Column("possible_value_labels", sa.JSON(), nullable=True) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("variables", "possible_value_labels") + op.drop_column("variables", "hidden_input") + op.drop_column("variables", "is_input_variable") + op.drop_column("variables", "module_name") + op.drop_column("variables", "unit") + op.drop_column("variables", "definition_period") + op.drop_column("variables", "value_type") + op.drop_column("tax_benefit_model_versions", "basic_inputs") + op.drop_column("tax_benefit_model_versions", "entities") + op.drop_column("tax_benefit_model_versions", "catalog_format") + op.drop_column("parameters", "household") + op.drop_column("parameters", "economy") + op.drop_column("parameters", "period") + op.drop_column("parameters", "metadata_label") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 4111c966a..4b707223e 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -15,6 +15,7 @@ from uuid import NAMESPACE_URL, UUID, uuid5 from policyengine_api.data.v2.catalog.records import ( + CATALOG_FORMAT, CountryCatalog, DatasetRecord, FallbackSummary, @@ -268,10 +269,156 @@ def _verify_bundle( return tuple(observed_dependencies) +def _country_system(source: object, *, country_id: str) -> object: + """Return the PolicyEngine Core system the public model was built from. + + PolicyEngine.py's public records omit presentation metadata the app shows: + entities, basic inputs, variable modules, input and hidden flags, enum + labels, parameter metadata (label, period, economy and household flags) + and bracket schedules. The model version keeps its country system behind + its ``_load_system`` hook, so the extractor reads that metadata through + PolicyEngine.py, by attribute, without importing Core or a country package. + """ + load_system = getattr(source, "_load_system", None) + if not callable(load_system): + msg = f"{country_id} public model does not expose its country system" + raise CatalogExtractionError(msg) + return load_system() + + +def _is_bracket_schedule(core_node: object) -> bool: + return any( + ancestor.__name__ == "ParameterScale" for ancestor in type(core_node).__mro__ + ) + + +def _core_parameter_records(system: object) -> dict[str, object]: + parameters = getattr(system, "parameters", None) + get_descendants = getattr(parameters, "get_descendants", None) + if not callable(get_descendants): + msg = "the country system has no parameter tree" + raise CatalogExtractionError(msg) + return { + name: record + for record in get_descendants() + if isinstance(name := getattr(record, "name", None), str) + } + + +def _core_record(records: Mapping[str, object], name: str, *, kind: str) -> object: + record = records.get(name) + if record is None: + msg = f"{kind} {name!r} is absent from the country system" + raise CatalogExtractionError(msg) + return record + + +def _core_metadata(core_record: object) -> Mapping[str, Any]: + metadata = getattr(core_record, "metadata", None) + return metadata if isinstance(metadata, Mapping) else {} + + +def _optional_flag(value: object, *, field_name: str, default: bool) -> bool: + if value is None: + return default + if not isinstance(value, bool): + msg = f"{field_name} must be a boolean" + raise CatalogExtractionError(msg) + return value + + +def _possible_value_labels( + core_variable: object, *, name: str, possible_values: list[str] | None +) -> dict[str, str] | None: + enum_type = getattr(core_variable, "possible_values", None) + if possible_values is None or enum_type is None: + return None + labels = { + _required_text( + getattr(member, "name", None), + field_name=f"variable {name} possible value", + maximum=255, + ): _required_text( + str(getattr(member, "value", "")), + field_name=f"variable {name} possible value label", + maximum=512, + ) + for member in enum_type + } + if set(labels) != set(possible_values): + msg = f"variable {name!r} enum labels do not match its possible values" + raise CatalogExtractionError(msg) + return {value: labels[value] for value in possible_values} + + +def _normalize_entities(system: object) -> tuple[dict[str, Any], ...]: + def description(record: object, field_name: str) -> str | None: + text = _optional_text(getattr(record, "doc", None), field_name=field_name) + return text.strip() or None if text is not None else None + + entities: list[dict[str, Any]] = [] + for entity in getattr(system, "entities", None) or (): + key = _required_text( + getattr(entity, "key", None), field_name="entity key", maximum=128 + ) + entities.append( + { + "key": key, + "plural": _optional_text( + getattr(entity, "plural", None), + field_name=f"entity {key} plural", + maximum=128, + ), + "label": _optional_text( + getattr(entity, "label", None), + field_name=f"entity {key} label", + maximum=255, + ), + "description": description(entity, f"entity {key} description"), + "is_person": bool(getattr(entity, "is_person", False)), + "roles": [ + { + "key": _required_text( + getattr(role, "key", None), + field_name=f"entity {key} role key", + maximum=128, + ), + "plural": _optional_text( + getattr(role, "plural", None), + field_name=f"entity {key} role plural", + maximum=128, + ), + "label": _optional_text( + getattr(role, "label", None), + field_name=f"entity {key} role label", + maximum=255, + ), + "description": description( + role, f"entity {key} role description" + ), + } + for role in getattr(entity, "roles", None) or () + ], + } + ) + if not entities: + msg = "the country system has no entities" + raise CatalogExtractionError(msg) + return tuple(entities) + + +def _normalize_basic_inputs(system: object) -> tuple[str, ...]: + return tuple( + _required_text(name, field_name="basic input", maximum=512) + for name in getattr(system, "basic_inputs", None) or () + ) + + def _normalize_variable( source: object, *, model_version_id: UUID, + core_variable: object, ) -> VariableRecord: name = _required_text( getattr(source, "name", None), field_name="variable name", maximum=512 @@ -331,13 +478,48 @@ def optional_names(attribute: str) -> list[str] | None: default_value=normalize_json_value(getattr(source, "default_value", None)), adds=optional_names("adds"), subtracts=optional_names("subtracts"), + value_type=_type_name(getattr(source, "value_type", None)), + definition_period=_optional_text( + getattr(source, "definition_period", None), + field_name=f"variable {name} definition_period", + maximum=32, + ), + unit=_optional_text( + getattr(source, "unit", None), + field_name=f"variable {name} unit", + maximum=128, + ), + module_name=_optional_text( + getattr(core_variable, "module_name", None), + field_name=f"variable {name} module_name", + maximum=512, + ), + is_input_variable=_optional_flag( + _call_flag(core_variable, "is_input_variable"), + field_name=f"variable {name} is_input_variable", + default=False, + ), + hidden_input=_optional_flag( + getattr(core_variable, "hidden_input", None), + field_name=f"variable {name} hidden_input", + default=False, + ), + possible_value_labels=_possible_value_labels( + core_variable, name=name, possible_values=normalized_possible_values + ), ) +def _call_flag(record: object, method_name: str) -> object: + method = getattr(record, method_name, None) + return method() if callable(method) else None + + def _normalize_parameter( source: object, *, model_version_id: UUID, + core_parameter: object, ) -> ParameterRecord: name = _required_text( getattr(source, "name", None), @@ -345,6 +527,7 @@ def _normalize_parameter( maximum=512, ) parameter_id = _identifier("parameter", model_version_id, name) + metadata = _core_metadata(core_parameter) source_values: list[tuple[datetime, datetime | None, Any]] = [] values_by_start: dict[datetime, tuple[datetime | None, Any]] = {} previous_start: datetime | None = None @@ -424,6 +607,26 @@ def _normalize_parameter( maximum=128, ), values=tuple(normalized_values), + metadata_label=_optional_text( + metadata.get("label"), + field_name=f"parameter {name} metadata label", + maximum=512, + ), + period=_optional_text( + metadata.get("period"), + field_name=f"parameter {name} period", + maximum=32, + ), + economy=_optional_flag( + metadata.get("economy"), + field_name=f"parameter {name} economy", + default=True, + ), + household=_optional_flag( + metadata.get("household"), + field_name=f"parameter {name} household", + default=True, + ), ) @@ -491,8 +694,20 @@ def _normalize_country( ) raise CatalogExtractionError(msg) + system = _country_system(source, country_id=country_id) + core_variables = getattr(system, "variables", None) + if not isinstance(core_variables, Mapping): + msg = f"{country_id} country system has no variable mapping" + raise CatalogExtractionError(msg) + core_parameters = _core_parameter_records(system) variables = tuple( - _normalize_variable(variable, model_version_id=model_version_id) + _normalize_variable( + variable, + model_version_id=model_version_id, + core_variable=_core_record( + core_variables, str(getattr(variable, "name", "")), kind="variable" + ), + ) for variable in _public_named_records( source, attribute="variables_by_name", @@ -531,15 +746,47 @@ def _normalize_country( ), ) ) - parameter_nodes = tuple(normalized_nodes) parameters = tuple( - _normalize_parameter(parameter, model_version_id=model_version_id) + _normalize_parameter( + parameter, + model_version_id=model_version_id, + core_parameter=_core_record( + core_parameters, str(getattr(parameter, "name", "")), kind="parameter" + ), + ) for parameter in _public_named_records( source, attribute="parameters_by_name", entity_name="parameter", ) ) + # PolicyEngine.py omits bracket schedules (Core ParameterScale nodes), + # whose labels name the schedules the app shows. + public_node_names = {node.name for node in normalized_nodes} + for schedule_name, core_node in sorted(core_parameters.items()): + if schedule_name in public_node_names or not _is_bracket_schedule(core_node): + continue + schedule_name = _required_text( + schedule_name, field_name="bracket schedule name", maximum=512 + ) + normalized_nodes.append( + ParameterNodeRecord( + id=_identifier("parameter-node", model_version_id, schedule_name), + model_version_id=model_version_id, + name=schedule_name, + label=_optional_text( + _core_metadata(core_node).get("label"), + field_name="bracket schedule label", + maximum=512, + ), + description=_optional_text( + getattr(core_node, "description", None), + field_name="bracket schedule description", + ), + is_scale=True, + ) + ) + parameter_nodes = tuple(normalized_nodes) reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] manifest_default = getattr(release_manifest, "default_dataset", None) @@ -693,6 +940,9 @@ def _normalize_country( ), current_law_id=CURRENT_LAW_IDS[country_id], metadata_time_periods=METADATA_TIME_PERIODS[country_id], + catalog_format=CATALOG_FORMAT, + entities=_normalize_entities(system), + basic_inputs=_normalize_basic_inputs(system), ), variables=variables, parameter_nodes=parameter_nodes, diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 98872c630..fa845e5a6 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -76,9 +76,42 @@ def _versioned_table_from(table: sa.Table) -> sa.Join: ) +def stored_catalog_format( + connection: Connection, country: CountryCatalog +) -> int | None: + """Return the catalog format of this country's stored version. + + None means the version was published before catalog format 2, so it has no + presentation metadata and no bracket-schedule nodes. + """ + statement = ( + sa.select(MODEL_VERSIONS.c.catalog_format) + .select_from( + MODEL_VERSIONS.join( + MODELS, + MODELS.c.id == MODEL_VERSIONS.c.model_id, + ) + ) + .where( + MODELS.c.name == country.model.name, + MODEL_VERSIONS.c.version == country.model_version.version, + ) + ) + return connection.execute(statement).scalar_one_or_none() + + def _comparison_pairs( country: CountryCatalog, + *, + stored_format: int | None, ) -> tuple[tuple[Select, Select], ...]: + """Pair each staged record set with the stored one for set comparison. + + A version published before catalog format 2 is compared on the fields it + was published with: its presentation fields are null and it has no + bracket-schedule nodes, and publication never rewrites a stored version. + """ + has_presentation = stored_format is not None staged_country = STAGE_CATALOG_MODELS.c.country_id == country.country_id selected_version = sa.and_( MODELS.c.name == country.model.name, @@ -89,12 +122,25 @@ def _comparison_pairs( STAGE_CATALOG_MODELS.c.version_description, STAGE_CATALOG_MODELS.c.current_law_id, STAGE_CATALOG_MODELS.c.metadata_time_periods, + *( + (STAGE_CATALOG_MODELS.c.entities, STAGE_CATALOG_MODELS.c.basic_inputs) + if has_presentation + else () + ), ).where(staged_country) actual_model_version = ( sa.select( MODEL_VERSIONS.c.description, MODEL_VERSIONS.c.current_law_id, sa.cast(MODEL_VERSIONS.c.metadata_time_periods, JSONB), + *( + ( + sa.cast(MODEL_VERSIONS.c.entities, JSONB), + sa.cast(MODEL_VERSIONS.c.basic_inputs, JSONB), + ) + if has_presentation + else () + ), ) .select_from( MODEL_VERSIONS.join( @@ -115,6 +161,19 @@ def _comparison_pairs( STAGE_CATALOG_VARIABLES.c.default_value, STAGE_CATALOG_VARIABLES.c.adds, STAGE_CATALOG_VARIABLES.c.subtracts, + *( + ( + STAGE_CATALOG_VARIABLES.c.value_type, + STAGE_CATALOG_VARIABLES.c.definition_period, + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + STAGE_CATALOG_VARIABLES.c.possible_value_labels, + ) + if has_presentation + else () + ), ).where(STAGE_CATALOG_VARIABLES.c.country_id == country.country_id) actual_variables = ( sa.select( @@ -127,6 +186,19 @@ def _comparison_pairs( sa.cast(VARIABLES.c.default_value, JSONB), sa.cast(VARIABLES.c.adds, JSONB), sa.cast(VARIABLES.c.subtracts, JSONB), + *( + ( + VARIABLES.c.value_type, + VARIABLES.c.definition_period, + VARIABLES.c.unit, + VARIABLES.c.module_name, + VARIABLES.c.is_input_variable, + VARIABLES.c.hidden_input, + sa.cast(VARIABLES.c.possible_value_labels, JSONB), + ) + if has_presentation + else () + ), ) .select_from(_versioned_table_from(VARIABLES)) .where(selected_version) @@ -137,6 +209,10 @@ def _comparison_pairs( STAGE_CATALOG_PARAMETER_NODES.c.label, STAGE_CATALOG_PARAMETER_NODES.c.description, ).where(STAGE_CATALOG_PARAMETER_NODES.c.country_id == country.country_id) + if not has_presentation: + staged_parameter_nodes = staged_parameter_nodes.where( + STAGE_CATALOG_PARAMETER_NODES.c.is_scale.is_(False) + ) actual_parameter_nodes = ( sa.select( PARAMETER_NODES.c.name, @@ -153,6 +229,16 @@ def _comparison_pairs( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + *( + ( + STAGE_CATALOG_PARAMETERS.c.metadata_label, + STAGE_CATALOG_PARAMETERS.c.period, + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, + ) + if has_presentation + else () + ), ).where(STAGE_CATALOG_PARAMETERS.c.country_id == country.country_id) actual_parameters = ( sa.select( @@ -161,6 +247,16 @@ def _comparison_pairs( PARAMETERS.c.description, PARAMETERS.c.data_type, PARAMETERS.c.unit, + *( + ( + PARAMETERS.c.metadata_label, + PARAMETERS.c.period, + PARAMETERS.c.economy, + PARAMETERS.c.household, + ) + if has_presentation + else () + ), ) .select_from(_versioned_table_from(PARAMETERS)) .where(selected_version) @@ -288,9 +384,11 @@ def assert_country_matches( """Raise CatalogPublicationError if the stored version differs from the staged one. Rows are compared as sets in both directions, so a missing row and an extra - row both fail. + row both fail. A version published before catalog format 2 is compared on + the fields it was published with. """ - for staged, actual in _comparison_pairs(country): + stored_format = stored_catalog_format(connection, country) + for staged, actual in _comparison_pairs(country, stored_format=stored_format): if _sets_differ(connection, staged, actual): msg = f"persisted {country.country_id} catalog differs from PolicyEngine.py" raise CatalogPublicationError(msg) @@ -320,6 +418,9 @@ def assert_country_matches( "description", "current_law_id", "metadata_time_periods", + "catalog_format", + "entities", + "basic_inputs", ], sa.select( STAGE_CATALOG_MODELS.c.version_id, @@ -332,6 +433,12 @@ def assert_country_matches( STAGE_CATALOG_MODELS.c.metadata_time_periods, MODEL_VERSIONS.c.metadata_time_periods.type, ), + STAGE_CATALOG_MODELS.c.catalog_format, + sa.cast(STAGE_CATALOG_MODELS.c.entities, MODEL_VERSIONS.c.entities.type), + sa.cast( + STAGE_CATALOG_MODELS.c.basic_inputs, + MODEL_VERSIONS.c.basic_inputs.type, + ), ) .select_from( STAGE_CATALOG_MODELS.join( @@ -395,6 +502,13 @@ def assert_country_matches( "default_value", "adds", "subtracts", + "value_type", + "definition_period", + "unit", + "module_name", + "is_input_variable", + "hidden_input", + "possible_value_labels", "tax_benefit_model_version_id", ], sa.select( @@ -418,6 +532,16 @@ def assert_country_matches( STAGE_CATALOG_VARIABLES.c.subtracts, VARIABLES.c.subtracts.type, ), + STAGE_CATALOG_VARIABLES.c.value_type, + STAGE_CATALOG_VARIABLES.c.definition_period, + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + sa.cast( + STAGE_CATALOG_VARIABLES.c.possible_value_labels, + VARIABLES.c.possible_value_labels.type, + ), MODEL_VERSIONS.c.id, ) .select_from( @@ -481,6 +605,10 @@ def assert_country_matches( "description", "data_type", "unit", + "metadata_label", + "period", + "economy", + "household", "tax_benefit_model_version_id", ], sa.select( @@ -491,6 +619,10 @@ def assert_country_matches( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + STAGE_CATALOG_PARAMETERS.c.metadata_label, + STAGE_CATALOG_PARAMETERS.c.period, + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, MODEL_VERSIONS.c.id, ) .select_from( diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index cd9b4675d..d4f397f25 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -38,6 +38,9 @@ sa.Column("version_description", sa.Text), sa.Column("current_law_id", sa.Integer, nullable=False), sa.Column("metadata_time_periods", JSONB, nullable=False), + sa.Column("catalog_format", sa.Integer, nullable=False), + sa.Column("entities", JSONB, nullable=False), + sa.Column("basic_inputs", JSONB, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -56,6 +59,13 @@ sa.Column("default_value", JSONB, nullable=False), sa.Column("adds", JSONB), sa.Column("subtracts", JSONB), + sa.Column("value_type", sa.Text), + sa.Column("definition_period", sa.Text), + sa.Column("unit", sa.Text), + sa.Column("module_name", sa.Text), + sa.Column("is_input_variable", sa.Boolean), + sa.Column("hidden_input", sa.Boolean), + sa.Column("possible_value_labels", JSONB), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -68,6 +78,7 @@ sa.Column("name", sa.Text, primary_key=True), sa.Column("label", sa.Text), sa.Column("description", sa.Text), + sa.Column("is_scale", sa.Boolean, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -82,6 +93,10 @@ sa.Column("description", sa.Text), sa.Column("data_type", sa.Text), sa.Column("unit", sa.Text), + sa.Column("metadata_label", sa.Text), + sa.Column("period", sa.Text), + sa.Column("economy", sa.Boolean), + sa.Column("household", sa.Boolean), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -166,6 +181,9 @@ def model_rows() -> Iterator[tuple[object, ...]]: country.model_version.description, country.model_version.current_law_id, Jsonb(country.model_version.metadata_time_periods), + country.model_version.catalog_format, + Jsonb(list(country.model_version.entities)), + Jsonb(list(country.model_version.basic_inputs)), ) def variable_rows() -> Iterator[tuple[object, ...]]: @@ -183,6 +201,13 @@ def variable_rows() -> Iterator[tuple[object, ...]]: Jsonb(record.default_value), _optional_json(record.adds), _optional_json(record.subtracts), + record.value_type, + record.definition_period, + record.unit, + record.module_name, + record.is_input_variable, + record.hidden_input, + _optional_json(record.possible_value_labels), ) def parameter_node_rows() -> Iterator[tuple[object, ...]]: @@ -197,6 +222,7 @@ def parameter_node_rows() -> Iterator[tuple[object, ...]]: record.name, record.label, record.description, + record.is_scale, ) def parameter_rows() -> Iterator[tuple[object, ...]]: @@ -210,6 +236,10 @@ def parameter_rows() -> Iterator[tuple[object, ...]]: record.description, record.data_type, record.unit, + record.metadata_label, + record.period, + record.economy, + record.household, ) def parameter_value_rows() -> Iterator[tuple[object, ...]]: diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index afccc4add..20dda4432 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -11,6 +11,17 @@ from uuid import UUID +CATALOG_FORMAT = 2 +"""The catalog content version this extractor publishes. + +Format 2 adds the presentation metadata read through the country system: +entities, basic inputs, variable value types, periods, units, modules, input +flags and enum labels, parameter metadata labels, periods and economy and +household flags, and bracket-schedule nodes. Versions published before it +store null for those fields. +""" + + @dataclass(frozen=True, slots=True) class ModelRecord: """One country's tax-benefit model as named in the catalog.""" @@ -35,6 +46,9 @@ class ModelVersionRecord: description: str | None current_law_id: int metadata_time_periods: tuple[int, ...] + catalog_format: int = CATALOG_FORMAT + entities: tuple[dict[str, Any], ...] = () + basic_inputs: tuple[str, ...] = () @dataclass(frozen=True, slots=True) @@ -52,17 +66,28 @@ class VariableRecord: default_value: Any adds: list[str] | None subtracts: list[str] | None + value_type: str | None = None + definition_period: str | None = None + unit: str | None = None + module_name: str | None = None + is_input_variable: bool | None = None + hidden_input: bool | None = None + possible_value_labels: dict[str, str] | None = None @dataclass(frozen=True, slots=True) class ParameterNodeRecord: - """A non-leaf node of the parameter tree; it groups parameters and has no values.""" + """A non-leaf node of the parameter tree; it groups parameters and has no values. + + is_scale marks a bracket schedule, which only catalog format 2 publishes. + """ id: UUID model_version_id: UUID name: str label: str | None description: str | None + is_scale: bool = False @dataclass(frozen=True, slots=True) @@ -88,6 +113,10 @@ class ParameterRecord: data_type: str | None unit: str | None values: tuple[ParameterValueRecord, ...] + metadata_label: str | None = None + period: str | None = None + economy: bool | None = None + household: bool | None = None @dataclass(frozen=True, slots=True) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..ae25c3068 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "72fab86fa97d" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 8d91864df..6fe2b5838 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -85,6 +85,10 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): description: str | None = None current_law_id: int metadata_time_periods: list[int] = Field(sa_type=sa.JSON) + # Versions published before catalog format 2 leave these null. + catalog_format: int | None = None + entities: list[dict[str, Any]] | None = Field(default=None, sa_type=sa.JSON) + basic_inputs: list[str] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") variables: list["Variable"] = Relationship( @@ -276,6 +280,14 @@ class Variable(IdentifiedModel, table=True): default_value: Any = Field(default=None, sa_type=sa.JSON) adds: list[str] | None = Field(default=None, sa_type=sa.JSON) subtracts: list[str] | None = Field(default=None, sa_type=sa.JSON) + # Catalog format 2 fields; null in versions published before it. + value_type: str | None = Field(default=None, max_length=64) + definition_period: str | None = Field(default=None, max_length=32) + unit: str | None = Field(default=None, max_length=128) + module_name: str | None = Field(default=None, max_length=512) + is_input_variable: bool | None = None + hidden_input: bool | None = None + possible_value_labels: dict[str, str] | None = Field(default=None, sa_type=sa.JSON) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", @@ -336,6 +348,11 @@ class Parameter(IdentifiedModel, table=True): description: str | None = None data_type: str | None = Field(default=None, max_length=128) unit: str | None = Field(default=None, max_length=128) + # Catalog format 2 fields; null in versions published before it. + metadata_label: str | None = Field(default=None, max_length=512) + period: str | None = Field(default=None, max_length=32) + economy: bool | None = None + household: bool | None = None tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index da5fefe35..1436f9704 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -24,7 +24,7 @@ ) from policyengine_api.services.v2.metadata.types import MetadataCatalogDocument -METADATA_CATALOG_FORMAT = 1 +METADATA_CATALOG_FORMAT = 2 """Changes whenever the document's shape changes, so validators never match old bodies.""" RUNNING_CATALOG_MAX_AGE_SECONDS = 300 diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index 0338e4cc0..8dfcda755 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -119,6 +119,10 @@ def _parameter_children_statement( col(Parameter.description).label("parameter_description"), col(Parameter.data_type).label("parameter_data_type"), col(Parameter.unit).label("parameter_unit"), + col(Parameter.metadata_label).label("parameter_metadata_label"), + col(Parameter.period).label("parameter_period"), + col(Parameter.economy).label("parameter_economy"), + col(Parameter.household).label("parameter_household"), ) .select_from( paths.outerjoin( diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 9fd461b5e..b42b29f17 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -151,20 +151,28 @@ def read_all_parameter_rows( col(Parameter.description), col(Parameter.data_type), col(Parameter.unit), + col(Parameter.metadata_label), + col(Parameter.period), + col(Parameter.economy), + col(Parameter.household), ) .where(col(Parameter.tax_benefit_model_version_id) == model_version_id) .order_by(col(Parameter.name)), ) return [ MetadataCatalogParameterRow( - id=parameter_id, - name=name, - label=label, - description=description, - data_type=data_type, - unit=unit, + id=row.id, + name=row.name, + label=row.label, + description=row.description, + data_type=row.data_type, + unit=row.unit, + metadata_label=row.metadata_label, + period=row.period, + economy=row.economy, + household=row.household, ) - for parameter_id, name, label, description, data_type, unit in rows + for row in rows ] diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 74f8851e3..ea9f65eda 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -13,6 +13,7 @@ MetadataDataset, MetadataDatasetOption, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, MetadataModelSelectionResult, MetadataModelVersionDetail, @@ -85,6 +86,16 @@ def metadata_model_version(selected: SelectedCatalog) -> MetadataModelVersionDet description=selected.model_version.description, current_law_id=selected.model_version.current_law_id, metadata_time_periods=selected.model_version.metadata_time_periods, + catalog_format=selected.model_version.catalog_format, + entities=( + None + if selected.model_version.entities is None + else [ + MetadataEntity.model_validate(entity) + for entity in selected.model_version.entities + ] + ), + basic_inputs=selected.model_version.basic_inputs, ) @@ -110,6 +121,13 @@ def metadata_variable(variable: Variable) -> MetadataVariable: default_value=variable.default_value, adds=variable.adds, subtracts=variable.subtracts, + value_type=variable.value_type, + definition_period=variable.definition_period, + unit=variable.unit, + module_name=variable.module_name, + is_input_variable=variable.is_input_variable, + hidden_input=variable.hidden_input, + possible_value_labels=variable.possible_value_labels, ) @@ -122,6 +140,10 @@ def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: description=parameter.description, data_type=parameter.data_type, unit=parameter.unit, + metadata_label=parameter.metadata_label, + period=parameter.period, + economy=parameter.economy, + household=parameter.household, ) @@ -190,6 +212,10 @@ def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild] description=row.parameter_description, data_type=row.parameter_data_type, unit=row.parameter_unit, + metadata_label=row.parameter_metadata_label, + period=row.parameter_period, + economy=row.parameter_economy, + household=row.parameter_household, ) items.append( MetadataParameterChild( @@ -258,6 +284,10 @@ def _catalog_parameter( "description": parameter.description, "data_type": parameter.data_type, "unit": parameter.unit, + "metadata_label": parameter.metadata_label, + "period": parameter.period, + "economy": parameter.economy, + "household": parameter.household, "values": values, } ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index b7aa1627b..2594c5374 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -64,12 +64,26 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None + value_type: str | None = None + """The model's value type: float, int, bool, str, date or Enum.""" + definition_period: str | None = None + unit: str | None = None + module_name: str | None = None + is_input_variable: bool | None = None + """Whether the variable is an input rather than calculated by a formula.""" + hidden_input: bool | None = None + """Whether input forms should leave the variable out.""" + possible_value_labels: dict[str, str] | None = None + """For an Enum variable, each possible value's display label.""" class MetadataParameterSummary(StrictResponseModel): """A policy parameter, described without its values. - Its current-law values are listed by GET /v2/parameter-values. + Its current-law values are listed by GET /v2/parameter-values. label is + PolicyEngine.py's descriptive label; metadata_label is the label written in + the parameter's own metadata, if any. economy and household say whether + the parameter applies to economy and household calculations. """ id: UUID @@ -78,6 +92,10 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None + metadata_label: str | None = None + period: str | None = None + economy: bool | None = None + household: bool | None = None class MetadataCanonicalParameterValue(StrictResponseModel): @@ -153,11 +171,40 @@ class MetadataDatasetOption(StrictResponseModel): default: Literal[True] = True +class MetadataEntityRole(StrictResponseModel): + """A role a member can hold in a group entity, such as a household member.""" + + key: str + plural: str | None + label: str | None + description: str | None + + +class MetadataEntity(StrictResponseModel): + """An entity of the model, such as a person, household or tax unit.""" + + key: str + plural: str | None + label: str | None + description: str | None + is_person: bool + roles: list[MetadataEntityRole] + + class MetadataModelVersionDetail(MetadataModelVersion): - """A model version, with its current-law policy id and the years offered.""" + """A model version, with its current-law policy id and the years offered. + + catalog_format is null for versions published before catalog format 2, + which also leave entities, basic_inputs and the format 2 fields of + variables and parameters null. basic_inputs names the variables a + household form asks for first. + """ current_law_id: int metadata_time_periods: list[int] + catalog_format: int | None = None + entities: list[MetadataEntity] | None = None + basic_inputs: list[str] | None = None class MetadataParameterChild(StrictResponseModel): @@ -226,6 +273,10 @@ class MetadataCatalogParameterRow: description: str | None data_type: str | None unit: str | None + metadata_label: str | None + period: str | None + economy: bool | None + household: bool | None ResourceT = TypeVar("ResourceT") diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 6e69f789b..2f8f5b167 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -19,6 +19,64 @@ } +class ParameterScale(SimpleNamespace): + """A Core bracket schedule; the extractor recognizes it by class name.""" + + +def _core_system(country_id: str) -> SimpleNamespace: + """Return a stand-in Core system holding the metadata the extractor reads.""" + core_variable = SimpleNamespace( + name="employment_income", + module_name="household.income", + hidden_input=False, + is_input_variable=lambda: True, + possible_values=None, + ) + core_node = SimpleNamespace( + name="gov.example", + metadata={"label": "Example policy"}, + description="Example policy parameters", + ) + core_parameter = SimpleNamespace( + name="gov.example.rate", + metadata={ + "label": "Rate", + "period": "year", + "economy": True, + "household": False, + }, + description="An example rate", + ) + descendants = [core_node, core_parameter] + return SimpleNamespace( + variables={core_variable.name: core_variable}, + parameters=SimpleNamespace(get_descendants=lambda: descendants), + entities=[ + SimpleNamespace( + key="person", + plural="people", + label="Person", + doc="An individual.", + is_person=True, + ), + SimpleNamespace( + key="household", + plural="households", + label="Household", + doc="\nA household.\n", + is_person=False, + roles=[ + SimpleNamespace( + key="member", plural="members", label="Member", doc=None + ) + ], + ), + ], + basic_inputs=["employment_income"], + country_id=country_id, + ) + + class RegionRegistry(list): """Small iterable with the public registry's country identifier.""" @@ -102,6 +160,9 @@ def _model_source( default_value=0.0, adds=None, subtracts=None, + value_type=float, + definition_period="year", + unit={"us": "currency-USD", "uk": "currency-GBP"}[country_id], ) parameter_node = SimpleNamespace( name="gov.example", @@ -149,6 +210,8 @@ def _model_source( version_holder.variables_by_name = {variable.name: variable} version_holder.parameter_nodes_by_name = {parameter_node.name: parameter_node} version_holder.parameters_by_name = {parameter.name: parameter} + core_system = _core_system(country_id) + version_holder._load_system = lambda: core_system return version_holder diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index 3cb7f1d3a..b0f2d5099 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -38,11 +38,13 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: ) # Counts for the 6.7.5 bundle's models: US 2.2.1 and UK 2.130.1. Keep # this exact assertion so later package changes require explicit review. + # Parameter nodes include the 907 bracket schedules (866 US, 41 UK) that + # catalog format 2 adds beside PolicyEngine.py's 29,306 public nodes. assert catalog.entity_counts() == { "models": 2, "model_versions": 2, "variables": 7_330, - "parameter_nodes": 29_306, + "parameter_nodes": 30_213, "parameters": 104_171, "parameter_values": 1_194_357, "datasets": 3, diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 06e48f64c..00708453f 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -15,10 +15,12 @@ from sqlalchemy.pool import NullPool from sqlmodel import Session, select +from policyengine_api.data.v2.catalog.extraction import extract_catalog from policyengine_api.data.v2.catalog.publication import ( CatalogPublicationError, publish_catalog, ) +from policyengine_api.data.v2.catalog.records import CATALOG_FORMAT from policyengine_api.data.v2.models import ( Dataset, DatasetVersion, @@ -30,7 +32,14 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from tests.fixtures.v2_catalog import normalized_catalog +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + ParameterScale, + bundle, + installed_version, + normalized_catalog, + source_models, +) REPO = Path(__file__).parents[2] DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" @@ -340,3 +349,119 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "reports": 0, "report_runs": 0, } + + +def _catalog_with_bracket_schedule(): + models = source_models() + system = models["us"]._load_system() + descendants = system.parameters.get_descendants() + schedule = ParameterScale( + name="gov.example.schedule", + metadata={"label": "Example schedule"}, + description="Rates by income band", + ) + system.parameters.get_descendants = lambda: [*descendants, schedule] + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +def _presentation_rows(engine: Engine) -> dict[str, object]: + with engine.connect() as connection: + return { + "model_versions": connection.execute( + text( + "SELECT catalog_format, entities::text, basic_inputs::text " + "FROM tax_benefit_model_versions ORDER BY id" + ) + ).all(), + "variables": connection.execute( + text( + "SELECT value_type, definition_period, unit, module_name, " + "is_input_variable, hidden_input FROM variables ORDER BY id" + ) + ).all(), + "parameters": connection.execute( + text( + "SELECT metadata_label, period, economy, household " + "FROM parameters ORDER BY id" + ) + ).all(), + "nodes": connection.execute( + text("SELECT name FROM parameter_nodes ORDER BY name") + ) + .scalars() + .all(), + } + + +def test_new_versions_store_presentation_metadata_and_bracket_schedules( + publication_engine: Engine, +) -> None: + catalog = _catalog_with_bracket_schedule() + + publish_catalog(publication_engine, catalog) + publish_catalog(publication_engine, catalog) + + rows = _presentation_rows(publication_engine) + assert {row[0] for row in rows["model_versions"]} == {CATALOG_FORMAT} + assert all('"person"' in row[1] for row in rows["model_versions"]) + assert set(rows["variables"]) == { + ("float", "year", "currency-USD", "household.income", True, False), + ("float", "year", "currency-GBP", "household.income", True, False), + } + assert set(rows["parameters"]) == {("Rate", "year", True, False)} + assert rows["nodes"] == ["gov.example", "gov.example", "gov.example.schedule"] + + +def test_versions_published_before_format_two_are_compared_and_never_rewritten( + publication_engine: Engine, +) -> None: + catalog = _catalog_with_bracket_schedule() + publish_catalog(publication_engine, catalog) + with publication_engine.begin() as connection: + connection.execute( + text( + "UPDATE tax_benefit_model_versions " + "SET catalog_format = NULL, entities = NULL, basic_inputs = NULL" + ) + ) + connection.execute( + text( + "UPDATE variables SET value_type = NULL, definition_period = NULL, " + "unit = NULL, module_name = NULL, is_input_variable = NULL, " + "hidden_input = NULL, possible_value_labels = NULL" + ) + ) + connection.execute( + text( + "UPDATE parameters SET metadata_label = NULL, period = NULL, " + "economy = NULL, household = NULL" + ) + ) + connection.execute( + text("DELETE FROM parameter_nodes WHERE name = 'gov.example.schedule'") + ) + before = _presentation_rows(publication_engine) + before_counts = _counts(publication_engine) + + publish_catalog(publication_engine, catalog) + + assert _presentation_rows(publication_engine) == before + assert _counts(publication_engine) == before_counts + assert {row[0] for row in before["model_versions"]} == {None} + + +def test_format_two_versions_compare_presentation_metadata( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + with publication_engine.begin() as connection: + connection.execute(text("UPDATE parameters SET economy = false")) + + with pytest.raises(CatalogPublicationError, match="differs"): + publish_catalog(publication_engine, catalog) diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index b29fd2eaf..831b39db5 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -235,6 +235,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + "fbb7488ebe8a", "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -495,7 +496,7 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" + assert 'revision: str = "fbb7488ebe8a"' in revision assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -565,6 +566,43 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None ) in upgrade +def test_catalog_presentation_revision_is_generated_and_reversible() -> None: + revision = ( + REPO + / "migrations/v2/versions/72fab86fa97d_store_catalog_presentation_metadata.py" + ).read_text(encoding="utf-8") + + assert V2_ALEMBIC_HEAD_REVISION == "72fab86fa97d" + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert 'down_revision: Union[str, None] = "fbb7488ebe8a"' in revision + added_columns = { + ("parameters", "metadata_label"), + ("parameters", "period"), + ("parameters", "economy"), + ("parameters", "household"), + ("tax_benefit_model_versions", "catalog_format"), + ("tax_benefit_model_versions", "entities"), + ("tax_benefit_model_versions", "basic_inputs"), + ("variables", "value_type"), + ("variables", "definition_period"), + ("variables", "unit"), + ("variables", "module_name"), + ("variables", "is_input_variable"), + ("variables", "hidden_input"), + ("variables", "possible_value_labels"), + } + for table_name, column_name in added_columns: + assert f'"{column_name}"' in revision + assert f'op.drop_column("{table_name}", "{column_name}")' in revision + assert revision.count("op.add_column(") == len(added_columns) + assert revision.count("nullable=True") == len(added_columns) + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index e060c1d43..c70bf5622 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -5,6 +5,7 @@ import ast import json from datetime import UTC, datetime +from enum import Enum from importlib.metadata import PackageNotFoundError from pathlib import Path from types import SimpleNamespace @@ -16,9 +17,11 @@ extract_catalog, normalize_json_value, ) +from policyengine_api.data.v2.catalog.records import CATALOG_FORMAT from tests.fixtures.v2_catalog import ( DEPENDENCY_VERSIONS, POLICYENGINE_VERSION, + ParameterScale, bundle, installed_version, normalized_catalog, @@ -474,3 +477,210 @@ def test_extractor_has_no_direct_core_country_or_v1_metadata_imports() -> None: ) for module in imported_modules ) + + +def test_extracts_presentation_metadata_through_the_country_system() -> None: + us = normalized_catalog().country("us") + variable = us.variables[0] + parameter = us.parameters[0] + + assert us.model_version.catalog_format == CATALOG_FORMAT + assert us.model_version.basic_inputs == ("employment_income",) + assert us.model_version.entities == ( + { + "key": "person", + "plural": "people", + "label": "Person", + "description": "An individual.", + "is_person": True, + "roles": [], + }, + { + "key": "household", + "plural": "households", + "label": "Household", + "description": "A household.", + "is_person": False, + "roles": [ + { + "key": "member", + "plural": "members", + "label": "Member", + "description": None, + } + ], + }, + ) + assert ( + variable.value_type, + variable.definition_period, + variable.unit, + variable.module_name, + variable.is_input_variable, + variable.hidden_input, + variable.possible_value_labels, + ) == ("float", "year", "currency-USD", "household.income", True, False, None) + assert ( + parameter.metadata_label, + parameter.period, + parameter.economy, + parameter.household, + ) == ("Rate", "year", True, False) + + +def test_parameters_without_flags_default_to_economy_and_household() -> None: + models = source_models() + core_parameter = models["uk"]._load_system().parameters.get_descendants()[1] + core_parameter.metadata = {} + + catalog = extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + parameter = catalog.country("uk").parameters[0] + + assert (parameter.metadata_label, parameter.period) == (None, None) + assert (parameter.economy, parameter.household) == (True, True) + + +def test_enum_variables_keep_each_value_label() -> None: + models = source_models() + + class Region(Enum): + NORTH_EAST = "North East" + LONDON = "London" + + public_variable = models["uk"].variables_by_name["employment_income"] + public_variable.value_type = type("Enum", (), {}) + public_variable.possible_values = ["NORTH_EAST", "LONDON"] + core_variable = models["uk"]._load_system().variables["employment_income"] + core_variable.possible_values = Region + + catalog = extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + variable = catalog.country("uk").variables[0] + + assert variable.value_type == "Enum" + assert variable.possible_values == ["NORTH_EAST", "LONDON"] + assert variable.possible_value_labels == { + "NORTH_EAST": "North East", + "LONDON": "London", + } + + +def test_adds_labelled_bracket_schedules_the_public_model_omits() -> None: + models = source_models() + system = models["us"]._load_system() + descendants = system.parameters.get_descendants() + schedule = ParameterScale( + name="gov.example.schedule", + metadata={"label": "Example schedule"}, + description="Rates by income band", + ) + system.parameters.get_descendants = lambda: [*descendants, schedule] + + catalog = extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + nodes = {node.name: node for node in catalog.country("us").parameter_nodes} + + assert nodes["gov.example"].is_scale is False + assert nodes["gov.example.schedule"].is_scale is True + assert nodes["gov.example.schedule"].label == "Example schedule" + assert nodes["gov.example.schedule"].description == "Rates by income band" + assert catalog.country("us").entity_counts()["parameter_nodes"] == 2 + + +def test_rejects_a_public_model_without_its_country_system() -> None: + models = source_models() + del models["us"]._load_system + + with pytest.raises(CatalogExtractionError, match="country system"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +@pytest.mark.parametrize("record_kind", ["variables", "parameters"]) +def test_rejects_records_missing_from_the_country_system(record_kind: str) -> None: + models = source_models() + system = models["us"]._load_system() + if record_kind == "variables": + system.variables.clear() + else: + system.parameters.get_descendants = list + + with pytest.raises(CatalogExtractionError, match="absent from the country system"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +def test_rejects_enum_labels_that_do_not_match_the_possible_values() -> None: + models = source_models() + + class Region(Enum): + NORTH_EAST = "North East" + + public_variable = models["uk"].variables_by_name["employment_income"] + public_variable.value_type = type("Enum", (), {}) + public_variable.possible_values = ["NORTH_EAST", "LONDON"] + core_variable = models["uk"]._load_system().variables["employment_income"] + core_variable.possible_values = Region + + with pytest.raises(CatalogExtractionError, match="enum labels do not match"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +@pytest.mark.parametrize( + ("flag_name", "flag_value"), + [("economy", "false"), ("household", 1)], +) +def test_rejects_parameter_flags_that_are_not_booleans( + flag_name: str, flag_value: object +) -> None: + models = source_models() + core_parameter = models["uk"]._load_system().parameters.get_descendants()[1] + core_parameter.metadata = {**core_parameter.metadata, flag_name: flag_value} + + with pytest.raises(CatalogExtractionError, match=f"{flag_name} must be a boolean"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + +def test_rejects_variable_flags_that_are_not_booleans() -> None: + models = source_models() + core_variable = models["uk"]._load_system().variables["employment_income"] + core_variable.hidden_input = "no" + + with pytest.raises(CatalogExtractionError, match="hidden_input must be a boolean"): + extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 37346a899..2477309ee 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -19,6 +19,7 @@ publication_reconciliation, publication_staging, ) +from policyengine_api.data.v2.catalog.records import CATALOG_FORMAT from policyengine_api.data.v2.migration_target import V2_ALEMBIC_HEAD_REVISION from policyengine_api.data.v2.models import ( Dataset, @@ -209,12 +210,20 @@ def test_publication_uses_mapped_tables_and_sqlalchemy_statements() -> None: assert all("VALUES" not in statement for statement in compiled_inserts) catalog = normalized_catalog() - comparisons = publication_reconciliation._comparison_pairs(catalog.country("us")) - assert all( - isinstance(statement, sa.sql.Select) and not isinstance(statement, TextClause) - for pair in comparisons - for statement in pair - ) + for stored_format in (None, CATALOG_FORMAT): + comparisons = publication_reconciliation._comparison_pairs( + catalog.country("us"), stored_format=stored_format + ) + assert all( + isinstance(statement, sa.sql.Select) + and not isinstance(statement, TextClause) + for pair in comparisons + for statement in pair + ) + assert all( + len(staged.selected_columns) == len(stored.selected_columns) + for staged, stored in comparisons + ) class Result: def scalar_one(self): diff --git a/tests/unit/v2/test_metadata_catalog.py b/tests/unit/v2/test_metadata_catalog.py index 2cc17eab3..b654a6881 100644 --- a/tests/unit/v2/test_metadata_catalog.py +++ b/tests/unit/v2/test_metadata_catalog.py @@ -19,6 +19,10 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.fastapi_routes.v2.metadata.catalog_routes import ( + METADATA_CATALOG_FORMAT, + catalog_entity_tag, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataCatalogResponse, ) @@ -113,6 +117,10 @@ def test_catalog_holds_every_node_parameter_value_and_variable( "description": "An example rate", "data_type": "float", "unit": "/1", + "metadata_label": "Rate", + "period": "year", + "economy": True, + "household": False, "values": {"2025-01-01": 0.1, "2026-01-01": 0.2}, } ] @@ -121,6 +129,9 @@ def test_catalog_holds_every_node_parameter_value_and_variable( ] assert result["variables"][0]["entity"] == "person" assert result["variables"][0]["label"] == "Employment income" + assert result["variables"][0]["value_type"] == "float" + assert result["variables"][0]["is_input_variable"] is True + assert result["variables"][0]["module_name"] == "household.income" def test_catalog_body_matches_the_published_response_schema( @@ -299,3 +310,69 @@ def failing_build() -> MetadataCatalogDocument: ) assert document.body == b"us-1.0.0" + + +DOCUMENT_FIELDS_BY_FORMAT = { + 2: { + "result": { + "policyengine_version", + "parameter_nodes", + "parameters", + "variables", + }, + "parameter_node": {"name", "label", "description"}, + "parameter": { + "id", + "name", + "label", + "description", + "data_type", + "unit", + "metadata_label", + "period", + "economy", + "household", + "values", + }, + "variable": { + "id", + "name", + "label", + "entity", + "description", + "data_type", + "possible_values", + "default_value", + "adds", + "subtracts", + "value_type", + "definition_period", + "unit", + "module_name", + "is_input_variable", + "hidden_input", + "possible_value_labels", + }, + }, +} +"""The fields of each document format. Adding or removing a field needs a new format.""" + + +def test_document_format_in_the_etag_names_the_served_fields( + catalog_session: Session, +) -> None: + """A client holding a body of an older format never revalidates it as current.""" + document = _service(catalog_session).get_model_catalog("us") + result = _result(document) + + served_fields = { + "result": set(result), + "parameter_node": set(result["parameter_nodes"][0]), + "parameter": set(result["parameters"][0]), + "variable": set(result["variables"][0]), + } + + assert DOCUMENT_FIELDS_BY_FORMAT.get(METADATA_CATALOG_FORMAT) == served_fields + assert catalog_entity_tag(document) == ( + f'"us:{POLICYENGINE_VERSION}:{METADATA_CATALOG_FORMAT}"' + ) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index b843679ad..8ad0f4cf9 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -73,6 +73,9 @@ def _insert_country( if time_periods is None else time_periods ), + catalog_format=country.model_version.catalog_format, + entities=list(country.model_version.entities), + basic_inputs=list(country.model_version.basic_inputs), ) ) session.add_all( @@ -88,6 +91,13 @@ def _insert_country( default_value=record.default_value, adds=record.adds, subtracts=record.subtracts, + value_type=record.value_type, + definition_period=record.definition_period, + unit=record.unit, + module_name=record.module_name, + is_input_variable=record.is_input_variable, + hidden_input=record.hidden_input, + possible_value_labels=record.possible_value_labels, ) for record in country.variables ) @@ -110,6 +120,10 @@ def _insert_country( description=record.description, data_type=record.data_type, unit=record.unit, + metadata_label=record.metadata_label, + period=record.period, + economy=record.economy, + household=record.household, ) for record in country.parameters ) @@ -626,3 +640,62 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: for method_name in method_names: method = getattr(V2MetadataService, method_name) assert method.__module__.endswith(".services") + + +def test_resources_publish_catalog_format_two_presentation_metadata( + catalog_session: Session, +) -> None: + service = _service(catalog_session) + + variable = service.list_variables("us").items[0] + parameter = service.list_parameters("us").items[0] + child = service.list_parameter_children("us", parent_path="gov.example").items[0] + selection = service.get_model_by_country("us") + + assert ( + variable.value_type, + variable.definition_period, + variable.unit, + variable.module_name, + variable.is_input_variable, + variable.hidden_input, + variable.possible_value_labels, + ) == ("float", "year", "currency-USD", "household.income", True, False, None) + assert ( + parameter.metadata_label, + parameter.period, + parameter.economy, + parameter.household, + ) == ("Rate", "year", True, False) + assert child.parameter == parameter + assert selection.model_version.catalog_format == 2 + assert selection.model_version.basic_inputs == ["employment_income"] + assert [entity.key for entity in selection.model_version.entities] == [ + "person", + "household", + ] + household = selection.model_version.entities[1] + assert (household.plural, household.is_person) == ("households", False) + assert [role.key for role in household.roles] == ["member"] + + +def test_versions_published_before_format_two_read_as_null_metadata( + catalog_session: Session, +) -> None: + model_version = _us_model_version(catalog_session) + model_version.catalog_format = None + model_version.entities = None + model_version.basic_inputs = None + for variable in catalog_session.exec(select(Variable)).all(): + variable.value_type = None + variable.is_input_variable = None + catalog_session.commit() + + service = _service(catalog_session) + selection = service.get_model_by_country("us") + variable = service.list_variables("us").items[0] + + assert selection.model_version.catalog_format is None + assert selection.model_version.entities is None + assert selection.model_version.basic_inputs is None + assert (variable.value_type, variable.is_input_variable) == (None, None)