From 14a662414c1f2b17b5dbe5b3baa24db5b650a2a3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:13:54 +0400 Subject: [PATCH 001/118] Derive every v2 country list from one list in code The Public API listed its countries in several places: the v2 routes' `CountryId` type (`Literal["us", "uk"]`), an unused `SUPPORTED_COUNTRY_IDS` set in the query models, catalog extraction's `SUPPORTED_COUNTRIES` and its required packages and installed models, the set of countries the v1 routes copy into v2, and the household copy command's options. `policyengine_api/supported_countries.py` now holds the one list, `SupportedCountryId`, and `SUPPORTED_COUNTRY_IDS` derived from it; each of those places derives from it. The list lives in its own module, not in the query models, because the query models import the catalog package, which must import the list too. Tests: in a fresh interpreter, adding "zz" to the one list adds it to every derived list (this failed before the change: `CountryId` rejected "zz"); the catalog's per-country settings must cover exactly the listed countries. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/catalog_selection.py | 3 +- .../data/v2/catalog/extraction.py | 10 +- policyengine_api/query_parameters.py | 4 +- policyengine_api/supported_countries.py | 19 +++ scripts/process_v1_household_mirror_event.py | 3 +- tests/unit/test_supported_countries.py | 139 ++++++++++++++++++ 6 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 policyengine_api/supported_countries.py create mode 100644 tests/unit/test_supported_countries.py diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index d9b50fc76..1e97b6e3c 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -12,8 +12,9 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS -SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) +SUPPORTED_V2_COUNTRY_IDS = frozenset(SUPPORTED_COUNTRY_IDS) MAXIMUM_POLICYENGINE_VERSION_LENGTH = 128 diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 0872eea09..0817e41ae 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -27,15 +27,15 @@ RegionRecord, VariableRecord, ) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from policyengine_api.json_types import JSONValue -SUPPORTED_COUNTRIES = ("us", "uk") +SUPPORTED_COUNTRIES = SUPPORTED_COUNTRY_IDS REQUIRED_DEPENDENCIES = ( "policyengine-core", - "policyengine-us", - "policyengine-uk", + *(f"policyengine-{country_id}" for country_id in SUPPORTED_COUNTRY_IDS), ) REVIEWED_DEFAULT_DATASETS = { "us": "populace_us_2024", @@ -928,7 +928,7 @@ def extract_installed_catalog() -> NormalizedCatalog: bundle=get_current_bundle(), policyengine_version=policyengine_version, models={ - "us": policyengine.us.model, - "uk": policyengine.uk.model, + country_id: getattr(policyengine, country_id).model + for country_id in SUPPORTED_COUNTRIES }, ) diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 06e38cf01..619c6924b 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -29,6 +29,7 @@ validate_policyengine_version, ) from policyengine_api.spm import SPMSelection +from policyengine_api.supported_countries import SupportedCountryId from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, BUDGET_WINDOW_MAX_YEARS, @@ -40,7 +41,6 @@ DEFAULT_QUERY_LIMIT = 100 MAXIMUM_QUERY_LIMIT = 500 MAXIMUM_USER_ID_LENGTH = 255 -SUPPORTED_COUNTRY_IDS = frozenset({"us", "uk"}) def normalize_country_id(value: object) -> object: @@ -57,7 +57,7 @@ def validate_legacy_user_id(value: str) -> str: CountryId = Annotated[ - Literal["us", "uk"], + SupportedCountryId, BeforeValidator(normalize_country_id), Field(description="Supported PolicyEngine country ID"), ] diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py new file mode 100644 index 000000000..3756e2bef --- /dev/null +++ b/policyengine_api/supported_countries.py @@ -0,0 +1,19 @@ +"""The countries the Public API serves, listed once in code. + +``SupportedCountryId`` is the one country list. Every other list derives from +it: the v2 routes' ``CountryId`` type, the countries catalog extraction +publishes, and the countries the v1 routes copy into v2 (dual-write). Adding +a country starts here. + +The module imports nothing from the package, so modules that the query +models import, such as the catalog package, can import it too. +""" + +from __future__ import annotations + +from typing import Final, Literal, get_args + +SupportedCountryId = Literal["us", "uk"] +SUPPORTED_COUNTRY_IDS: Final[tuple[SupportedCountryId, ...]] = get_args( + SupportedCountryId +) diff --git a/scripts/process_v1_household_mirror_event.py b/scripts/process_v1_household_mirror_event.py index f7af31072..49717019b 100755 --- a/scripts/process_v1_household_mirror_event.py +++ b/scripts/process_v1_household_mirror_event.py @@ -20,6 +20,7 @@ process_household_event_after_commit, ) from policyengine_api.services.household_service import HouseholdService +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -149,7 +150,7 @@ def _parser() -> argparse.ArgumentParser: required=True, choices=("staging", "production"), ) - parser.add_argument("--country-id", required=True, choices=("us", "uk")) + parser.add_argument("--country-id", required=True, choices=SUPPORTED_COUNTRY_IDS) parser.add_argument("--legacy-household-id", required=True, type=int) return parser diff --git a/tests/unit/test_supported_countries.py b/tests/unit/test_supported_countries.py new file mode 100644 index 000000000..429f27657 --- /dev/null +++ b/tests/unit/test_supported_countries.py @@ -0,0 +1,139 @@ +"""The Public API's one country list, and the lists that derive from it.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from typing import get_args + +import pytest +from pydantic import TypeAdapter, ValidationError + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog import extraction +from policyengine_api.query_parameters import CountryId +from policyengine_api.supported_countries import ( + SUPPORTED_COUNTRY_IDS, + SupportedCountryId, +) + +# Runs in a fresh interpreter: it adds a country to the one list before any +# module that derives from it is imported, then reports what each derived +# list contains. PolicyEngine.py is replaced by a stand-in, because importing +# the real package builds every country model. +EXTENDED_LIST_SCRIPT = """ +import json +import sys +from types import ModuleType, SimpleNamespace +from typing import Literal, get_args + +import policyengine_api.supported_countries as supported_countries + +supported_countries.SupportedCountryId = Literal["us", "uk", "zz"] +supported_countries.SUPPORTED_COUNTRY_IDS = get_args( + supported_countries.SupportedCountryId +) + +installed_policyengine = ModuleType("policyengine") +for country_id in ("us", "uk", "zz", "ca"): + setattr( + installed_policyengine, + country_id, + SimpleNamespace(model=f"{country_id} model"), + ) +installed_bundle = ModuleType("policyengine.bundle") +installed_bundle.get_current_bundle = dict +sys.modules["policyengine"] = installed_policyengine +sys.modules["policyengine.bundle"] = installed_bundle + +from pydantic import TypeAdapter + +from policyengine_api.data.v2.catalog import catalog_selection, extraction +from policyengine_api.query_parameters import CountryId +from scripts import process_v1_household_mirror_event + +captured_arguments = {} + + +def capture_extraction(**arguments): + captured_arguments.update(arguments) + + +extraction.extract_catalog = capture_extraction +extraction.extract_installed_catalog() +country_option = next( + action + for action in process_v1_household_mirror_event._parser()._actions + if action.dest == "country_id" +) +print( + json.dumps( + { + "country_type": TypeAdapter(CountryId).validate_python("ZZ"), + "extracted_countries": list(extraction.SUPPORTED_COUNTRIES), + "required_dependencies": list(extraction.REQUIRED_DEPENDENCIES), + "installed_models": captured_arguments["models"], + "dual_write_countries": sorted(catalog_selection.SUPPORTED_V2_COUNTRY_IDS), + "household_copy_countries": list(country_option.choices), + } + ) +) +""" + + +def test_the_country_list_is_the_us_and_the_uk_in_catalog_order() -> None: + assert SUPPORTED_COUNTRY_IDS == ("us", "uk") + assert get_args(SupportedCountryId) == SUPPORTED_COUNTRY_IDS + + +def test_the_v2_country_type_accepts_exactly_the_listed_countries() -> None: + country_id_adapter = TypeAdapter(CountryId) + + for country_id in SUPPORTED_COUNTRY_IDS: + assert country_id_adapter.validate_python(country_id) == country_id + assert country_id_adapter.validate_python(country_id.upper()) == country_id + for unlisted_country_id in ("ca", "ng", "il", ""): + with pytest.raises(ValidationError): + country_id_adapter.validate_python(unlisted_country_id) + + +def test_every_country_list_follows_a_country_added_to_the_one_list() -> None: + result = subprocess.run( + [sys.executable, "-c", EXTENDED_LIST_SCRIPT], + cwd=REPO, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + derived_lists = json.loads(result.stdout.strip().splitlines()[-1]) + assert derived_lists == { + "country_type": "zz", + "extracted_countries": ["us", "uk", "zz"], + "required_dependencies": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "policyengine-zz", + ], + # The installed package's other countries are not extracted. + "installed_models": { + "us": "us model", + "uk": "uk model", + "zz": "zz model", + }, + "dual_write_countries": ["uk", "us", "zz"], + "household_copy_countries": ["us", "uk", "zz"], + } + + +def test_catalog_settings_cover_exactly_the_listed_countries() -> None: + # Each listed country needs reviewed catalog settings, so adding a country + # without them fails here rather than during publication. + listed_countries = set(SUPPORTED_COUNTRY_IDS) + + assert set(extraction.REVIEWED_DEFAULT_DATASETS) == listed_countries + assert set(extraction.CURRENT_LAW_IDS) == listed_countries + assert set(extraction.METADATA_TIME_PERIODS) == listed_countries From 2729c37f4c212e4c2787281bb03d8f015231a99a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:29:39 +0400 Subject: [PATCH 002/118] Record each served country in a countries table that country columns reference Adds the v2 `countries` table: one row per country the Public API serves, with `household_enabled` and `society_wide_enabled` saying which calculations it supports (a check requires society-wide countries to be household capable). `users.primary_country` and the `country_id` of households, policies and the four user association tables reference it, with `ON DELETE RESTRICT`, instead of each checking `IN ('us', 'uk')`. The tables that pair v1 records with their v2 copies keep their checks. Revision `fb1a3e40d3d6` was autogenerated. Its one hand-written operation inserts the US and UK rows, both capable of both, between the table and the foreign keys, which otherwise fail on every database whose rows already name either country. This insert is an exception to the rule that revisions contain only generated operations, approved for this revision alone; the revision's comment and the Alembic skill say so. The generated downgrade restores the checks and drops the table with its rows. The privilege manifest gives the countries table no role access yet. Tests (failed first because the `countries` table and `Country` model did not exist): model structure and foreign keys; SQLite persistence (an unlisted country is refused, a listed country in use cannot be deleted, a household-only row lets records name that country); the revision's operations; and the Postgres lifecycle (upgrade keeps existing US and UK rows, unlisted countries are refused, downgrade and re-upgrade, a downgrade is refused while records name another country, and the previous release's table usage still works after the upgrade). Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2-countries-table.added.md | 1 + docs/engineering/skills/alembic-migrations.md | 24 ++ .../fb1a3e40d3d6_add_countries_table.py | 195 ++++++++++ policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/__init__.py | 2 + .../data/v2/models/associations.py | 40 +- policyengine_api/data/v2/models/countries.py | 37 ++ policyengine_api/data/v2/models/households.py | 10 +- policyengine_api/data/v2/models/policies.py | 10 +- policyengine_api/data/v2/models/users.py | 11 +- policyengine_api/data/v2/privileges.py | 2 + policyengine_api/supported_countries.py | 4 + tests/fixtures/v2_countries.py | 28 ++ .../integration/test_alembic_v2_lifecycle.py | 349 ++++++++++++++++-- tests/unit/v2/test_alembic_v2.py | 91 ++++- tests/unit/v2/test_household_models.py | 3 + tests/unit/v2/test_import_side_effects.py | 2 +- tests/unit/v2/test_model_persistence.py | 78 +++- tests/unit/v2/test_models.py | 90 ++++- tests/unit/v2/user_service_support.py | 3 + 20 files changed, 909 insertions(+), 73 deletions(-) create mode 100644 changelog.d/v2-countries-table.added.md create mode 100644 migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py create mode 100644 policyengine_api/data/v2/models/countries.py create mode 100644 tests/fixtures/v2_countries.py diff --git a/changelog.d/v2-countries-table.added.md b/changelog.d/v2-countries-table.added.md new file mode 100644 index 000000000..5c7e480ac --- /dev/null +++ b/changelog.d/v2-countries-table.added.md @@ -0,0 +1 @@ +Added a v2 `countries` table that records, for each country the Public API serves, whether it supports household and society-wide calculations. Its revision lists the US and the UK, both with both. Users' primary country and the country of every v2 household, policy and user association now reference this table instead of a fixed US-and-UK check. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index d4a043d91..7a5819fa9 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -294,3 +294,27 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### API v2 countries rows + +Revision `fb1a3e40d3d6` was autogenerated after adding the `countries` table +and replacing the US-and-UK checks on `users.primary_country` and on the +`country_id` of households, policies and the four user association tables +with foreign keys to it. It contains one hand-written operation: it inserts +the `uk` and `us` rows, both household and society-wide capable, between the +generated table creation and the generated foreign keys, which otherwise fail +on any database whose rows already name either country. The rows are exactly +the values the replaced checks allowed. The generated downgrade removes them +with the table, and its restored checks fail if any record names another +country. + +Inserting rows is a data operation, which this skill otherwise forbids. The +maintainers approved this insert as a one-off exception, because the new +foreign keys need the rows in the same revision. Unit tests pin the +operation, and the v2 Postgres lifecycle test covers an upgrade that keeps +existing rows, the downgrade, a downgrade refused by another country's +records, and the re-upgrade. + +The table must list exactly the countries in +`policyengine_api/supported_countries.py`. Listing another country later +needs a revision that inserts its row, and the same human migration decision. diff --git a/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py new file mode 100644 index 000000000..1b02ec819 --- /dev/null +++ b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py @@ -0,0 +1,195 @@ +"""add countries table + +Revision ID: fb1a3e40d3d6 +Revises: fbb7488ebe8a +Create Date: 2026-10-22 02:16:50.126715 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "fb1a3e40d3d6" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + countries = op.create_table( + "countries", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False), + sa.Column("household_enabled", sa.Boolean(), nullable=False), + sa.Column("society_wide_enabled", sa.Boolean(), nullable=False), + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name=op.f("ck_countries_society_wide_requires_household"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_countries")), + ) + # Approved hand-written data operation: the one exception to generated-only + # revisions, decided by the maintainers for this table alone. The foreign + # keys below fail on any database whose rows already name the US or the UK + # unless both countries are listed first, so their rows are inserted here, + # in the same revision. They are the countries the replaced checks + # allowed, and both support household and society-wide calculations. The + # generated downgrade removes them with the table. Listing another + # country later needs its own revision and the same kind of decision. + op.bulk_insert( + countries, + [ + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, + ], + ) + op.create_foreign_key( + op.f("fk_households_country_id_countries"), + "households", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_households_country"), "households", type_="check") + op.create_foreign_key( + op.f("fk_policies_country_id_countries"), + "policies", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_policies_country"), "policies", type_="check") + op.create_foreign_key( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_policies_country_id_countries"), + "user_policies", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_user_policies_country"), "user_policies", type_="check") + op.create_foreign_key( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_users_primary_country_countries"), + "users", + "countries", + ["primary_country"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_users_primary_country"), "users", type_="check") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_check_constraint( + op.f("ck_users_primary_country"), + "users", + "primary_country::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_users_primary_country_countries"), "users", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), + "user_policies", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_policies_country_id_countries"), + "user_policies", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_policies_country"), + "policies", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_policies_country_id_countries"), "policies", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_households_country"), + "households", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_households_country_id_countries"), "households", type_="foreignkey" + ) + op.drop_table("countries") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..2c7b6986d 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "fb1a3e40d3d6" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index c63655bb6..a3aedae09 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -31,6 +31,7 @@ Stage12RunStatus, Stage12SimulationRole, ) +from policyengine_api.data.v2.models.countries import Country # noqa: E402 from policyengine_api.data.v2.models.household_mappings import ( # noqa: E402 LegacyHouseholdMapping, ) @@ -102,6 +103,7 @@ "ChangeAggregate", "CongressionalDistrictImpact", "ConstituencyImpact", + "Country", "Dataset", "DatasetVersion", "DecileImpact", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index 2c5cd21b1..dc7e7d757 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -51,10 +51,6 @@ class UserHouseholdAssociation(TimestampedModel, table=True): "country_id", name="uq_user_household_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_household_associations_country", - ), sa.CheckConstraint( "household_id IS NOT NULL OR legacy_household_id IS NOT NULL", name="ck_user_household_associations_household_ref", @@ -91,7 +87,11 @@ class UserHouseholdAssociation(TimestampedModel, table=True): ) household_id: UUID | None = Field(default=None, index=True) legacy_household_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) @@ -113,10 +113,6 @@ class UserPolicy(TimestampedModel, table=True): "country_id", name="uq_user_policies_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_policies_country", - ), sa.CheckConstraint( "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", name="ck_user_policies_policy_ref", @@ -157,7 +153,11 @@ class UserPolicy(TimestampedModel, table=True): ) policy_id: UUID | None = Field(default=None, index=True) legacy_policy_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) baseline: str | None = Field(default=None, max_length=16) @@ -185,10 +185,6 @@ class UserSimulationAssociation(TimestampedModel, table=True): "country_id", name="uq_user_simulation_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_simulation_associations_country", - ), sa.CheckConstraint( "simulation_id IS NOT NULL OR legacy_simulation_id IS NOT NULL", name="ck_user_simulation_associations_simulation_ref", @@ -214,7 +210,11 @@ class UserSimulationAssociation(TimestampedModel, table=True): ) simulation_id: UUID | None = Field(default=None, index=True) legacy_simulation_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) # Retired; see the module docstring. @@ -239,10 +239,6 @@ class UserReportAssociation(TimestampedModel, table=True): "country_id", name="uq_user_report_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_report_associations_country", - ), sa.CheckConstraint( "report_id IS NOT NULL OR legacy_report_id IS NOT NULL", name="ck_user_report_associations_report_ref", @@ -273,7 +269,11 @@ class UserReportAssociation(TimestampedModel, table=True): ) report_id: UUID | None = Field(default=None, index=True) legacy_report_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) user_policy_id: UUID | None = Field( diff --git a/policyengine_api/data/v2/models/countries.py b/policyengine_api/data/v2/models/countries.py new file mode 100644 index 000000000..fc3ae8677 --- /dev/null +++ b/policyengine_api/data/v2/models/countries.py @@ -0,0 +1,37 @@ +"""Canonical SQLModel table recording what each served country supports. + +Which countries exist is decided in code: ``policyengine_api.supported_countries`` +lists them, and this table must hold exactly one row per listed country. A +row records the calculations its country supports: a household-only country +has ``household_enabled`` set and ``society_wide_enabled`` unset. + +``users.primary_country`` and the ``country_id`` of households, policies and +the four user association tables reference ``countries.id``. Other v2 country +columns do not: the mapping tables that pair v1 records with their v2 copies +keep their US-and-UK checks, and the simulation-domain tables, the comparison +run tables and the retired association ``country`` columns store unconstrained +text. +""" + +import sqlalchemy as sa +from sqlmodel import Field, SQLModel + + +class Country(SQLModel, table=True): + """One served country and the calculations it supports. + + A society-wide country must also be household capable. A country that a + user, household, policy or user association names cannot be deleted. + """ + + __tablename__ = "countries" + __table_args__ = ( + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name="ck_countries_society_wide_requires_household", + ), + ) + + id: str = Field(primary_key=True, max_length=2) + household_enabled: bool + society_wide_enabled: bool diff --git a/policyengine_api/data/v2/models/households.py b/policyengine_api/data/v2/models/households.py index 78e76b2ab..181266d9d 100644 --- a/policyengine_api/data/v2/models/households.py +++ b/policyengine_api/data/v2/models/households.py @@ -53,10 +53,6 @@ class Household(TimestampedModel, table=True): "content_hash", name="uq_households_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_households_country", - ), sa.CheckConstraint( "default_year IS NULL OR default_year BETWEEN 1900 AND 2200", name="ck_households_default_year", @@ -78,7 +74,11 @@ class Household(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) default_year: int | None = None household_data: dict[str, Any] = Field( sa_type=sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index 97e4a619e..f0da6e8cc 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -40,10 +40,6 @@ class Policy(TimestampedModel, table=True): "content_hash", name="uq_policies_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_policies_country", - ), sa.CheckConstraint( "canonicalization_version > 0", name="ck_policies_canonicalization_version", @@ -64,7 +60,11 @@ class Policy(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index 85c0ea19a..f3b987123 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -44,10 +44,6 @@ class User(IdentifiedModel, table=True): __tablename__ = "users" __table_args__ = ( sa.UniqueConstraint("email", name="uq_users_email"), - sa.CheckConstraint( - "primary_country IN ('us', 'uk')", - name="ck_users_primary_country", - ), sa.CheckConstraint( "user_type IN ('unauthenticated', 'standard')", name="ck_users_user_type", @@ -61,7 +57,12 @@ class User(IdentifiedModel, table=True): first_name: str | None = Field(default=None, max_length=255) last_name: str | None = Field(default=None, max_length=255) email: str | None = Field(default=None, max_length=320, index=True) - primary_country: str | None = Field(default=None, max_length=2) + primary_country: str | None = Field( + default=None, + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) # The server default keeps inserts by earlier Public API revisions, which # do not map this column, creating standard users. user_type: str = Field( diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 8f8c34515..6fd0cc750 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -132,6 +132,8 @@ def _access( # Stage 10 saved-policy mirror. "legacy_user_mappings": _access(runtime=READ_WRITE), "legacy_user_policy_mappings": _access(runtime=READ_WRITE), + # Served countries; the migration that creates the table inserts its rows. + "countries": _access(), # Users and their saved work (Stage 12A). "user_household_associations": _access(runtime=READ_WRITE), "user_identities": _access(runtime=READ_WRITE), diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py index 3756e2bef..51a495dd4 100644 --- a/policyengine_api/supported_countries.py +++ b/policyengine_api/supported_countries.py @@ -5,6 +5,10 @@ publishes, and the countries the v1 routes copy into v2 (dual-write). Adding a country starts here. +The v2 ``countries`` table records what each country supports, and must hold +one row for each listed country and no other: a v2 Alembic revision inserts +the row when a country is added. + The module imports nothing from the package, so modules that the query models import, such as the catalog package, can import it too. """ diff --git a/tests/fixtures/v2_countries.py b/tests/fixtures/v2_countries.py new file mode 100644 index 000000000..36d4b6604 --- /dev/null +++ b/tests/fixtures/v2_countries.py @@ -0,0 +1,28 @@ +"""Countries rows for SQLite tests that enforce v2 foreign keys. + +``V2_METADATA.create_all`` creates an empty countries table, while the v2 +Alembic revision that adds the table inserts the US and UK rows. Tests that +turn on SQLite foreign-key enforcement add the same rows before writing +records that name a country. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.models import Country + +if TYPE_CHECKING: + from sqlmodel import Session + +# The rows the countries revision inserts, ordered by country ID. +LISTED_COUNTRY_ROWS = ( + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, +) + + +def add_listed_countries(session: Session) -> None: + """Insert and commit the US and UK countries rows.""" + session.add_all(Country(**row) for row in LISTED_COUNTRY_ROWS) + session.commit() diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 814971236..2df169511 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -3,6 +3,7 @@ import itertools import os from collections.abc import Iterator +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 @@ -30,6 +31,7 @@ STAGE_12_PREVIOUS_REVISION = "724b1b11a33e" STAGE_12_COMPARISON_PREVIOUS_REVISION = "439303be14fe" STAGE_12A_PREVIOUS_REVISION = "91af4fc8d8c1" +COUNTRIES_PREVIOUS_REVISION = "fbb7488ebe8a" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -187,7 +189,6 @@ def _assert_stage_12a_association_shape(engine) -> None: for constraint in inspect(engine).get_check_constraints("users") } assert { - "ck_users_primary_country", "ck_users_user_type", "ck_users_standard_primary_country", } <= user_checks @@ -352,6 +353,75 @@ def _assert_base(engine: Engine) -> None: ) +# Every column that names a record's country, with the check it replaced. +COUNTRY_REFERENCE_COLUMNS = { + ("users", "primary_country"): "ck_users_primary_country", + ("households", "country_id"): "ck_households_country", + ("policies", "country_id"): "ck_policies_country", + ("user_household_associations", "country_id"): ( + "ck_user_household_associations_country" + ), + ("user_policies", "country_id"): "ck_user_policies_country", + ("user_simulation_associations", "country_id"): ( + "ck_user_simulation_associations_country" + ), + ("user_report_associations", "country_id"): "ck_user_report_associations_country", +} +# The rows the countries revision inserts: the countries the replaced checks +# allowed, both household and society-wide capable. +LISTED_COUNTRIES = [("uk", True, True), ("us", True, True)] + + +def _country_foreign_keys(engine, table_name: str) -> list[dict]: + return [ + foreign_key + for foreign_key in inspect(engine).get_foreign_keys(table_name) + if foreign_key["referred_table"] == "countries" + ] + + +def _check_names(engine, table_name: str) -> set[str]: + return { + constraint["name"] + for constraint in inspect(engine).get_check_constraints(table_name) + } + + +def _listed_countries(engine) -> list[tuple]: + with engine.connect() as connection: + return [ + tuple(row) + for row in connection.execute( + text( + "SELECT id, household_enabled, society_wide_enabled " + "FROM countries ORDER BY id" + ) + ) + ] + + +def _assert_countries_shape(engine) -> None: + assert _listed_countries(engine) == LISTED_COUNTRIES + assert "ck_countries_society_wide_requires_household" in _check_names( + engine, "countries" + ) + for (table_name, column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + country_keys = _country_foreign_keys(engine, table_name) + assert len(country_keys) == 1, table_name + assert country_keys[0]["name"] == f"fk_{table_name}_{column_name}_countries" + assert country_keys[0]["constrained_columns"] == [column_name] + assert country_keys[0]["referred_columns"] == ["id"] + assert country_keys[0]["options"]["ondelete"] == "RESTRICT" + assert check_name not in _check_names(engine, table_name) + + +def _assert_pre_countries_shape(engine) -> None: + assert "countries" not in inspect(engine).get_table_names(schema="public") + for (table_name, _column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + assert _country_foreign_keys(engine, table_name) == [] + assert check_name in _check_names(engine, table_name) + + def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: database_url = _disposable_url() config = _config() @@ -364,6 +434,13 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: } _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) + + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + _assert_revision(engine, COUNTRIES_PREVIOUS_REVISION) + _assert_pre_countries_shape(engine) + _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) _assert_revision(engine, STAGE_12A_PREVIOUS_REVISION) @@ -396,10 +473,10 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -# Dormant at the parent revision: its models map these tables, and deleting a -# user cascades through them, but no parent-revision code inserts into them. -# They are the only tables Stage 12A may give a new required column. -PARENT_REVISION_READ_ONLY_TABLES = frozenset( +# Dormant at the Stage 12A parent revision: its models map these tables, and +# deleting a user cascades through them, but no parent-revision code inserts +# into them. They are the only tables Stage 12A may give a new required column. +STAGE_12A_PARENT_READ_ONLY_TABLES = frozenset( {"user_simulation_associations", "user_report_associations"} ) USER_ASSOCIATION_TABLES = ( @@ -409,8 +486,18 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_report_associations", ) COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) -# Nullable references that a parent-revision check constraint requires. -REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) +# Nullable references that a parent-revision check constraint requires: a +# standard user's country, and one of each association's two references. +REFERENCES_REQUIRED_BY_CHECKS = frozenset( + { + ("simulations", "household_id"), + ("users", "primary_country"), + ("user_policies", "policy_id"), + ("user_household_associations", "household_id"), + ("user_simulation_associations", "simulation_id"), + ("user_report_associations", "report_id"), + } +) # Required values that a parent-revision check constraint restricts. CHECKED_VALUES = {("datasets", "year"): 2026} @@ -547,14 +634,33 @@ def _rewrite_mapped_columns(connection, table: sa.Table, row: dict) -> None: assert result.rowcount == 1, table.name +@dataclass(frozen=True) +class PreviousRevisionSchemas: + """A previous revision's reflected schema and head's, after the upgrade. + + ``read_only_tables`` are the previous revision's tables that only new code + writes, so their test rows are inserted through head's table. + """ + + parent: sa.MetaData + head: sa.MetaData + read_only_tables: frozenset[str] + + def insert_table(self, table_name: str) -> sa.Table | None: + """Head's table for a read-only table, otherwise ``None``.""" + if table_name in self.read_only_tables: + return self.head.tables[table_name] + return None + + def _assert_user_deletion_cascades( connection, - parent: sa.MetaData, - head: sa.MetaData, + schemas: PreviousRevisionSchemas, stored_rows: dict[str, dict], sequence: Iterator[int], ) -> None: - """Delete, through the parent's users table, a user who owns Stage 12A rows.""" + """Delete, through the parent's users table, a user who owns association rows.""" + parent, head = schemas.parent, schemas.head users = parent.tables["users"] owner = _insert_and_read_back( connection, users, _row_values(users, stored_rows, sequence) @@ -562,11 +668,7 @@ def _assert_user_deletion_cascades( owner_rows = {**stored_rows, "users": owner} for table_name in USER_ASSOCIATION_TABLES: table = parent.tables[table_name] - head_table = ( - head.tables[table_name] - if table_name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + head_table = schemas.insert_table(table_name) _insert_and_read_back( connection, table, @@ -596,22 +698,26 @@ def _assert_user_deletion_cascades( assert remaining == 0, table_name -def test_stage_12a_schema_serves_the_previous_revision() -> None: - """Run the previous Public API revision's table usage against head. +def _assert_head_serves_previous_revision( + previous_revision: str, + read_only_tables: frozenset[str], +) -> None: + """Run a previous Public API revision's table usage against head. The parent revision's schema is reflected and then used, after upgrading, to insert a row into every table it maps (supplying only what it requires), read the row back with every mapped column, rewrite those - columns, delete it, and delete a user who owns Stage 12A rows so the + columns, delete it, and delete a user who owns association rows so the cascades run. Head may not give any table the parent revision writes a - new required column without a server default. + new required column without a server default; ``read_only_tables`` are the + parent's tables that only new code writes. """ database_url = _disposable_url() config = _config() engine = create_engine(database_url) try: - command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) + command.downgrade(config, previous_revision) parent = sa.MetaData() parent.reflect(engine) command.upgrade(config, "head") @@ -634,21 +740,18 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: for table in parent_tables if _new_required_columns(table, head.tables[table.name]) } - assert set(tables_with_new_required_columns) <= ( - PARENT_REVISION_READ_ONLY_TABLES - ), tables_with_new_required_columns + assert set(tables_with_new_required_columns) <= read_only_tables, ( + tables_with_new_required_columns + ) + schemas = PreviousRevisionSchemas(parent, head, read_only_tables) sequence = itertools.count() with engine.connect() as connection: transaction = connection.begin() try: stored_rows: dict[str, dict] = {} for table in parent_tables: - head_table = ( - head.tables[table.name] - if table.name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + head_table = schemas.insert_table(table.name) stored_rows[table.name] = _insert_and_read_back( connection, table, @@ -660,7 +763,7 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: _rewrite_mapped_columns(connection, table, stored_rows[table.name]) _assert_user_deletion_cascades( - connection, parent, head, stored_rows, sequence + connection, schemas, stored_rows, sequence ) for table in reversed(parent_tables): @@ -677,6 +780,21 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: engine.dispose() +def test_stage_12a_schema_serves_the_previous_revision() -> None: + _assert_head_serves_previous_revision( + STAGE_12A_PREVIOUS_REVISION, + STAGE_12A_PARENT_READ_ONLY_TABLES, + ) + + +def test_countries_schema_serves_the_previous_revision() -> None: + """The release before the countries table keeps working during a deploy.""" + _assert_head_serves_previous_revision( + COUNTRIES_PREVIOUS_REVISION, + frozenset(), + ) + + def test_upgrade_to_head_validates_the_resulting_schema_against_metadata() -> None: database_url = _disposable_url() config = _config() @@ -957,3 +1075,176 @@ def reflected(name: str) -> sa.Table: connection.execute(sa.delete(table).where(table.c.id == row_id)) command.upgrade(config, "head") engine.dispose() + + +def test_countries_revision_keeps_existing_rows_and_rejects_unlisted_countries() -> ( + None +): + """Rows saved before the revision survive it, and only listed countries pass.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + def reflected(name: str) -> sa.Table: + return sa.Table(name, sa.MetaData(), autoload_with=engine) + + try: + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + with engine.begin() as connection: + connection.execute( + sa.insert(reflected("users")).values(id=user_id, primary_country="uk") + ) + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="uk", + legacy_report_id=7, + ) + ) + + command.upgrade(config, "head") + command.check(config) + _assert_countries_shape(engine) + + with engine.connect() as connection: + stored_countries = connection.execute( + text( + "SELECT users.primary_country, " + "user_report_associations.country_id FROM users " + "JOIN user_report_associations " + "ON user_report_associations.user_id = users.id " + "WHERE users.id = :user_id" + ), + {"user_id": user_id}, + ).one() + assert tuple(stored_countries) == ("uk", "uk") + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="ca", + legacy_report_id=8, + ) + ) + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute(text("DELETE FROM countries WHERE id = 'uk'")) + finally: + with engine.begin() as connection: + users = reflected("users") + connection.execute(sa.delete(users).where(users.c.id == user_id)) + command.upgrade(config, "head") + engine.dispose() + + +def test_countries_downgrade_refuses_records_of_an_unlisted_country() -> None: + """The restored US-and-UK checks stop a downgrade that would orphan records.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text("INSERT INTO users (id, primary_country) VALUES (:user_id, 'be')"), + {"user_id": user_id}, + ) + + with pytest.raises(sa.exc.IntegrityError): + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == V2_ALEMBIC_HEAD_REVISION + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = :user_id"), {"user_id": user_id} + ) + connection.execute(text("DELETE FROM countries WHERE id = 'be'")) + command.upgrade(config, "head") + engine.dispose() + + +def test_a_household_only_country_row_lets_records_name_that_country() -> None: + """Once a household-only country has a row, its records can be stored.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + household_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.connect() as connection: + transaction = connection.begin() + try: + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + connection.execute( + text( + "INSERT INTO households (id, country_id, household_data, " + "canonicalization_version, content_hash) VALUES " + "(:household_id, 'be', '{}'::jsonb, 1, :content_hash)" + ), + {"household_id": household_id, "content_hash": "c" * 64}, + ) + connection.execute( + text( + "INSERT INTO user_household_associations " + "(id, user_id, household_id, country_id) VALUES " + "(:association_id, :user_id, :household_id, 'be')" + ), + { + "association_id": uuid4(), + "user_id": user_id, + "household_id": household_id, + }, + ) + + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('xx', false, true)" + ) + ) + finally: + transaction.rollback() + finally: + engine.dispose() diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 4c0e6ee09..56e31410b 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -40,6 +40,7 @@ V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) POST_BASELINE_TABLE_NAMES = frozenset( { + "countries", "legacy_household_mappings", "legacy_policy_mappings", "legacy_user_mappings", @@ -235,6 +236,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + "fbb7488ebe8a", "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -495,7 +497,6 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -565,6 +566,94 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None ) in upgrade +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policies", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) +# The generated downgrade restores each check in PostgreSQL's reflected form +# of `` IN ('us', 'uk')``. +US_AND_UK = "ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])" +REPLACED_COUNTRY_CHECKS = { + "ck_users_primary_country": ("users", f"primary_country::text = {US_AND_UK}"), + "ck_households_country": ("households", f"country_id::text = {US_AND_UK}"), + "ck_policies_country": ("policies", f"country_id::text = {US_AND_UK}"), + "ck_user_household_associations_country": ( + "user_household_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_policies_country": ("user_policies", f"country_id::text = {US_AND_UK}"), + "ck_user_simulation_associations_country": ( + "user_simulation_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_report_associations_country": ( + "user_report_associations", + f"country_id::text = {US_AND_UK}", + ), +} + + +def _countries_revision_source() -> str: + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + countries_revision = next( + revision + for revision in script.walk_revisions() + if revision.down_revision == "fbb7488ebe8a" + ) + assert countries_revision.revision == V2_ALEMBIC_HEAD_REVISION + assert countries_revision.path is not None + return Path(countries_revision.path).read_text(encoding="utf-8") + + +def test_countries_revision_inserts_us_and_uk_and_replaces_country_checks() -> None: + revision = _countries_revision_source() + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert revision.count("op.create_table(") == 1 + assert 'op.create_table(\n "countries"' in upgrade + assert "ck_countries_society_wide_requires_household" in upgrade + assert 'op.drop_table("countries")' in downgrade + # The one hand-written operation, an approved exception to generated-only + # revisions: the US and UK rows, inserted before the foreign keys that + # existing rows must satisfy. The generated downgrade drops them with the + # table. + assert revision.count("Approved hand-written data operation") == 1 + assert revision.count("Post-generation correction") == 0 + assert revision.count("op.bulk_insert(") == 1 + assert "op.execute(" not in revision + inserted_rows = upgrade[ + upgrade.index("op.bulk_insert(") : upgrade.index("op.create_foreign_key(") + ] + for country_id in ("uk", "us"): + assert f'"id": "{country_id}"' in inserted_rows + assert inserted_rows.count('"household_enabled": True') == 2 + assert inserted_rows.count('"society_wide_enabled": True') == 2 + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + key_name = f"fk_{table_name}_{column_name}_countries" + assert key_name in upgrade + assert key_name in downgrade + assert upgrade.count('ondelete="RESTRICT"') == len(COUNTRY_REFERENCE_COLUMNS) + for check_name, (table_name, expression) in REPLACED_COUNTRY_CHECKS.items(): + assert check_name in upgrade + assert f'"{table_name}"' in upgrade + assert check_name in downgrade + assert expression in downgrade + assert "op.drop_column(" not in upgrade + assert "op.add_column(" not in upgrade + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_household_models.py b/tests/unit/v2/test_household_models.py index f0798e35e..bef59c843 100644 --- a/tests/unit/v2/test_household_models.py +++ b/tests/unit/v2/test_household_models.py @@ -18,6 +18,7 @@ User, UserHouseholdAssociation, ) +from tests.fixtures.v2_countries import add_listed_countries def _engine(): @@ -30,6 +31,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 7b753004d..af6913897 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 39 +assert len(V2_METADATA.tables) == 40 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index be7879424..732fa7d98 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -10,7 +10,9 @@ from policyengine_api.data.v2.models import ( V2_METADATA, + Country, Dynamic, + Household, LegacyPolicyMapping, LegacyUserMapping, LegacyUserPolicyMapping, @@ -20,9 +22,11 @@ TaxBenefitModel, TaxBenefitModelVersion, User, + UserHouseholdAssociation, UserPolicy, ) from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS +from tests.fixtures.v2_countries import add_listed_countries def _relational_sqlite_engine(): @@ -35,6 +39,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine @@ -99,9 +105,8 @@ def test_direct_sqlalchemy_categories_are_complete_and_documented() -> None: assert all(DIRECT_SQLALCHEMY_EXCEPTIONS.values()) -def test_user_primary_country_can_change_between_us_and_uk_only() -> None: - engine = create_engine("sqlite://") - V2_METADATA.create_all(engine) +def test_user_primary_country_must_be_a_listed_country() -> None: + engine = _relational_sqlite_engine() with Session(engine) as session: user = User( @@ -126,6 +131,73 @@ def test_user_primary_country_can_change_between_us_and_uk_only() -> None: engine.dispose() +def test_a_country_row_lets_records_use_a_household_only_country() -> None: + engine = _relational_sqlite_engine() + household_only_country = Country( + id="be", + household_enabled=True, + society_wide_enabled=False, + ) + + with Session(engine) as session: + session.add(User(primary_country="be")) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.add(household_only_country) + session.commit() + user = User(primary_country="be") + household = Household( + country_id="be", + household_data={"people": {}}, + canonicalization_version=1, + content_hash="b" * 64, + ) + session.add_all([user, household]) + session.flush() + session.add( + UserHouseholdAssociation( + user_id=user.id, + household_id=household.id, + country_id="be", + ) + ) + session.commit() + + stored_country = session.exec(select(UserHouseholdAssociation.country_id)).one() + assert stored_country == "be" + + engine.dispose() + + +def test_a_listed_country_cannot_be_deleted_while_records_use_it() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add(User(primary_country="uk")) + session.commit() + + session.delete(session.get(Country, "uk")) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_a_society_wide_country_must_also_be_household_capable() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add( + Country(id="be", household_enabled=False, society_wide_enabled=True) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + def test_only_unauthenticated_users_may_omit_a_primary_country() -> None: engine = create_engine("sqlite://") V2_METADATA.create_all(engine) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 20947ffcb..baa0e2590 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -8,6 +8,7 @@ from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.models import ( V2_METADATA, + Country, DatasetVersion, Dynamic, Household, @@ -29,6 +30,18 @@ UserType, ) +# Every column that names the country a record belongs to. Each references the +# countries table instead of checking a fixed list of country IDs. +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policies", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) + RUN_OUTPUT_TABLES = frozenset( { "aggregates", @@ -48,6 +61,7 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: expected_modules = { + Country: "countries", User: "users", UserIdentity: "users", Policy: "policies", @@ -354,8 +368,8 @@ def test_standard_users_require_a_supported_primary_country() -> None: for constraint in users.constraints if isinstance(constraint, sa.CheckConstraint) } + # The countries table, not a check, lists the countries a user may choose. assert checks == { - "ck_users_primary_country": "primary_country IN ('us', 'uk')", "ck_users_user_type": "user_type IN ('unauthenticated', 'standard')", "ck_users_standard_primary_country": ( "user_type = 'unauthenticated' OR primary_country IS NOT NULL" @@ -474,7 +488,8 @@ def test_named_checks_and_required_indexes_cover_core_invariants() -> None: assert { "ck_regions_required_filter_values", "ck_simulations_type_input", - "ck_users_primary_country", + "ck_users_standard_primary_country", + "ck_countries_society_wide_requires_household", "ck_report_runs_terminal_completion", "ck_parameter_values_single_owner", }.issubset(constraint_names) @@ -607,10 +622,11 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None assert table.c.label.nullable assert { foreign_key.target_fullname for foreign_key in simulations.foreign_keys - } == {"users.id"} + } == {"users.id", "countries.id"} assert {foreign_key.target_fullname for foreign_key in reports.foreign_keys} == { "users.id", "user_policies.id", + "countries.id", } user_policy_key = next(iter(reports.c.user_policy_id.foreign_keys)) assert user_policy_key.ondelete == "SET NULL" @@ -635,3 +651,71 @@ def test_user_policy_records_v3_reform_fields() -> None: assert "current-policy" in str(baseline_check.sqltext) postgres_type = policies.c.provenance.type.dialect_impl(postgresql.dialect()) assert isinstance(postgres_type, postgresql.JSONB) + + +def test_countries_record_household_and_society_wide_capability() -> None: + countries = V2_METADATA.tables["countries"] + + assert countries.c.keys() == ["id", "household_enabled", "society_wide_enabled"] + assert countries.primary_key.columns.keys() == ["id"] + assert countries.primary_key.name == "pk_countries" + assert countries.c.id.type.length == 2 + for flag_name in ("household_enabled", "society_wide_enabled"): + assert isinstance(countries.c[flag_name].type, sa.Boolean) + assert not countries.c[flag_name].nullable + assert countries.c[flag_name].server_default is None + checks = { + constraint.name: str(constraint.sqltext) + for constraint in countries.constraints + if isinstance(constraint, sa.CheckConstraint) + } + # Society-wide reports need household calculations, so a society-wide + # country is always household capable too. + assert checks == { + "ck_countries_society_wide_requires_household": ( + "household_enabled OR NOT society_wide_enabled" + ), + } + + +def test_country_columns_reference_the_countries_table_instead_of_a_fixed_list() -> ( + None +): + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + table = V2_METADATA.tables[table_name] + country_keys = [ + foreign_key + for foreign_key in table.c[column_name].foreign_keys + if foreign_key.column.table.name == "countries" + ] + assert len(country_keys) == 1, table_name + assert country_keys[0].target_fullname == "countries.id" + assert country_keys[0].ondelete == "RESTRICT" + assert country_keys[0].constraint.name == ( + f"fk_{table_name}_{column_name}_countries" + ) + fixed_country_checks = [ + constraint.name + for constraint in table.constraints + if isinstance(constraint, sa.CheckConstraint) + and "'us', 'uk'" in str(constraint.sqltext) + ] + assert fixed_country_checks == [], table_name + + +def test_v1_copy_mappings_keep_their_us_and_uk_checks() -> None: + # These tables pair v1 records with their v2 copies. Each already + # references a v2 record of the same country, and they are removed with + # dual-write, so they keep their fixed checks. + for table_name in ( + "legacy_household_mappings", + "legacy_policy_mappings", + "legacy_user_policy_mappings", + ): + table = V2_METADATA.tables[table_name] + assert f"ck_{table_name}_country" in { + constraint.name for constraint in table.constraints + } + assert "countries.id" not in { + foreign_key.target_fullname for foreign_key in table.foreign_keys + } diff --git a/tests/unit/v2/user_service_support.py b/tests/unit/v2/user_service_support.py index ce54237e3..209a63dc5 100644 --- a/tests/unit/v2/user_service_support.py +++ b/tests/unit/v2/user_service_support.py @@ -31,6 +31,7 @@ from policyengine_api.services.v2.legacy_refs.validators import ( require_legacy_references, ) +from tests.fixtures.v2_countries import add_listed_countries if TYPE_CHECKING: from collections.abc import Collection, Iterator, Mapping @@ -128,6 +129,8 @@ def begin_transaction(connection) -> None: V2_METADATA.create_all(engine) sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + with sessions() as session: + add_listed_countries(session) with sessions.begin() as session: model = TaxBenefitModel(name="policyengine-us") version = TaxBenefitModelVersion( From 0d30696ebab2e0350454cea3429d98945bbbb58b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:21:36 +0400 Subject: [PATCH 003/118] Serve the countries table at GET /v2/countries `GET /v2/countries` lists every row of the countries table, ordered by ID: each country's `id`, `household_enabled` and `society_wide_enabled`. It takes no query parameters (any is refused with a 422), needs no user credential, and answers 503 when the v2 database is not configured or cannot be reached, and a secret-safe 500 for anything else, through the shared v2 error mapping. Requests are logged under the `metadata` route group. The route follows the v2 layout: a `countries` route package, and a service package with a read connector, a session container, a transformation and a read type. The Public API's runtime role gets SELECT on `countries` in the privilege manifest, which the deploy grants after the schema upgrade. The migration contract registry gains the `country_selection_v2` workflow, and the cross-database integration workflow runs the new integration test. Tests (failed first: the route and service tests could not import `policyengine_api.services.v2.countries`, the OpenAPI test did not find `/v2/countries`, the generated contract files were stale, and the integration workflow did not name the new test): route responses, the query refusal, 405 for other methods, 503 and 500 mapping and the OpenAPI document; the service against SQLite; and, against migrated disposable Postgres as the runtime role, the route returning the two rows the revision inserted. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/v2-integration-check.yml | 4 +- changelog.d/v2-countries-route.added.md | 1 + docs/engineering/migration-contracts.md | 15 +- docs/generated/migration_contracts.json | 24 ++- policyengine_api/data/v2/privileges.py | 5 +- .../fastapi_routes/dependencies.py | 23 ++ .../fastapi_routes/v2/countries/__init__.py | 1 + .../v2/countries/response_models.py | 71 +++++++ .../fastapi_routes/v2/countries/routes.py | 84 ++++++++ policyengine_api/fastapi_routes/v2/routes.py | 4 + policyengine_api/migration_registry.py | 1 + policyengine_api/query_parameters.py | 4 + .../services/v2/countries/__init__.py | 1 + .../countries/database_connectors/__init__.py | 1 + .../v2/countries/database_connectors/reads.py | 12 ++ .../services/v2/countries/database_session.py | 26 +++ .../services/v2/countries/services.py | 29 +++ .../services/v2/countries/transformations.py | 19 ++ .../services/v2/countries/types.py | 19 ++ pyproject.toml | 2 + tests/contract/registry.py | 18 ++ .../test_app_v2_workflow_contracts.py | 2 + tests/integration/test_v2_country_routes.py | 71 +++++++ tests/unit/test_alembic_workflows.py | 1 + .../unit/test_migration_contract_artifacts.py | 5 +- tests/unit/v2/test_country_routes.py | 198 ++++++++++++++++++ tests/unit/v2/test_country_service.py | 49 +++++ tests/unit/v2/test_metadata_routes.py | 1 + 28 files changed, 680 insertions(+), 11 deletions(-) create mode 100644 changelog.d/v2-countries-route.added.md create mode 100644 policyengine_api/fastapi_routes/v2/countries/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/countries/response_models.py create mode 100644 policyengine_api/fastapi_routes/v2/countries/routes.py create mode 100644 policyengine_api/services/v2/countries/__init__.py create mode 100644 policyengine_api/services/v2/countries/database_connectors/__init__.py create mode 100644 policyengine_api/services/v2/countries/database_connectors/reads.py create mode 100644 policyengine_api/services/v2/countries/database_session.py create mode 100644 policyengine_api/services/v2/countries/services.py create mode 100644 policyengine_api/services/v2/countries/transformations.py create mode 100644 policyengine_api/services/v2/countries/types.py create mode 100644 tests/integration/test_v2_country_routes.py create mode 100644 tests/unit/v2/test_country_routes.py create mode 100644 tests/unit/v2/test_country_service.py diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 3f402ae80..79d264eee 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -77,8 +77,8 @@ jobs: run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py env: RUN_V2_CATALOG_COMPATIBILITY: "1" - - name: Test v2 metadata publication and resource routes - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + - name: Test v2 metadata publication, resource routes and the countries list + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations diff --git a/changelog.d/v2-countries-route.added.md b/changelog.d/v2-countries-route.added.md new file mode 100644 index 000000000..66b8d8e46 --- /dev/null +++ b/changelog.d/v2-countries-route.added.md @@ -0,0 +1 @@ +Added `GET /v2/countries`, which lists every row of the countries table: each served country's ID and whether it supports household and society-wide calculations. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..f7975a813 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 16 | -| request count | 89 | +| workflow count | 17 | +| request count | 90 | | db entity count | 6 | | sim flow count | 3 | @@ -18,7 +18,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | @@ -190,6 +190,15 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/us/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | | `GET` | `/uk/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | +### `country_selection_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Countries table and one country list in code + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `GET` | `/v2/countries` | 200 | `metadata` | none | `status`, `message`, `result.items` | + ### `metadata_resources_v2_preview` - Current contract: `typed_v2_resources` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..9ee0774f5 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 90, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 16 + "workflow_count": 17 }, "route_groups": [ { @@ -31,6 +31,7 @@ "name": "metadata", "path_segments": [ "metadata", + "countries", "datasets", "economy-options", "parameter-values", @@ -1348,6 +1349,25 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Countries table and one country list in code", + "name": "country_selection_v2", + "requests": [ + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/countries", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.items" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "Later metadata read cutover and v2 route-prefix removal", diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 6fd0cc750..cec2fa66e 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -132,8 +132,9 @@ def _access( # Stage 10 saved-policy mirror. "legacy_user_mappings": _access(runtime=READ_WRITE), "legacy_user_policy_mappings": _access(runtime=READ_WRITE), - # Served countries; the migration that creates the table inserts its rows. - "countries": _access(), + # Served countries; the migration that creates the table inserts its rows, + # and the Public API reads them. + "countries": _access(runtime=READ), # Users and their saved work (Stage 12A). "user_household_associations": _access(runtime=READ_WRITE), "user_identities": _access(runtime=READ_WRITE), diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..b2fcbc0ff 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -22,6 +22,7 @@ from policyengine_api.services.v2.associations.types import ( LegacyReferenceChecker, ) + from policyengine_api.services.v2.countries.types import CountryRead from policyengine_api.services.v2.households.types import ( HouseholdPage, HouseholdRead, @@ -312,6 +313,14 @@ def get_economy_options( ... +class V2CountryResourceService(Protocol): + """Route-independent countries list read for one request.""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and its capability, ordered by ID.""" + ... + + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" @@ -742,6 +751,18 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: ) +def _default_v2_country_service_factory() -> V2CountryResourceService: + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -917,6 +938,7 @@ class NativeRouteDependencies: metadata_reader_factory: Callable[[], MetadataReader] specification_provider: Callable[[], JSONObject] v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None + v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( @@ -947,6 +969,7 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_country_service_factory=_default_v2_country_service_factory, v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), diff --git a/policyengine_api/fastapi_routes/v2/countries/__init__.py b/policyengine_api/fastapi_routes/v2/countries/__init__.py new file mode 100644 index 000000000..3d7f8eaf0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for the v2 countries list.""" diff --git a/policyengine_api/fastapi_routes/v2/countries/response_models.py b/policyengine_api/fastapi_routes/v2/countries/response_models.py new file mode 100644 index 000000000..3ea3c7ee8 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/response_models.py @@ -0,0 +1,71 @@ +"""Strict HTTP response models for the native v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import BaseModel, ConfigDict + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse + +if TYPE_CHECKING: + from policyengine_api.services.v2.countries.types import CountryRead + + +class StrictCountryAPIModel(BaseModel): + """Strict response base with dataclass conversion support.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + +class CountryItem(StrictCountryAPIModel): + """One served country and the calculations it supports. + + ``id`` is the value other v2 resources take as ``country_id``. + ``household_enabled`` means household calculations and saved households + are available; ``society_wide_enabled`` means society-wide reports are, + and is only set for a country that is also household enabled. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool + + @classmethod + def from_read(cls, country: CountryRead) -> CountryItem: + """Build the response item from the service read record.""" + return cls.model_validate(country) + + +class CountryListResult(StrictCountryAPIModel): + """Every served country, ordered by ``id``.""" + + items: list[CountryItem] + + +class CountryListResponse(StrictCountryAPIModel): + """Success envelope: ``status`` "ok", a null ``message`` and the countries.""" + + status: Literal["ok"] = "ok" + message: None = None + result: CountryListResult + + +COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 405: { + "model": V2ErrorResponse, + "description": "The countries list supports GET only.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request has query parameters; the list takes none.", + }, + 500: { + "model": V2ErrorResponse, + "description": "The countries query failed internally.", + }, + 503: { + "model": V2ErrorResponse, + "description": "The v2 database is not configured or cannot be reached.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/countries/routes.py b/policyengine_api/fastapi_routes/v2/countries/routes.py new file mode 100644 index 000000000..279aaf54f --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/routes.py @@ -0,0 +1,84 @@ +"""Native FastAPI route for the v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import APIRouter, Depends +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_country_service_factory, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.countries.response_models import ( + COUNTRY_ERROR_RESPONSES, + CountryItem, + CountryListResponse, + CountryListResult, +) +from policyengine_api.fastapi_routes.v2.errors import V2ErrorMapping, run_v2_operation +from policyengine_api.query_parameters import CountryListQuery + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2CountryResourceService, + ) + + +CountryListQueryDependency = Annotated[ + CountryListQuery, Depends(query_dependency(CountryListQuery)) +] + +COUNTRY_ERROR_MAPPINGS = ( + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 countries are unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2CountryResourceService]: + if dependencies.v2_country_service_factory is not None: + return dependencies.v2_country_service_factory + return _default_v2_country_service_factory + + +def build_v2_country_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the countries route without opening a database connection.""" + router = APIRouter(prefix="/v2", responses=COUNTRY_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.get( + "/countries", + response_model=CountryListResponse, + summary="List the served countries and the calculations each supports", + ) + def list_countries( + _query: CountryListQueryDependency, + ) -> CountryListResponse | JSONResponse: + def read() -> CountryListResponse: + countries = service_factory().list_countries() + return CountryListResponse( + result=CountryListResult( + items=[CountryItem.from_read(country) for country in countries] + ) + ) + + return run_v2_operation( + read, + COUNTRY_ERROR_MAPPINGS, + fallback_message="V2 countries query failed", + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..fa39d7783 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -7,6 +7,9 @@ from fastapi import APIRouter, Request from starlette.responses import JSONResponse +from policyengine_api.fastapi_routes.v2.countries.routes import ( + build_v2_country_router, +) from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse from policyengine_api.fastapi_routes.v2.households.routes import ( build_v2_household_router, @@ -53,6 +56,7 @@ def build_v2_router( router.include_router(build_v2_user_policy_router(dependencies)) router.include_router(build_v2_household_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) + router.include_router(build_v2_country_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..5daadd05a 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -37,6 +37,7 @@ class RouteGroupConfig: name="metadata", path_segments=( "metadata", + "countries", "datasets", "economy-options", "parameter-values", diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 619c6924b..086d33131 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -214,6 +214,10 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CountryListQuery(StrictQueryParameters): + """Query contract for the countries list, which takes no parameters.""" + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/countries/__init__.py b/policyengine_api/services/v2/countries/__init__.py new file mode 100644 index 000000000..f0f7b972b --- /dev/null +++ b/policyengine_api/services/v2/countries/__init__.py @@ -0,0 +1 @@ +"""Application services for the v2 countries list.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/__init__.py b/policyengine_api/services/v2/countries/database_connectors/__init__.py new file mode 100644 index 000000000..01c1fa018 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for the v2 countries service.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/reads.py b/policyengine_api/services/v2/countries/database_connectors/reads.py new file mode 100644 index 000000000..5741285d5 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/reads.py @@ -0,0 +1,12 @@ +"""Database selections used by the v2 countries list.""" + +from __future__ import annotations + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Country + + +def read_countries(session: Session) -> list[Country]: + """Return every countries row, ordered by country ID.""" + return list(session.exec(select(Country).order_by(col(Country.id))).all()) diff --git a/policyengine_api/services/v2/countries/database_session.py b/policyengine_api/services/v2/countries/database_session.py new file mode 100644 index 000000000..455d4aa39 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for the v2 countries service.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class CountryDatabaseSession: + """Open the read sessions the countries service uses.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session diff --git a/policyengine_api/services/v2/countries/services.py b/policyengine_api/services/v2/countries/services.py new file mode 100644 index 000000000..a7373636d --- /dev/null +++ b/policyengine_api/services/v2/countries/services.py @@ -0,0 +1,29 @@ +"""Router-facing service for the v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.database_connectors.reads import ( + read_countries, +) +from policyengine_api.services.v2.countries.transformations import country_read + +if TYPE_CHECKING: + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.types import CountryRead + + +class V2CountryService: + """List the served countries with the calculations each supports.""" + + def __init__(self, database_session: CountryDatabaseSession) -> None: + """Read through ``database_session``, which opens one session per call.""" + self._database_session = database_session + + def list_countries(self) -> list[CountryRead]: + """Return every countries row, ordered by country ID.""" + with self._database_session.read() as session: + return [country_read(country) for country in read_countries(session)] diff --git a/policyengine_api/services/v2/countries/transformations.py b/policyengine_api/services/v2/countries/transformations.py new file mode 100644 index 000000000..fd1d65fe2 --- /dev/null +++ b/policyengine_api/services/v2/countries/transformations.py @@ -0,0 +1,19 @@ +"""Conversions between stored countries rows and service read records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.types import CountryRead + +if TYPE_CHECKING: + from policyengine_api.data.v2.models import Country + + +def country_read(country: Country) -> CountryRead: + """Convert one stored countries row to the service's read record.""" + return CountryRead( + id=country.id, + household_enabled=country.household_enabled, + society_wide_enabled=country.society_wide_enabled, + ) diff --git a/policyengine_api/services/v2/countries/types.py b/policyengine_api/services/v2/countries/types.py new file mode 100644 index 000000000..b4a1e74fc --- /dev/null +++ b/policyengine_api/services/v2/countries/types.py @@ -0,0 +1,19 @@ +"""Framework-independent data exchanged by the v2 countries layers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CountryRead: + """One served country and the calculations it supports. + + ``household_enabled`` allows household calculations and saved households; + ``society_wide_enabled`` allows society-wide reports, and implies + ``household_enabled``. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool diff --git a/pyproject.toml b/pyproject.toml index ed101271e..10a5cd6f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,6 +212,8 @@ banned-module-level-imports = [ "policyengine", "policyengine_api.data.v2.database", "policyengine_api.libs.auth0_management", + "policyengine_api.services.v2.countries.database_session", + "policyengine_api.services.v2.countries.services", "policyengine_api.services.v2.households.database_session", "policyengine_api.services.v2.households.services", "policyengine_api.services.v2.legacy_refs.database_session", diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..a04522cbf 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -587,6 +587,24 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="country_selection_v2", + current_contract="typed_v2_resources", + future_owner_pr="Countries table and one country list in code", + requests=( + ContractRequest( + method="GET", + path="/v2/countries", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + ), + route_group="metadata", + ), + ), + ), WorkflowContract( name="metadata_resources_v2_preview", current_contract="typed_v2_resources", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index ff861db14..f74640678 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -20,6 +20,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", @@ -31,6 +32,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "typed_v2_resources" if workflow.name in { + "country_selection_v2", "metadata_resources_v2_preview", "policy_resources_v2", "user_policy_associations_v2", diff --git a/tests/integration/test_v2_country_routes.py b/tests/integration/test_v2_country_routes.py new file mode 100644 index 000000000..f31c28b5d --- /dev/null +++ b/tests/integration/test_v2_country_routes.py @@ -0,0 +1,71 @@ +"""Serve the v2 countries list from a migrated disposable Postgres database. + +The route runs as the Public API's runtime role, so the test proves the role's +declared privileges cover it (rule +``api.v2.runtime-privileges-tested-as-runtime-role``). +""" + +from __future__ import annotations + +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.constants import REPO +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from tests.fixtures.v2_countries import LISTED_COUNTRY_ROWS + + +def test_postgres_countries_list_the_rows_the_revision_inserted( + managed_v2_roles, + v2_engine_as_role, +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + sessions = sessionmaker(runtime_engine, class_=Session) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: V2CountryService( + CountryDatabaseSession(sessions) + ), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), + dependencies=dependencies, + route_settings=settings, + ) + ) + + try: + response = client.get("/v2/countries") + finally: + runtime_engine.dispose() + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": {"items": list(LISTED_COUNTRY_ROWS)}, + } diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 797c82848..fa69a8378 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -221,6 +221,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow + assert "test_v2_country_routes.py" in workflow assert "test_v2_policy_persistence.py" in workflow assert "test_v1_policy_dual_write.py" in workflow assert "test_mysql_policy_dual_write.py" in workflow diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..62ddabd71 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -13,8 +13,8 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 16, - "request_count": 89, + "workflow_count": 17, + "request_count": 90, "db_entity_count": 6, "sim_flow_count": 3, } @@ -31,6 +31,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", diff --git a/tests/unit/v2/test_country_routes.py b/tests/unit/v2/test_country_routes.py new file mode 100644 index 000000000..7fd2e2b91 --- /dev/null +++ b/tests/unit/v2/test_country_routes.py @@ -0,0 +1,198 @@ +"""Route coverage for the native v2 countries list.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from flask import Flask, jsonify +from sqlalchemy.exc import OperationalError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, + infer_route_group, +) +from policyengine_api.services.v2.countries.types import CountryRead + +UNITED_KINGDOM = CountryRead(id="uk", household_enabled=True, society_wide_enabled=True) +UNITED_STATES = CountryRead(id="us", household_enabled=True, society_wide_enabled=True) +HOUSEHOLD_ONLY_COUNTRY = CountryRead( + id="be", household_enabled=True, society_wide_enabled=False +) + + +class FakeCountryService: + """Return fixed countries, or raise ``error``, and count the calls.""" + + def __init__( + self, + countries: list[CountryRead], + *, + error: Exception | None = None, + ) -> None: + self.countries = countries + self.error = error + self.list_calls = 0 + + def list_countries(self) -> list[CountryRead]: + self.list_calls += 1 + if self.error is not None: + raise self.error + return self.countries + + +def _client(service: FakeCountryService) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app(flask_app, dependencies=dependencies, route_settings=settings), + raise_server_exceptions=False, + ) + return client, flask_calls + + +def test_countries_list_every_country_with_its_capability() -> None: + service = FakeCountryService( + [HOUSEHOLD_ONLY_COUNTRY, UNITED_KINGDOM, UNITED_STATES] + ) + client, flask_calls = _client(service) + + response = client.get("/v2/countries") + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "items": [ + { + "id": "be", + "household_enabled": True, + "society_wide_enabled": False, + }, + { + "id": "uk", + "household_enabled": True, + "society_wide_enabled": True, + }, + { + "id": "us", + "household_enabled": True, + "society_wide_enabled": True, + }, + ] + }, + } + assert service.list_calls == 1 + assert flask_calls["count"] == 0 + + +def test_countries_list_needs_no_user_credential() -> None: + client, _flask_calls = _client(FakeCountryService([UNITED_STATES])) + + response = client.get("/v2/countries", headers={"Authorization": ""}) + + assert response.status_code == 200 + + +def test_countries_list_rejects_query_parameters() -> None: + service = FakeCountryService([UNITED_STATES]) + client, _flask_calls = _client(service) + + response = client.get("/v2/countries?country_id=us") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert service.list_calls == 0 + + +@pytest.mark.parametrize( + "error", + [ + V2ConfigurationError("V2_RUNTIME_DATABASE_URL is required"), + OperationalError("SELECT", {}, Exception("connection refused")), + ], + ids=["configuration", "database"], +) +def test_countries_list_reports_an_unavailable_database_as_503( + error: Exception, +) -> None: + client, _flask_calls = _client(FakeCountryService([], error=error)) + + response = client.get("/v2/countries") + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 countries are unavailable", + } + + +def test_countries_list_hides_unexpected_failures_behind_a_500() -> None: + client, _flask_calls = _client( + FakeCountryService([], error=RuntimeError("secret detail")) + ) + + response = client.get("/v2/countries") + + assert response.status_code == 500 + assert response.json() == { + "status": "error", + "message": "V2 countries query failed", + } + + +def test_countries_list_supports_get_only() -> None: + service = FakeCountryService([UNITED_STATES]) + client, flask_calls = _client(service) + + response = client.post("/v2/countries", json={}) + + assert response.status_code == 405 + assert service.list_calls == 0 + assert flask_calls["count"] == 0 + + +def test_countries_list_is_documented_in_the_v2_openapi_document() -> None: + client, _flask_calls = _client(FakeCountryService([])) + + document = client.get("/v2/openapi.json").json() + operation = document["paths"]["/v2/countries"]["get"] + + assert operation.get("parameters", []) == [] + assert operation["responses"]["200"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/CountryListResponse" + } + assert {"422", "500", "503"} <= operation["responses"].keys() + item_schema = document["components"]["schemas"]["CountryItem"] + assert item_schema["required"] == [ + "id", + "household_enabled", + "society_wide_enabled", + ] + assert item_schema["additionalProperties"] is False + + +def test_countries_requests_are_logged_with_the_metadata_route_group() -> None: + assert infer_route_group("/v2/countries") == "metadata" diff --git a/tests/unit/v2/test_country_service.py b/tests/unit/v2/test_country_service.py new file mode 100644 index 000000000..aae0ff982 --- /dev/null +++ b/tests/unit/v2/test_country_service.py @@ -0,0 +1,49 @@ +"""SQLite-backed coverage for the v2 countries read service.""" + +from __future__ import annotations + +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import V2_METADATA, Country +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.types import CountryRead +from tests.fixtures.v2_countries import add_listed_countries + + +def _service_with_countries(*extra_countries: Country) -> V2CountryService: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + sessions = sessionmaker(engine, class_=Session) + with sessions() as session: + add_listed_countries(session) + session.add_all(extra_countries) + session.commit() + return V2CountryService(CountryDatabaseSession(sessions)) + + +def test_countries_are_listed_by_id_with_their_capability_flags() -> None: + service = _service_with_countries( + Country(id="be", household_enabled=True, society_wide_enabled=False) + ) + + countries = service.list_countries() + + assert countries == [ + CountryRead(id="be", household_enabled=True, society_wide_enabled=False), + CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ] + + +def test_an_empty_countries_table_lists_no_countries() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + assert service.list_countries() == [] diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..1c2e542a5 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -538,6 +538,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/variables/{variable_id}", } native_paths = { + "/v2/countries", "/v2/households", "/v2/households/{household_id}", "/v2/policies", From f9a19c11e663215740ea7437ca168e58aca2c583 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:30:54 +0400 Subject: [PATCH 004/118] Refuse to start when the countries table and the country list differ `policyengine_api.asgi` now checks, while the application loads, that the v2 `countries` table holds one row for each country in `policyengine_api/supported_countries.py` and no other. A worker that finds a difference raises `CountryListMismatchError`, naming the listed countries without a row and the rows not listed, so it never serves a request; on Cloud Run the revision then fails its health checks. The check runs on Cloud Run (`K_SERVICE` is set) and wherever a v2 runtime database URL or its secret resource is configured. A local or test startup without v2 configuration opens no connection. A missing table, a missing privilege and unusable v2 configuration also stop startup. A database that cannot be reached is logged, by error type only, and startup continues: the v1 routes do not read the v2 database, and their copies into it already continue without it. The countries service gains `verify_countries_match_code_list`, which reads the row IDs through a new read connector and compares them in a validator. Tests (failed first: `policyengine_api.services.v2.countries.validators` did not exist): the comparison in every direction; the service against SQLite; the startup function's selection, each failure that stops startup, and the logged skip without the error's detail; importing the ASGI module with an unusable v2 runtime URL failing without printing the password; and, against migrated disposable Postgres as the runtime role, the check passing on the revision's rows and failing once an extra row exists. Co-Authored-By: Claude Opus 5.5 --- .../v2-countries-startup-check.added.md | 1 + docs/engineering/skills/alembic-migrations.md | 6 +- policyengine_api/asgi.py | 5 + .../v2/countries/database_connectors/reads.py | 7 +- .../services/v2/countries/services.py | 24 ++- .../services/v2/countries/validators.py | 48 +++++ policyengine_api/startup_country_check.py | 96 +++++++++ policyengine_api/supported_countries.py | 3 +- tests/integration/test_v2_country_routes.py | 46 +++- tests/unit/test_startup_country_check.py | 196 ++++++++++++++++++ tests/unit/v2/test_country_service.py | 38 +++- tests/unit/v2/test_country_validators.py | 68 ++++++ 12 files changed, 528 insertions(+), 10 deletions(-) create mode 100644 changelog.d/v2-countries-startup-check.added.md create mode 100644 policyengine_api/services/v2/countries/validators.py create mode 100644 policyengine_api/startup_country_check.py create mode 100644 tests/unit/test_startup_country_check.py create mode 100644 tests/unit/v2/test_country_validators.py diff --git a/changelog.d/v2-countries-startup-check.added.md b/changelog.d/v2-countries-startup-check.added.md new file mode 100644 index 000000000..66f42aad2 --- /dev/null +++ b/changelog.d/v2-countries-startup-check.added.md @@ -0,0 +1 @@ +The Public API refuses to start when the v2 `countries` table and its country list in code name different countries. The check runs on Cloud Run and wherever a v2 runtime database is configured; an unreachable database is logged and skipped. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 7a5819fa9..2111ace59 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -316,5 +316,7 @@ existing rows, the downgrade, a downgrade refused by another country's records, and the re-upgrade. The table must list exactly the countries in -`policyengine_api/supported_countries.py`. Listing another country later -needs a revision that inserts its row, and the same human migration decision. +`policyengine_api/supported_countries.py`: the Public API refuses to start +when they differ (`policyengine_api/startup_country_check.py`). Listing +another country later needs a revision that inserts its row, and the same +human migration decision. diff --git a/policyengine_api/asgi.py b/policyengine_api/asgi.py index 1f981c56a..fc5809cf1 100644 --- a/policyengine_api/asgi.py +++ b/policyengine_api/asgi.py @@ -10,6 +10,7 @@ from policyengine_api.observability import get_runtime from policyengine_api.readiness import mark_not_ready, mark_ready from policyengine_api.runtime_cache.client import close_runtime_cache_clients +from policyengine_api.startup_country_check import check_countries_table_at_startup from policyengine_api.warmup import run_startup_warmup @@ -24,6 +25,10 @@ def _close_runtime_resources() -> None: shutdown_callback=_close_runtime_resources, ) +# Refuse to start when the v2 countries table and the country list in code +# disagree (see policyengine_api.startup_country_check). +check_countries_table_at_startup() + # Warm the simulation machinery before serving (see policyengine_api.warmup). # POLICYENGINE_API_STARTUP_WARMUP=0 skips it. if os.environ.get("POLICYENGINE_API_STARTUP_WARMUP", "1") != "0": diff --git a/policyengine_api/services/v2/countries/database_connectors/reads.py b/policyengine_api/services/v2/countries/database_connectors/reads.py index 5741285d5..a13a8f3e4 100644 --- a/policyengine_api/services/v2/countries/database_connectors/reads.py +++ b/policyengine_api/services/v2/countries/database_connectors/reads.py @@ -1,4 +1,4 @@ -"""Database selections used by the v2 countries list.""" +"""Database selections used by the v2 countries list and the startup check.""" from __future__ import annotations @@ -10,3 +10,8 @@ def read_countries(session: Session) -> list[Country]: """Return every countries row, ordered by country ID.""" return list(session.exec(select(Country).order_by(col(Country.id))).all()) + + +def read_country_ids(session: Session) -> list[str]: + """Return the ID of every countries row, ordered.""" + return list(session.exec(select(Country.id).order_by(col(Country.id))).all()) diff --git a/policyengine_api/services/v2/countries/services.py b/policyengine_api/services/v2/countries/services.py index a7373636d..96e0ad601 100644 --- a/policyengine_api/services/v2/countries/services.py +++ b/policyengine_api/services/v2/countries/services.py @@ -1,4 +1,4 @@ -"""Router-facing service for the v2 countries list.""" +"""Router-facing and startup service for the v2 countries table.""" from __future__ import annotations @@ -6,10 +6,16 @@ from policyengine_api.services.v2.countries.database_connectors.reads import ( read_countries, + read_country_ids, ) from policyengine_api.services.v2.countries.transformations import country_read +from policyengine_api.services.v2.countries.validators import ( + validate_countries_match_code_list, +) if TYPE_CHECKING: + from collections.abc import Iterable + from policyengine_api.services.v2.countries.database_session import ( CountryDatabaseSession, ) @@ -17,7 +23,7 @@ class V2CountryService: - """List the served countries with the calculations each supports.""" + """List the served countries, and check them against the list in code.""" def __init__(self, database_session: CountryDatabaseSession) -> None: """Read through ``database_session``, which opens one session per call.""" @@ -27,3 +33,17 @@ def list_countries(self) -> list[CountryRead]: """Return every countries row, ordered by country ID.""" with self._database_session.read() as session: return [country_read(country) for country in read_countries(session)] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these. + + The application runs this at startup with the country list in code. + """ + with self._database_session.read() as session: + stored_country_ids = read_country_ids(session) + validate_countries_match_code_list( + stored_country_ids=stored_country_ids, + listed_country_ids=listed_country_ids, + ) diff --git a/policyengine_api/services/v2/countries/validators.py b/policyengine_api/services/v2/countries/validators.py new file mode 100644 index 000000000..fbb30ca5b --- /dev/null +++ b/policyengine_api/services/v2/countries/validators.py @@ -0,0 +1,48 @@ +"""Checks of the stored countries rows against the country list in code.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class CountryListMismatchError(RuntimeError): + """The countries table and the country list in code name different countries. + + ``listed_without_row`` holds the listed countries that have no row, and + ``rows_not_listed`` the rows for countries the list does not name, each + sorted by ID. + """ + + def __init__( + self, + *, + listed_without_row: tuple[str, ...], + rows_not_listed: tuple[str, ...], + ) -> None: + """Record both differences and describe them in the message.""" + self.listed_without_row = listed_without_row + self.rows_not_listed = rows_not_listed + super().__init__( + "the countries table does not match the country list in code: " + f"listed without a row: {', '.join(listed_without_row) or 'none'}; " + f"rows not listed: {', '.join(rows_not_listed) or 'none'}" + ) + + +def validate_countries_match_code_list( + *, + stored_country_ids: Iterable[str], + listed_country_ids: Iterable[str], +) -> None: + """Raise ``CountryListMismatchError`` unless both name the same countries.""" + stored = set(stored_country_ids) + listed = set(listed_country_ids) + if stored == listed: + return + raise CountryListMismatchError( + listed_without_row=tuple(sorted(listed - stored)), + rows_not_listed=tuple(sorted(stored - listed)), + ) diff --git a/policyengine_api/startup_country_check.py b/policyengine_api/startup_country_check.py new file mode 100644 index 000000000..43963d077 --- /dev/null +++ b/policyengine_api/startup_country_check.py @@ -0,0 +1,96 @@ +"""Refuse to start when the v2 countries table and the country list differ. + +Which countries the Public API serves is decided in code +(``policyengine_api.supported_countries``). The v2 ``countries`` table records +what each supports, and must hold one row for each listed country and no +other. ``policyengine_api.asgi`` runs this check while the application loads, +so a disagreement stops the worker before it serves a request. + +The check runs only where a v2 runtime database is selected: on Cloud Run +(``K_SERVICE`` is set), or when a v2 runtime database URL or its secret +resource is configured. A local or test startup without v2 configuration +opens no connection. + +A disagreement, a missing table, a missing privilege or unusable v2 +configuration stops startup. A database that cannot be reached is logged and +startup continues: the v1 routes do not read the v2 database, and their +copies into it already continue without it. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Protocol + +from sqlalchemy.exc import OperationalError + +from policyengine_api.data.v2.settings import ( + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + +logger = logging.getLogger(__name__) + +CLOUD_RUN_SERVICE = "K_SERVICE" +V2_RUNTIME_SELECTION_VARIABLES = ( + CLOUD_RUN_SERVICE, + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, +) + + +class CountryListVerifier(Protocol): + """Compare the stored countries rows with a list of country IDs.""" + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these.""" + ... + + +def _default_country_list_verifier() -> CountryListVerifier: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + +def v2_runtime_database_selected(environ: Mapping[str, str]) -> bool: + """Whether this process runs on Cloud Run or has a v2 runtime database.""" + return any(environ.get(name, "").strip() for name in V2_RUNTIME_SELECTION_VARIABLES) + + +def check_countries_table_at_startup( + environ: Mapping[str, str] | None = None, + verifier_factory: Callable[[], CountryListVerifier] | None = None, +) -> None: + """Raise unless the countries table names exactly the countries in code. + + Raises ``CountryListMismatchError`` on a disagreement, SQLAlchemy's + ``ProgrammingError`` when the table or the privilege to read it is + missing, and ``V2ConfigurationError`` for unusable v2 configuration. An + unreachable database is logged, without its error detail, and skipped. + """ + values = os.environ if environ is None else environ + if not v2_runtime_database_selected(values): + return + factory = verifier_factory or _default_country_list_verifier + try: + factory().verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + except OperationalError as error: + # The error's detail can name the database host, so only its type is + # logged. + logger.warning( + "Skipped the countries table check at startup: the v2 database " + "could not be reached (%s)", + type(error).__name__, + ) diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py index 51a495dd4..15b9d1b60 100644 --- a/policyengine_api/supported_countries.py +++ b/policyengine_api/supported_countries.py @@ -7,7 +7,8 @@ The v2 ``countries`` table records what each country supports, and must hold one row for each listed country and no other: a v2 Alembic revision inserts -the row when a country is added. +the row when a country is added. The application refuses to start when they +differ (``policyengine_api.startup_country_check``). The module imports nothing from the package, so modules that the query models import, such as the catalog package, can import it too. diff --git a/tests/integration/test_v2_country_routes.py b/tests/integration/test_v2_country_routes.py index f31c28b5d..ee6408ddc 100644 --- a/tests/integration/test_v2_country_routes.py +++ b/tests/integration/test_v2_country_routes.py @@ -1,16 +1,18 @@ -"""Serve the v2 countries list from a migrated disposable Postgres database. +"""Serve and check the v2 countries from a migrated disposable Postgres database. -The route runs as the Public API's runtime role, so the test proves the role's -declared privileges cover it (rule +The route and the startup check run as the Public API's runtime role, so the +tests prove the role's declared privileges cover them (rule ``api.v2.runtime-privileges-tested-as-runtime-role``). """ from __future__ import annotations +import pytest from alembic import command from alembic.config import Config from fastapi.testclient import TestClient from flask import Flask +from sqlalchemy import create_engine, text from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool from sqlmodel import Session @@ -26,6 +28,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS from tests.fixtures.v2_countries import LISTED_COUNTRY_ROWS @@ -69,3 +75,37 @@ def test_postgres_countries_list_the_rows_the_revision_inserted( "message": None, "result": {"items": list(LISTED_COUNTRY_ROWS)}, } + + +def test_postgres_countries_match_the_code_list_as_the_runtime_role( + managed_v2_roles, + v2_engine_as_role, + disposable_v2_database_url: str, +) -> None: + """The startup check passes on the revision's rows and fails on an extra row.""" + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(runtime_engine, class_=Session)) + ) + owner_engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + + try: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + with owner_engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('ca', true, false)" + ) + ) + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + assert raised.value.rows_not_listed == ("ca",) + finally: + with owner_engine.begin() as connection: + connection.execute(text("DELETE FROM countries WHERE id = 'ca'")) + owner_engine.dispose() + runtime_engine.dispose() diff --git a/tests/unit/test_startup_country_check.py b/tests/unit/test_startup_country_check.py new file mode 100644 index 000000000..53cefa172 --- /dev/null +++ b/tests/unit/test_startup_country_check.py @@ -0,0 +1,196 @@ +"""The startup check that the v2 countries table matches the country list in code.""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest +from sqlalchemy.exc import OperationalError, ProgrammingError + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.startup_country_check import check_countries_table_at_startup +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + +V2_RUNTIME_SECRET_RESOURCE = "projects/example/secrets/v2-runtime/versions/latest" + + +class FakeCountryListVerifier: + """Record each verification, then raise ``error`` if one is set.""" + + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.verified_country_lists: list[tuple[str, ...]] = [] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + self.verified_country_lists.append(tuple(listed_country_ids)) + if self.error is not None: + raise self.error + + +class CountingVerifierFactory: + """Return one verifier and count how often startup asks for it.""" + + def __init__(self, verifier: FakeCountryListVerifier) -> None: + self.verifier = verifier + self.calls = 0 + + def __call__(self) -> FakeCountryListVerifier: + self.calls += 1 + return self.verifier + + +def test_startup_without_a_v2_runtime_database_skips_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={"FLASK_DEBUG": "1"}, verifier_factory=factory + ) + + assert factory.calls == 0 + + +def test_blank_selection_variables_skip_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={ + "K_SERVICE": "", + "V2_RUNTIME_DATABASE_URL": " ", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": "", + }, + verifier_factory=factory, + ) + + assert factory.calls == 0 + + +@pytest.mark.parametrize( + "environ", + [ + {"K_SERVICE": "policyengine-api"}, + {"V2_RUNTIME_DATABASE_URL": "postgresql+psycopg://configured"}, + {"V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": V2_RUNTIME_SECRET_RESOURCE}, + ], + ids=["cloud-run", "runtime-url", "runtime-secret"], +) +def test_a_selected_v2_runtime_database_is_checked_against_the_code_list( + environ: dict[str, str], +) -> None: + verifier = FakeCountryListVerifier() + + check_countries_table_at_startup( + environ=environ, verifier_factory=CountingVerifierFactory(verifier) + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + + +@pytest.mark.parametrize( + "error", + [ + CountryListMismatchError(listed_without_row=("us",), rows_not_listed=()), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception('relation "countries" does not exist'), + ), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception("permission denied for table countries"), + ), + ], + ids=["mismatch", "missing-table", "missing-privilege"], +) +def test_a_failed_check_stops_startup(error: Exception) -> None: + verifier = FakeCountryListVerifier(error=error) + + with pytest.raises(type(error)): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + +def test_unusable_v2_configuration_stops_startup() -> None: + def unusable_configuration() -> FakeCountryListVerifier: + msg = "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" + raise V2ConfigurationError(msg) + + with pytest.raises(V2ConfigurationError): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=unusable_configuration, + ) + + +def test_an_unreachable_database_is_logged_and_startup_continues( + caplog: pytest.LogCaptureFixture, +) -> None: + verifier = FakeCountryListVerifier( + error=OperationalError( + "SELECT countries.id FROM countries", + {}, + Exception("connection to db.secret-host failed: password=hunter2"), + ) + ) + + with caplog.at_level(logging.WARNING): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + assert [record.levelno for record in caplog.records] == [logging.WARNING] + message = caplog.records[0].getMessage() + assert "countries table" in message + assert "OperationalError" in message + assert "secret-host" not in message + assert "hunter2" not in message + + +def test_importing_the_asgi_application_runs_the_check(tmp_path: Path) -> None: + """An unusable v2 runtime database URL stops the application from loading.""" + environment = os.environ.copy() + environment.update( + { + "POLICYENGINE_API_STARTUP_WARMUP": "0", + "V2_RUNTIME_DATABASE_URL": ( + "postgresql+psycopg://runtime:hunter2@localhost:5432/postgres" + "?sslmode=require" + ), + "V2_SUPABASE_PROJECT_REF": "abcdefghijklmnopqrst", + "V2_SUPABASE_ENVIRONMENT": "test-startup", + } + ) + for name in ("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", "K_SERVICE"): + environment.pop(name, None) + + result = subprocess.run( + [sys.executable, "-c", "import policyengine_api.asgi"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" in ( + result.stderr + ) + assert "hunter2" not in result.stderr diff --git a/tests/unit/v2/test_country_service.py b/tests/unit/v2/test_country_service.py index aae0ff982..253278599 100644 --- a/tests/unit/v2/test_country_service.py +++ b/tests/unit/v2/test_country_service.py @@ -1,7 +1,8 @@ -"""SQLite-backed coverage for the v2 countries read service.""" +"""SQLite-backed coverage for the v2 countries service: the list and the check.""" from __future__ import annotations +import pytest from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -11,6 +12,10 @@ ) from policyengine_api.services.v2.countries.services import V2CountryService from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS from tests.fixtures.v2_countries import add_listed_countries @@ -47,3 +52,34 @@ def test_an_empty_countries_table_lists_no_countries() -> None: ) assert service.list_countries() == [] + + +def test_rows_for_exactly_the_listed_countries_pass_verification() -> None: + service = _service_with_countries() + + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + +def test_a_row_for_an_unlisted_country_fails_verification() -> None: + service = _service_with_countries( + Country(id="ca", household_enabled=True, society_wide_enabled=False) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.rows_not_listed == ("ca",) + assert raised.value.listed_without_row == () + + +def test_an_empty_countries_table_fails_verification() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.listed_without_row == tuple(sorted(SUPPORTED_COUNTRY_IDS)) diff --git a/tests/unit/v2/test_country_validators.py b/tests/unit/v2/test_country_validators.py new file mode 100644 index 000000000..d56db5c00 --- /dev/null +++ b/tests/unit/v2/test_country_validators.py @@ -0,0 +1,68 @@ +"""The comparison of stored countries rows with the country list in code.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, + validate_countries_match_code_list, +) + + +def test_the_same_countries_in_any_order_match() -> None: + validate_countries_match_code_list( + stored_country_ids=["us", "uk"], + listed_country_ids=("uk", "us"), + ) + + +def test_a_listed_country_without_a_row_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["uk"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("us",) + assert raised.value.rows_not_listed == () + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: us; rows not listed: none" + ) + + +def test_a_row_for_an_unlisted_country_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ca", "uk", "us"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == () + assert raised.value.rows_not_listed == ("ca",) + + +def test_every_difference_is_reported_in_order() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ng", "ca"], + listed_country_ids=("us", "uk"), + ) + + assert raised.value.listed_without_row == ("uk", "us") + assert raised.value.rows_not_listed == ("ca", "ng") + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: uk, us; rows not listed: ca, ng" + ) + + +def test_an_empty_table_misses_every_listed_country() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=[], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("uk", "us") From 6b8cbd2bd1dfdeb09534e8b33a39c42dc034fc49 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:55:08 +0400 Subject: [PATCH 005/118] Stop startup only when the countries table answers with a disagreement Review found that the startup check sorted its failures by exception type in a way that stopped startup for a brief Secret Manager outage (reported as unusable v2 configuration) while only logging rejected credentials (a connection error). The check now stops startup only when the database answers and the answer disagrees with the country list: different countries, a missing table, or no privilege to read it. When it cannot get an answer, because the v2 configuration cannot be used, the secret cannot be resolved, or the database cannot be reached or queried, it logs why and startup continues; the readiness check already reports unusable v2 configuration as not ready and recovers once the secret resolves. The rule `api.startup.no-import-side-effects` now says what deployed startup does: it opens no network connections unless the process runs on Cloud Run or has a v2 runtime database configured. The country list and the startup check join the mypy file list and rule `api.quality.mypy-v2`. The country list's docstring and the Alembic skill now say that a deploy runs a country-adding revision before the new release starts, so the previous release refuses to start afterwards, and list the US-and-UK checks a new country must also widen. Tests (failed first: configuration errors and `InterfaceError` were raised, and importing the ASGI module with an unusable v2 URL exited with an error): configuration and secret failures are logged with their message and startup continues, connection and interface errors are logged by type only, and importing the ASGI module with an unusable v2 URL logs the skipped check without the password. Co-Authored-By: Claude Opus 5.5 --- .../v2-countries-startup-check.added.md | 2 +- docs/engineering/skills/alembic-migrations.md | 6 ++- docs/generated/rules.md | 4 +- policyengine_api/startup_country_check.py | 34 ++++++++----- policyengine_api/supported_countries.py | 11 ++++- pyproject.toml | 2 + rules/registry.toml | 8 ++-- tests/unit/test_startup_country_check.py | 48 ++++++++++++++----- 8 files changed, 82 insertions(+), 33 deletions(-) diff --git a/changelog.d/v2-countries-startup-check.added.md b/changelog.d/v2-countries-startup-check.added.md index 66f42aad2..8e30ae192 100644 --- a/changelog.d/v2-countries-startup-check.added.md +++ b/changelog.d/v2-countries-startup-check.added.md @@ -1 +1 @@ -The Public API refuses to start when the v2 `countries` table and its country list in code name different countries. The check runs on Cloud Run and wherever a v2 runtime database is configured; an unreachable database is logged and skipped. +The Public API refuses to start when the v2 `countries` table and its country list in code name different countries, or when the table or the privilege to read it is missing. The check runs on Cloud Run and wherever a v2 runtime database is configured; when it cannot query the table, it logs why and startup continues. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 2111ace59..0b5c165b4 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -319,4 +319,8 @@ The table must list exactly the countries in `policyengine_api/supported_countries.py`: the Public API refuses to start when they differ (`policyengine_api/startup_country_check.py`). Listing another country later needs a revision that inserts its row, and the same -human migration decision. +human migration decision. The deploy runs that revision before it starts +the new release, so instances of the previous release that start afterwards, +and a rollback to it, refuse to start until the revision is downgraded. The +module docstring of `supported_countries.py` lists the other checks a new +country must widen. diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 8c3cc5f5d..e7f4bfedb 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -164,7 +164,7 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `api.catalog.initializer-not-at-startup` | Application startup modules never import the metadata catalog initializer. | mechanical | enforced | yes | pytest `tests/unit/v2/test_catalog_initialization.py` | `pr.yml:test` | **unproven** | | `api.startup.fail-closed-configuration` | Default Cloud SQL startup needs no Supabase configuration, and a selected v2 runtime without its database URL fails closed. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | -| `api.startup.no-import-side-effects` | Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | +| `api.startup.no-import-side-effects` | Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | | `api.startup.no-runtime-sqlite-modules` | Runtime SQLite modules are absent from the production package. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | ```mermaid @@ -294,7 +294,7 @@ flowchart LR | `api.quality.command-line-modules-ruff-exemptions` | Application modules that also run as command-line programs (if __name__ == "__main__") may print their output (T201); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of T201 for `policyengine_api/{data/v2/catalog/initialization,data/v2/household_migration_qualification,data/v2/policy_migration_qualification,data/v2/privilege_reconciliation,data/v2/user_association_migration_qualification,worker_spm_release}.py` | `pr.yml:lint` | — | | `api.quality.command-line-scripts-ruff-exemptions` | Command-line scripts under scripts/ may print their output, run other programs, fetch fixed URLs, stamp today's date and pick random samples (T201, S603, S607, S310, DTZ011, S311); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of DTZ011, S310, S311, S603, S607, T201 for `scripts/**` | `pr.yml:lint` | — | | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — | -| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | +| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `api.quality.ruff-lint` | All Python code passes ruff check with every ruff rule selected (ALL), with no ratchet, except the ignores in pyproject.toml: COM812 and ISC001, which conflict with ruff format, and D203 and D213, which conflict with the kept D211 and D212. | mechanical | enforced | yes | ruff check (ALL) | `pr.yml:lint` | — | | `api.quality.runtime-evaluated-models-resolve` | Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. | mechanical | enforced | yes | pytest `tests/unit/test_runtime_evaluated_annotations.py` | `pr.yml:test` | **unproven** | diff --git a/policyengine_api/startup_country_check.py b/policyengine_api/startup_country_check.py index 43963d077..69f233988 100644 --- a/policyengine_api/startup_country_check.py +++ b/policyengine_api/startup_country_check.py @@ -11,10 +11,14 @@ resource is configured. A local or test startup without v2 configuration opens no connection. -A disagreement, a missing table, a missing privilege or unusable v2 -configuration stops startup. A database that cannot be reached is logged and -startup continues: the v1 routes do not read the v2 database, and their -copies into it already continue without it. +Startup stops when the database answers and its answer disagrees with the +list: different countries, a missing table, or no privilege to read it. +When the check cannot get an answer, it logs why and startup continues: the +v1 routes do not read the v2 database, and their copies into it already +continue without it. That covers a database that cannot be reached or +refuses the credentials, and v2 configuration that cannot be used, including +a secret that cannot be resolved; the readiness check already reports the +last as not ready, and recovers once the secret resolves. """ from __future__ import annotations @@ -23,11 +27,12 @@ import os from typing import TYPE_CHECKING, Protocol -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import InterfaceError, OperationalError from policyengine_api.data.v2.settings import ( V2_RUNTIME_DATABASE_URL, V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, + V2ConfigurationError, ) from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS @@ -75,10 +80,11 @@ def check_countries_table_at_startup( ) -> None: """Raise unless the countries table names exactly the countries in code. - Raises ``CountryListMismatchError`` on a disagreement, SQLAlchemy's + Raises ``CountryListMismatchError`` on a disagreement and SQLAlchemy's ``ProgrammingError`` when the table or the privilege to read it is - missing, and ``V2ConfigurationError`` for unusable v2 configuration. An - unreachable database is logged, without its error detail, and skipped. + missing. Unusable v2 configuration is logged with its message, which + never contains a secret, and a database connection failure by its error + type only, as its detail can name the host; both skip the check. """ values = os.environ if environ is None else environ if not v2_runtime_database_selected(values): @@ -86,11 +92,15 @@ def check_countries_table_at_startup( factory = verifier_factory or _default_country_list_verifier try: factory().verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) - except OperationalError as error: - # The error's detail can name the database host, so only its type is - # logged. + except V2ConfigurationError as error: + logger.warning( + "Skipped the countries table check at startup: the v2 runtime " + "configuration cannot be used (%s)", + error, + ) + except (OperationalError, InterfaceError) as error: logger.warning( "Skipped the countries table check at startup: the v2 database " - "could not be reached (%s)", + "could not be queried (%s)", type(error).__name__, ) diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py index 15b9d1b60..0c5ad01a1 100644 --- a/policyengine_api/supported_countries.py +++ b/policyengine_api/supported_countries.py @@ -8,7 +8,16 @@ The v2 ``countries`` table records what each country supports, and must hold one row for each listed country and no other: a v2 Alembic revision inserts the row when a country is added. The application refuses to start when they -differ (``policyengine_api.startup_country_check``). +differ (``policyengine_api.startup_country_check``). Because a deploy runs +that revision before it starts the new release, instances of the previous +release that start afterwards, or a rollback to it, refuse to start until +the revision is downgraded. + +A new country also needs the US-and-UK checks widened on the tables that +pair v1 records with their v2 copies (``legacy_policy_mappings``, +``legacy_user_policy_mappings``, ``legacy_household_mappings``) and on the +v1 ``household_mirror_events`` table, which the v1 household insert writes +in its own transaction. The module imports nothing from the package, so modules that the query models import, such as the catalog package, can import it too. diff --git a/pyproject.toml b/pyproject.toml index 10a5cd6f3..ae1ce0e70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,8 @@ files = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] explicit_package_bases = true follow_imports = "silent" diff --git a/rules/registry.toml b/rules/registry.toml index e517ec50a..46718196b 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -396,7 +396,7 @@ invocation = "make test" [[rule]] id = "api.startup.no-import-side-effects" category = "startup" -statement = "Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases." +statement = "Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases." rationale = "docs/engineering/skills/testing.md#focused-commands" scope = ["policyengine_api/**/*.py"] tier = "mechanical" @@ -919,9 +919,9 @@ invocation = "ruff check ." [[rule]] id = "api.quality.mypy-v2" category = "quality" -statement = "Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed." +statement = "Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed." rationale = "docs/engineering/skills/v2-code-organization.md" -scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py"] +scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", "policyengine_api/startup_country_check.py", "policyengine_api/supported_countries.py"] tier = "mechanical" status = "enforced" autofix = false @@ -935,6 +935,8 @@ paths = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" diff --git a/tests/unit/test_startup_country_check.py b/tests/unit/test_startup_country_check.py index 53cefa172..bf479c868 100644 --- a/tests/unit/test_startup_country_check.py +++ b/tests/unit/test_startup_country_check.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING import pytest -from sqlalchemy.exc import OperationalError, ProgrammingError +from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.services.v2.countries.validators import ( @@ -125,23 +125,44 @@ def test_a_failed_check_stops_startup(error: Exception) -> None: ) -def test_unusable_v2_configuration_stops_startup() -> None: +@pytest.mark.parametrize( + "message", + [ + "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE could not be resolved", + ], + ids=["invalid-url", "secret-unavailable"], +) +def test_unusable_v2_configuration_is_logged_and_left_to_readiness( + message: str, + caplog: pytest.LogCaptureFixture, +) -> None: + """Readiness already reports not ready for it, and recovers when it does.""" + def unusable_configuration() -> FakeCountryListVerifier: - msg = "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" - raise V2ConfigurationError(msg) + raise V2ConfigurationError(message) - with pytest.raises(V2ConfigurationError): + with caplog.at_level(logging.WARNING): check_countries_table_at_startup( environ={"K_SERVICE": "policyengine-api"}, verifier_factory=unusable_configuration, ) + assert [record.levelno for record in caplog.records] == [logging.WARNING] + assert message in caplog.records[0].getMessage() + +@pytest.mark.parametrize( + "error_type", + [OperationalError, InterfaceError], + ids=["operational", "interface"], +) def test_an_unreachable_database_is_logged_and_startup_continues( + error_type: type[OperationalError | InterfaceError], caplog: pytest.LogCaptureFixture, ) -> None: verifier = FakeCountryListVerifier( - error=OperationalError( + error=error_type( "SELECT countries.id FROM countries", {}, Exception("connection to db.secret-host failed: password=hunter2"), @@ -158,13 +179,13 @@ def test_an_unreachable_database_is_logged_and_startup_continues( assert [record.levelno for record in caplog.records] == [logging.WARNING] message = caplog.records[0].getMessage() assert "countries table" in message - assert "OperationalError" in message + assert error_type.__name__ in message assert "secret-host" not in message assert "hunter2" not in message def test_importing_the_asgi_application_runs_the_check(tmp_path: Path) -> None: - """An unusable v2 runtime database URL stops the application from loading.""" + """The check runs at import; an unusable v2 runtime URL is only logged.""" environment = os.environ.copy() environment.update( { @@ -189,8 +210,9 @@ def test_importing_the_asgi_application_runs_the_check(tmp_path: Path) -> None: check=False, ) - assert result.returncode != 0 - assert "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" in ( - result.stderr - ) - assert "hunter2" not in result.stderr + # The application's log handler writes structured records to stdout. + output = result.stdout + result.stderr + assert result.returncode == 0, result.stderr + assert "Skipped the countries table check" in output + assert "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" in output + assert "hunter2" not in output From 975c80fc12df75301bcfa25c3c01240cbf0e014f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:13:15 +0400 Subject: [PATCH 006/118] Keep the startup check's log tests independent of earlier Alembic tests In a full test run, the tests that run Alembic commands load Alembic's logging configuration, which disables every logger that already exists, including the startup check's. Its four log tests then captured nothing and failed, though they passed on their own. An autouse fixture now re-enables the check's logger for each of its tests. Tests (failed first: running the Alembic unit tests before the startup check tests failed all four log tests); with the fixture, both files pass together (46 passed). Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_startup_country_check.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/test_startup_country_check.py b/tests/unit/test_startup_country_check.py index bf479c868..64bd0f219 100644 --- a/tests/unit/test_startup_country_check.py +++ b/tests/unit/test_startup_country_check.py @@ -11,6 +11,7 @@ import pytest from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError +from policyengine_api import startup_country_check from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.services.v2.countries.validators import ( CountryListMismatchError, @@ -25,6 +26,16 @@ V2_RUNTIME_SECRET_RESOURCE = "projects/example/secrets/v2-runtime/versions/latest" +@pytest.fixture(autouse=True) +def enabled_startup_check_logger(monkeypatch: pytest.MonkeyPatch) -> None: + """Re-enable the check's logger, which Alembic's logging setup disables. + + Tests that run Alembic commands load its logging configuration, which + disables every logger that already exists, including this module's. + """ + monkeypatch.setattr(startup_country_check.logger, "disabled", False) + + class FakeCountryListVerifier: """Record each verification, then raise ``error`` if one is set.""" From e86d9ce22991183343eefb1bca4bbd00e99a6d5d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:10:44 +0400 Subject: [PATCH 007/118] Let a saved policy be repointed to a v2 policy PATCH /v2/user-policies/{id} now accepts policy_id. Supplied alone, it points the saved policy at that v2 policy and clears legacy_policy_id, which no longer describes the saved content. Supplied with legacy_policy_id, both are stored, and policy_id must be the v2 copy of that v1 policy when one exists, exactly as creation decides. The policy must exist in the row's country. The row keeps its ID, owner, name and creation time, so an edited reform keeps the ID that reports link to. Co-Authored-By: Claude Opus 5.5 --- ...v2plan-1f-121-repoint-saved-items.added.md | 1 + docs/migration/stage-12a-user-login.md | 6 + .../fastapi_routes/v2/user_policies/routes.py | 10 +- .../database_connectors/updates.py | 23 ++- .../services/v2/user_policies/services.py | 39 ++-- .../services/v2/user_policies/types.py | 47 ++++- tests/unit/v2/test_user_policy_routes.py | 22 ++- tests/unit/v2/test_user_policy_service.py | 173 +++++++++++++++++- 8 files changed, 277 insertions(+), 44 deletions(-) create mode 100644 changelog.d/v2plan-1f-121-repoint-saved-items.added.md diff --git a/changelog.d/v2plan-1f-121-repoint-saved-items.added.md b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md new file mode 100644 index 000000000..fae825693 --- /dev/null +++ b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md @@ -0,0 +1 @@ +PATCH /v2/user-policies/{id} accepts policy_id, so a saved policy can point at another v2 policy in its country while keeping its ID, name and creation time. Supplied alone, policy_id clears the v1 reference (legacy_policy_id); supplied with legacy_policy_id, both are stored, and policy_id must be the v2 copy of that v1 policy when one exists. Unknown, other-country and conflicting policies are refused as on creation, and rows maintained by the v1 saved-policy mirror still return 409. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index c01657bc3..62e915479 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -183,6 +183,12 @@ Concurrency on Postgres: `legacy_policy_mappings` / `legacy_household_mappings` when a mapping exists. - Changing the legacy ID re-resolves the v2 reference, or clears it. + - `PATCH /v2/user-policies` also accepts `policy_id`, which must name a v2 + policy in the row's country. Supplied alone, it points the row at that + policy and clears `legacy_policy_id`, so an edited reform keeps its saved + ID, name and creation time. Supplied with `legacy_policy_id`, both are + stored, and `policy_id` must be the mirror copy of that v1 policy when + one exists. - `user-simulations` and `user-reports` accept only legacy IDs in 12A. - **Stage 10 mirror rows.** A `user_policies` row with a `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index bb5a06acc..6ff192fee 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -168,9 +168,13 @@ def read() -> UserPolicyPageResponse: response_model=UserPolicyDetailResponse, summary="Update or repoint one of the current user's saved policies", description=( - "Changing legacy_policy_id repoints the row and re-resolves policy_id " - "from the Stage 10 mirror mapping, or clears it. Rows maintained by " - "the v1 saved-policy mirror return 409." + "The row keeps its id, owner and creation time. Changing " + "legacy_policy_id repoints the row and re-resolves policy_id from the " + "Stage 10 mirror mapping, or clears it. Changing policy_id alone points " + "the row at that v2 policy, which must exist in the same country, and " + "clears legacy_policy_id. Supplied together, both are stored, and " + "policy_id must be the mirror copy of legacy_policy_id when one exists. " + "Rows maintained by the v1 saved-policy mirror return 409." ), ) def patch_user_policy_route( diff --git a/policyengine_api/services/v2/user_policies/database_connectors/updates.py b/policyengine_api/services/v2/user_policies/database_connectors/updates.py index 328c3d32d..2208e5636 100644 --- a/policyengine_api/services/v2/user_policies/database_connectors/updates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/updates.py @@ -5,15 +5,17 @@ from typing import TYPE_CHECKING from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_policies.types import ( + USER_POLICY_REFERENCE_FIELDS, +) if TYPE_CHECKING: - from uuid import UUID - from sqlmodel import Session from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyRevision, + UserPolicyReferences, UserPolicyUpdateInput, ) @@ -23,17 +25,20 @@ def update_user_policy( association: UserPolicy, association_input: UserPolicyUpdateInput, *, - repointed_policy_id: UUID | None = None, - repoint: bool = False, + references: UserPolicyReferences | None = None, ) -> UserPolicy: - """Apply the supplied fields, bump ``updated_at`` and return the row. + """Apply the supplied presentation fields, bump ``updated_at`` and return the row. - With ``repoint``, also set ``policy_id`` to ``repointed_policy_id``. + With ``references``, also store both of its policy references; the service + resolves them, so the request's own reference fields are never copied. """ - for field_name, value in association_input.model_dump(exclude_unset=True).items(): + for field_name, value in association_input.model_dump( + exclude_unset=True, exclude=set(USER_POLICY_REFERENCE_FIELDS) + ).items(): setattr(association, field_name, value) - if repoint: - association.policy_id = repointed_policy_id + if references is not None: + association.policy_id = references.policy_id + association.legacy_policy_id = references.legacy_policy_id association.updated_at = utc_now() session.add(association) session.flush() diff --git a/policyengine_api/services/v2/user_policies/services.py b/policyengine_api/services/v2/user_policies/services.py index 4ccbad927..8bf3dbe4a 100644 --- a/policyengine_api/services/v2/user_policies/services.py +++ b/policyengine_api/services/v2/user_policies/services.py @@ -61,6 +61,7 @@ UserPolicyListing, UserPolicyPage, UserPolicyRead, + UserPolicyReferences, UserPolicyUpdateInput, ) from policyengine_api.services.v2.user_policies.validators import ( @@ -500,13 +501,18 @@ def patch_user_policy( ) -> UserPolicyRead: """Update the supplied fields of the principal's association in this country. - A new ``legacy_policy_id`` must exist in v1 and repoints ``policy_id`` to the - v2 copy of that policy, or to None when there is no copy. Raise - UserPolicyNotFoundError when the association is not the principal's, and - MirrorOwnedAssociationError when the v1 saved-policy mirror maintains it. + The association keeps its id, owner, country and creation time. A new + ``legacy_policy_id`` must exist in v1 (LegacyReferenceNotFoundError) and + repoints ``policy_id`` to the v2 copy of that policy, or to None when there + is no copy. A new ``policy_id`` must exist in this country + (AssociationPolicyNotFoundError or AssociationCountryConflictError). + Without ``legacy_policy_id`` it clears the v1 reference; with it, both are + stored, and it must be the v2 copy of that v1 policy when one exists + (AssociationReferenceConflictError). Raise UserPolicyNotFoundError when the + association is not the principal's, and MirrorOwnedAssociationError when + the v1 saved-policy mirror maintains it. """ - repoint = "legacy_policy_id" in association_input.model_fields_set - if repoint: + if "legacy_policy_id" in association_input.model_fields_set: if association_input.legacy_policy_id is None: msg = "legacy_policy_id must not be null" raise TypeError(msg) @@ -530,13 +536,17 @@ def patch_user_policy( ), association_id=association.id, ) - repointed_policy_id = ( - read_mirrored_policy_ids( - session, - country_id=country_id, - legacy_policy_ids=(association_input.legacy_policy_id,), - ).get(association_input.legacy_policy_id) - if repoint and association_input.legacy_policy_id is not None + references = ( + UserPolicyReferences( + policy_id=resolve_user_policy_reference( + session, + country_id=country_id, + policy_id=association_input.policy_id, + legacy_policy_id=association_input.legacy_policy_id, + ), + legacy_policy_id=association_input.legacy_policy_id, + ) + if association_input.repoints else None ) return user_policy_read( @@ -544,8 +554,7 @@ def patch_user_policy( session, association, association_input, - repointed_policy_id=repointed_policy_id, - repoint=repoint, + references=references, ) ) diff --git a/policyengine_api/services/v2/user_policies/types.py b/policyengine_api/services/v2/user_policies/types.py index c24b2d49c..296fc821c 100644 --- a/policyengine_api/services/v2/user_policies/types.py +++ b/policyengine_api/services/v2/user_policies/types.py @@ -82,13 +82,22 @@ def require_policy_reference(self) -> UserPolicyCreationInput: return self +USER_POLICY_REFERENCE_FIELDS = frozenset({"policy_id", "legacy_policy_id"}) + + class UserPolicyUpdateInput(StrictAssociationInput): """A partial update of a saved policy; only supplied fields are changed. - At least one field must be supplied. ``legacy_policy_id`` may point the - association at another v1 policy but cannot be cleared. + At least one field must be supplied. ``policy_id`` points the association at + another v2 policy; supplied without ``legacy_policy_id``, it also clears the + v1 reference, which no longer describes the saved policy. ``legacy_policy_id`` + alone points the association at another v1 policy and takes ``policy_id`` from + that policy's v2 copy, or clears it when there is none. Supplied together, + both are stored, and ``policy_id`` must be that v2 copy when one exists. + Neither reference can be set to null. """ + policy_id: ResourceId | None = None legacy_policy_id: LegacyRecordId | None = None name: AssociationName | None = None description: AssociationDescription | None = None @@ -97,21 +106,41 @@ class UserPolicyUpdateInput(StrictAssociationInput): @model_validator(mode="after") def require_supplied_field(self) -> UserPolicyUpdateInput: - """Require a supplied field, and reject an explicit null legacy_policy_id.""" + """Require a supplied field, and reject an explicit null policy reference.""" supplied = self.model_fields_set.intersection( - {"legacy_policy_id", "name", "description", "baseline", "provenance"} + { + *USER_POLICY_REFERENCE_FIELDS, + "name", + "description", + "baseline", + "provenance", + } ) if not supplied: msg = ( - "At least one of legacy_policy_id, name, description, baseline, " - "or provenance must be supplied" + "At least one of policy_id, legacy_policy_id, name, description, " + "baseline, or provenance must be supplied" ) raise ValueError(msg) - if "legacy_policy_id" in supplied and self.legacy_policy_id is None: - msg = "legacy_policy_id must not be null" - raise ValueError(msg) + for reference_field in sorted(USER_POLICY_REFERENCE_FIELDS): + if reference_field in supplied and getattr(self, reference_field) is None: + msg = f"{reference_field} must not be null" + raise ValueError(msg) return self + @property + def repoints(self) -> bool: + """Whether the update changes which policy the association references.""" + return bool(self.model_fields_set & USER_POLICY_REFERENCE_FIELDS) + + +@dataclass(frozen=True) +class UserPolicyReferences: + """The v2 and v1 policy references a repointed association stores.""" + + policy_id: UUID | None + legacy_policy_id: int | None + class LegacyUserPolicySnapshot(StrictAssociationInput): """Detached complete committed v1 saved-policy row.""" diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 10d837038..37c969add 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -250,7 +250,8 @@ def test_patch_repoints_legacy_reference_and_rejects_invalid_changes() -> None: json=body, ) for body in ( - {"policy_id": str(POLICY_ID)}, + {"policy_id": None}, + {"policy_id": "not-a-uuid"}, {"user_id": str(OWNER_ID)}, {"client_ref": "rf-2"}, {"legacy_policy_id": None}, @@ -262,11 +263,27 @@ def test_patch_repoints_legacy_reference_and_rejects_invalid_changes() -> None: association_input = service.calls[0][1]["association_input"] assert association_input.legacy_policy_id == 101 assert association_input.model_fields_set == {"legacy_policy_id", "name"} - assert [response.status_code for response in invalid] == [422] * 5 + assert [response.status_code for response in invalid] == [422] * 6 assert [call[0] for call in service.calls] == ["patch_user_policy"] assert flask_calls["count"] == 0 +def test_patch_repoints_to_a_v2_policy_reference() -> None: + client, service, flask_calls = _client() + + repointed = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"policy_id": str(POLICY_ID)}, + ) + + assert repointed.status_code == 200 + assert service.calls[0][0] == "patch_user_policy" + association_input = service.calls[0][1]["association_input"] + assert association_input.policy_id == POLICY_ID + assert association_input.model_fields_set == {"policy_id"} + assert flask_calls["count"] == 0 + + def test_delete_returns_no_content_for_the_session_user() -> None: client, service, _flask_calls = _client() @@ -383,6 +400,7 @@ def test_openapi_publishes_signed_in_association_contracts() -> None: patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] assert patch_schema["additionalProperties"] is False assert set(patch_schema["properties"]) == { + "policy_id", "legacy_policy_id", "name", "description", diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 70352bb34..f1a69c048 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -164,7 +164,7 @@ def test_create_rejects_bad_references_before_writing(store) -> None: def test_other_users_rows_are_not_found_and_lists_are_owner_scoped(store) -> None: - service, _selected, _legacy_references = store + service, selected, _legacy_references = store alice_row = service.create_user_policy( principal=ALICE, association_input=_fields(name="Alice") ) @@ -183,6 +183,12 @@ def test_other_users_rows_are_not_found_and_lists_are_owner_scoped(store) -> Non association_id=alice_row.id, association_input=UserPolicyUpdateInput(name="stolen"), ), + lambda: service.patch_user_policy( + principal=BOB, + country_id="us", + association_id=alice_row.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ), lambda: service.delete_user_policy( principal=BOB, country_id="us", association_id=alice_row.id ), @@ -253,6 +259,145 @@ def test_repointing_re_resolves_or_clears_the_v2_reference(store) -> None: assert legacy_references.calls[-1] == ("us", {"policy": [404]}) +def test_repointing_to_a_v2_policy_keeps_the_saved_row_and_clears_the_v1_reference( + store, +) -> None: + service, selected, legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + v1_checks_before_repoint = list(legacy_references.calls) + + repointed = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ) + + assert (repointed.policy_id, repointed.legacy_policy_id) == ( + selected.policy_id, + None, + ) + assert (repointed.id, repointed.name, repointed.description) == ( + created.id, + "Saved reform", + "Personal note", + ) + assert repointed.created_at == created.created_at + assert repointed.updated_at >= created.updated_at + assert legacy_references.calls == v1_checks_before_repoint + with selected.sessions() as session: + stored = session.get(UserPolicy, created.id) + assert stored is not None + assert (stored.policy_id, stored.legacy_policy_id) == ( + selected.policy_id, + None, + ) + + +def test_repointing_to_both_references_stores_both(store) -> None: + service, selected, legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + + mirrored = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput( + policy_id=selected.policy_id, + legacy_policy_id=MAPPED_LEGACY_POLICY_ID, + ), + ) + assert (mirrored.policy_id, mirrored.legacy_policy_id) == ( + selected.policy_id, + MAPPED_LEGACY_POLICY_ID, + ) + assert legacy_references.calls[-1] == ( + "us", + {"policy": [MAPPED_LEGACY_POLICY_ID]}, + ) + + # v1 policy 3 has no v2 copy, so nothing can contradict the v2 reference: + # both are stored, as creation stores them. + unmirrored = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput( + policy_id=selected.policy_id, legacy_policy_id=3 + ), + ) + assert (unmirrored.policy_id, unmirrored.legacy_policy_id) == ( + selected.policy_id, + 3, + ) + assert unmirrored.id == created.id + + +def test_repointing_to_a_missing_foreign_or_conflicting_policy_changes_nothing( + store, +) -> None: + service, selected, _legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + with selected.sessions.begin() as session: + mirrored_policy = session.get(Policy, selected.policy_id) + assert mirrored_policy is not None + other_us_policy = Policy( + country_id="us", + tax_benefit_model_id=mirrored_policy.tax_benefit_model_id, + tax_benefit_model_version_id=mirrored_policy.tax_benefit_model_version_id, + canonicalization_version=1, + content_hash="e" * 64, + ) + session.add(other_us_policy) + session.flush() + other_us_policy_id = other_us_policy.id + + rejected_updates = ( + ( + AssociationPolicyNotFoundError, + UserPolicyUpdateInput( + policy_id=UUID("00000000-0000-0000-0000-000000000099"), name="Lost" + ), + ), + ( + AssociationCountryConflictError, + UserPolicyUpdateInput(policy_id=selected.uk_policy_id, name="Lost"), + ), + ( + AssociationReferenceConflictError, + UserPolicyUpdateInput( + policy_id=other_us_policy_id, + legacy_policy_id=MAPPED_LEGACY_POLICY_ID, + name="Lost", + ), + ), + ( + LegacyReferenceNotFoundError, + UserPolicyUpdateInput( + policy_id=selected.policy_id, legacy_policy_id=404, name="Lost" + ), + ), + ) + for expected_error, rejected_update in rejected_updates: + with pytest.raises(expected_error): + service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=rejected_update, + ) + + unchanged = service.get_user_policy( + principal=ALICE, country_id="us", association_id=created.id + ) + assert (unchanged.policy_id, unchanged.legacy_policy_id, unchanged.name) == ( + None, + 1, + "Saved reform", + ) + assert unchanged.updated_at == created.updated_at + + def test_rows_owned_by_the_stage_10_mirror_conflict(store) -> None: service, selected, _legacy_references = store created = service.create_user_policy(principal=ALICE, association_input=_fields()) @@ -274,6 +419,13 @@ def test_rows_owned_by_the_stage_10_mirror_conflict(store) -> None: association_id=created.id, association_input=UserPolicyUpdateInput(name="Renamed"), ) + with pytest.raises(MirrorOwnedAssociationError): + service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ) with pytest.raises(MirrorOwnedAssociationError): service.delete_user_policy( principal=ALICE, country_id="us", association_id=created.id @@ -331,13 +483,22 @@ def test_stage_10_projection_records_the_v1_reform_id() -> None: def test_patch_input_requires_a_supported_field() -> None: - with pytest.raises(ValueError, match="At least one of legacy_policy_id"): + with pytest.raises(ValueError, match="At least one of policy_id, legacy_policy_id"): UserPolicyUpdateInput() - with pytest.raises( - ValueError, match=r"policy_id\n Extra inputs are not permitted" - ): + with pytest.raises(ValueError, match=r"user_id\n Extra inputs are not permitted"): UserPolicyUpdateInput.model_validate( - {"policy_id": "00000000-0000-0000-0000-000000000001"} + {"user_id": "00000000-0000-0000-0000-0000000000a1"} ) with pytest.raises(ValueError, match="legacy_policy_id must not be null"): UserPolicyUpdateInput(legacy_policy_id=None) + with pytest.raises(ValueError, match="policy_id must not be null"): + UserPolicyUpdateInput(policy_id=None) + + +def test_patch_input_accepts_a_v2_policy_reference() -> None: + repoint = UserPolicyUpdateInput.model_validate( + {"policy_id": "00000000-0000-0000-0000-000000000001"} + ) + + assert repoint.policy_id == UUID("00000000-0000-0000-0000-000000000001") + assert repoint.model_fields_set == {"policy_id"} From 8be3803281a32549374aa50ac564de245d23e849 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:21:42 +0400 Subject: [PATCH 008/118] Let a saved household be repointed to a v2 household PATCH /v2/user-households/{id} now accepts household_id, with the same rules as a saved policy's policy_id: supplied alone, it points the saved household at that v2 household and clears legacy_household_id; supplied with legacy_household_id, both are stored, and household_id must be the v2 copy of that v1 household when one exists. The household must exist in the row's country, and the row keeps its ID, owner, name and creation time. Without this, editing a saved household changed its ID. Co-Authored-By: Claude Opus 5.5 --- ...v2plan-1f-121-repoint-saved-items.added.md | 2 +- docs/migration/stage-12a-user-login.md | 4 + .../v2/user_households/routes.py | 9 +- .../database_connectors/updates.py | 23 ++- .../services/v2/user_households/services.py | 37 ++-- .../services/v2/user_households/types.py | 41 +++- tests/unit/v2/test_user_household_routes.py | 37 +++- tests/unit/v2/test_user_household_service.py | 176 +++++++++++++++++- tests/unit/v2/user_service_support.py | 11 +- 9 files changed, 295 insertions(+), 45 deletions(-) diff --git a/changelog.d/v2plan-1f-121-repoint-saved-items.added.md b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md index fae825693..14b849f5b 100644 --- a/changelog.d/v2plan-1f-121-repoint-saved-items.added.md +++ b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md @@ -1 +1 @@ -PATCH /v2/user-policies/{id} accepts policy_id, so a saved policy can point at another v2 policy in its country while keeping its ID, name and creation time. Supplied alone, policy_id clears the v1 reference (legacy_policy_id); supplied with legacy_policy_id, both are stored, and policy_id must be the v2 copy of that v1 policy when one exists. Unknown, other-country and conflicting policies are refused as on creation, and rows maintained by the v1 saved-policy mirror still return 409. +PATCH /v2/user-policies/{id} accepts policy_id and PATCH /v2/user-households/{id} accepts household_id, so a saved policy or household can point at another v2 record in its country while keeping its ID, name and creation time. Supplied alone, the v2 reference clears the v1 reference (legacy_policy_id or legacy_household_id); supplied with it, both are stored, and the v2 reference must be the v2 copy of that v1 record when one exists. Unknown, other-country and conflicting records are refused as on creation, and saved policies maintained by the v1 saved-policy mirror still return 409. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index 62e915479..3fc391392 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -189,6 +189,10 @@ Concurrency on Postgres: ID, name and creation time. Supplied with `legacy_policy_id`, both are stored, and `policy_id` must be the mirror copy of that v1 policy when one exists. + - `PATCH /v2/user-households` accepts `household_id` the same way: it must + name a v2 household in the row's country; alone, it clears + `legacy_household_id`; with `legacy_household_id`, both are stored, and + it must be the copy of that v1 household when one exists. - `user-simulations` and `user-reports` accept only legacy IDs in 12A. - **Stage 10 mirror rows.** A `user_policies` row with a `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The diff --git a/policyengine_api/fastapi_routes/v2/user_households/routes.py b/policyengine_api/fastapi_routes/v2/user_households/routes.py index c642a79d8..4e86c9b86 100644 --- a/policyengine_api/fastapi_routes/v2/user_households/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_households/routes.py @@ -179,8 +179,13 @@ def read() -> UserHouseholdPageResponse: response_model=UserHouseholdDetailResponse, summary="Update or repoint one of the current user's saved households", description=( - "Changing legacy_household_id repoints the row and re-resolves " - "household_id from the Stage 11 copy mapping, or clears it." + "The row keeps its id, owner and creation time. Changing " + "legacy_household_id repoints the row and re-resolves household_id " + "from the Stage 11 copy mapping, or clears it. Changing household_id " + "alone points the row at that v2 household, which must exist in the " + "same country, and clears legacy_household_id. Supplied together, both " + "are stored, and household_id must be the copy of legacy_household_id " + "when one exists." ), ) def patch_user_household_route( diff --git a/policyengine_api/services/v2/user_households/database_connectors/updates.py b/policyengine_api/services/v2/user_households/database_connectors/updates.py index ed6cfb989..e0de95d8e 100644 --- a/policyengine_api/services/v2/user_households/database_connectors/updates.py +++ b/policyengine_api/services/v2/user_households/database_connectors/updates.py @@ -5,14 +5,16 @@ from typing import TYPE_CHECKING from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_households.types import ( + USER_HOUSEHOLD_REFERENCE_FIELDS, +) if TYPE_CHECKING: - from uuid import UUID - from sqlmodel import Session from policyengine_api.data.v2.models import UserHouseholdAssociation from policyengine_api.services.v2.user_households.types import ( + UserHouseholdReferences, UserHouseholdUpdateInput, ) @@ -22,17 +24,20 @@ def update_user_household( association: UserHouseholdAssociation, association_input: UserHouseholdUpdateInput, *, - repointed_household_id: UUID | None = None, - repoint: bool = False, + references: UserHouseholdReferences | None = None, ) -> UserHouseholdAssociation: - """Apply the supplied fields, bump ``updated_at`` and return the row. + """Apply the supplied presentation fields, bump ``updated_at`` and return the row. - With ``repoint``, also set ``household_id`` to ``repointed_household_id``. + With ``references``, also store both of its household references; the + service resolves them, so the request's own reference fields are never copied. """ - for field_name, value in association_input.model_dump(exclude_unset=True).items(): + for field_name, value in association_input.model_dump( + exclude_unset=True, exclude=set(USER_HOUSEHOLD_REFERENCE_FIELDS) + ).items(): setattr(association, field_name, value) - if repoint: - association.household_id = repointed_household_id + if references is not None: + association.household_id = references.household_id + association.legacy_household_id = references.legacy_household_id association.updated_at = utc_now() session.add(association) session.flush() diff --git a/policyengine_api/services/v2/user_households/services.py b/policyengine_api/services/v2/user_households/services.py index a2630e897..e2c791b76 100644 --- a/policyengine_api/services/v2/user_households/services.py +++ b/policyengine_api/services/v2/user_households/services.py @@ -42,6 +42,7 @@ from policyengine_api.services.v2.user_households.types import ( UserHouseholdFilter, UserHouseholdListing, + UserHouseholdReferences, ) from policyengine_api.services.v2.user_households.validators import ( require_user_household, @@ -267,12 +268,17 @@ def patch_user_household( ) -> UserHouseholdRead: """Update the supplied fields of the principal's association in this country. - A new ``legacy_household_id`` must exist in v1 and repoints ``household_id`` - to the v2 copy of that household, or to None when there is no copy. Raise - UserHouseholdNotFoundError when the association is not the principal's. + The association keeps its id, owner, country and creation time. A new + ``legacy_household_id`` must exist in v1 (LegacyReferenceNotFoundError) and + repoints ``household_id`` to the v2 copy of that household, or to None when + there is no copy. A new ``household_id`` must exist in this country + (AssociationHouseholdNotFoundError or AssociationCountryConflictError). + Without ``legacy_household_id`` it clears the v1 reference; with it, both + are stored, and it must be the v2 copy of that v1 household when one exists + (AssociationReferenceConflictError). Raise UserHouseholdNotFoundError when + the association is not the principal's. """ - repoint = "legacy_household_id" in association_input.model_fields_set - if repoint: + if "legacy_household_id" in association_input.model_fields_set: if association_input.legacy_household_id is None: msg = "legacy_household_id must not be null" raise TypeError(msg) @@ -290,13 +296,17 @@ def patch_user_household( country_id=country_id, association_id=association_id, ) - repointed_household_id = ( - read_copied_household_ids( - session, - country_id=country_id, - legacy_household_ids=(association_input.legacy_household_id,), - ).get(association_input.legacy_household_id) - if repoint and association_input.legacy_household_id is not None + references = ( + UserHouseholdReferences( + household_id=resolve_user_household_reference( + session, + country_id=country_id, + household_id=association_input.household_id, + legacy_household_id=association_input.legacy_household_id, + ), + legacy_household_id=association_input.legacy_household_id, + ) + if association_input.repoints else None ) return user_household_read( @@ -304,8 +314,7 @@ def patch_user_household( session, association, association_input, - repointed_household_id=repointed_household_id, - repoint=repoint, + references=references, ) ) diff --git a/policyengine_api/services/v2/user_households/types.py b/policyengine_api/services/v2/user_households/types.py index 599c1e35d..40309e868 100644 --- a/policyengine_api/services/v2/user_households/types.py +++ b/policyengine_api/services/v2/user_households/types.py @@ -61,33 +61,56 @@ def require_household_reference(self) -> UserHouseholdCreationInput: return self +USER_HOUSEHOLD_REFERENCE_FIELDS = frozenset({"household_id", "legacy_household_id"}) + + class UserHouseholdUpdateInput(StrictUserHouseholdInput): """A partial update of a saved household; only supplied fields are changed. - At least one field must be supplied. ``legacy_household_id`` may point the - association at another v1 household but cannot be cleared. + At least one field must be supplied. ``household_id`` points the association + at another v2 household; supplied without ``legacy_household_id``, it also + clears the v1 reference, which no longer describes the saved household. + ``legacy_household_id`` alone points the association at another v1 household + and takes ``household_id`` from that household's v2 copy, or clears it when + there is none. Supplied together, both are stored, and ``household_id`` must + be that v2 copy when one exists. Neither reference can be set to null. """ + household_id: ResourceId | None = None legacy_household_id: LegacyRecordId | None = None name: AssociationName | None = None description: AssociationDescription | None = None @model_validator(mode="after") def require_valid_supplied_field(self) -> UserHouseholdUpdateInput: - """Require a supplied field, and reject an explicit null legacy_household_id.""" - mutable_fields = {"legacy_household_id", "name", "description"} + """Require a supplied field, and reject an explicit null household reference.""" + mutable_fields = {*USER_HOUSEHOLD_REFERENCE_FIELDS, "name", "description"} supplied = self.model_fields_set.intersection(mutable_fields) if not supplied: msg = ( - "At least one of legacy_household_id, name, or description must be " - "supplied" + "At least one of household_id, legacy_household_id, name, or " + "description must be supplied" ) raise ValueError(msg) - if "legacy_household_id" in supplied and self.legacy_household_id is None: - msg = "legacy_household_id must not be null" - raise ValueError(msg) + for reference_field in sorted(USER_HOUSEHOLD_REFERENCE_FIELDS): + if reference_field in supplied and getattr(self, reference_field) is None: + msg = f"{reference_field} must not be null" + raise ValueError(msg) return self + @property + def repoints(self) -> bool: + """Whether the update changes which household the association references.""" + return bool(self.model_fields_set & USER_HOUSEHOLD_REFERENCE_FIELDS) + + +@dataclass(frozen=True) +class UserHouseholdReferences: + """The v2 and v1 household references a repointed association stores.""" + + household_id: UUID | None + legacy_household_id: int | None + @dataclass(frozen=True) class UserHouseholdRead: diff --git a/tests/unit/v2/test_user_household_routes.py b/tests/unit/v2/test_user_household_routes.py index b9bf0df75..2e1553bcb 100644 --- a/tests/unit/v2/test_user_household_routes.py +++ b/tests/unit/v2/test_user_household_routes.py @@ -141,17 +141,26 @@ def test_list_and_patch_use_the_session_user_and_legacy_fields() -> None: f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", json={"legacy_household_id": 124}, ) - reassigned = client.patch( - f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", - json={"household_id": str(HOUSEHOLD_ID)}, - ) user_filter = client.get(f"/v2/user-households?country_id=us&user_id={OWNER_ID}") + invalid_patches = [ + client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json=body, + ) + for body in ( + {"household_id": None}, + {"household_id": "not-a-uuid"}, + {"user_id": str(OWNER_ID)}, + {"legacy_household_id": None}, + {}, + ) + ] assert page.status_code == 200 assert page.json()["result"]["has_more"] is True assert patched.status_code == 200 - assert reassigned.status_code == 422 assert user_filter.status_code == 422 + assert [response.status_code for response in invalid_patches] == [422] * 5 assert service.calls[0] == ( "list_user_households", { @@ -168,6 +177,22 @@ def test_list_and_patch_use_the_session_user_and_legacy_fields() -> None: assert len(service.calls) == 2 +def test_patch_repoints_to_a_v2_household_reference() -> None: + client, service, flask_calls = _client() + + repointed = client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json={"household_id": str(HOUSEHOLD_ID)}, + ) + + assert repointed.status_code == 200 + assert service.calls[0][0] == "patch_user_household" + association_input = service.calls[0][1]["association_input"] + assert association_input.household_id == HOUSEHOLD_ID + assert association_input.model_fields_set == {"household_id"} + assert flask_calls["count"] == 0 + + @pytest.mark.parametrize( ("error", "status"), [ @@ -241,7 +266,9 @@ def test_openapi_publishes_signed_in_household_contract() -> None: patch = schema["paths"]["/v2/user-households/{association_id}"]["patch"] patch_ref = patch["requestBody"]["content"]["application/json"]["schema"]["$ref"] patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] + assert patch_schema["additionalProperties"] is False assert set(patch_schema["properties"]) == { + "household_id", "legacy_household_id", "name", "description", diff --git a/tests/unit/v2/test_user_household_service.py b/tests/unit/v2/test_user_household_service.py index 944018c53..e084c4df6 100644 --- a/tests/unit/v2/test_user_household_service.py +++ b/tests/unit/v2/test_user_household_service.py @@ -2,10 +2,15 @@ from __future__ import annotations +from uuid import UUID + import pytest from sqlmodel import select from policyengine_api.data.v2.models import Household, UserHouseholdAssociation +from policyengine_api.services.v2.associations.validators import ( + AssociationReferenceConflictError, +) from policyengine_api.services.v2.legacy_refs.validators import ( LegacyReferenceNotFoundError, ) @@ -20,6 +25,7 @@ UserHouseholdUpdateInput, ) from policyengine_api.services.v2.user_households.validators import ( + AssociationCountryConflictError, AssociationHouseholdNotFoundError, UserHouseholdNotFoundError, ) @@ -101,6 +107,15 @@ def test_ownership_repointing_and_deletion(store) -> None: service.delete_user_household( principal=BOB, country_id="us", association_id=created.id ) + with pytest.raises(UserHouseholdNotFoundError): + service.patch_user_household( + principal=BOB, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id + ), + ) assert service.list_user_households(principal=BOB, country_id="us").items == () repointed = service.patch_user_household( @@ -132,14 +147,167 @@ def test_ownership_repointing_and_deletion(store) -> None: assert session.get(Household, selected.household_id) is not None +def test_repointing_to_a_v2_household_keeps_the_saved_row_and_clears_the_v1_reference( + store, +) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields(name="Family", description="Note") + ) + + repointed = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput(household_id=selected.household_id), + ) + + assert (repointed.household_id, repointed.legacy_household_id) == ( + selected.household_id, + None, + ) + assert (repointed.id, repointed.name, repointed.description) == ( + created.id, + "Family", + "Note", + ) + assert repointed.created_at == created.created_at + assert repointed.updated_at >= created.updated_at + with selected.sessions() as session: + stored = session.get(UserHouseholdAssociation, created.id) + assert stored is not None + assert (stored.household_id, stored.legacy_household_id) == ( + selected.household_id, + None, + ) + + +def test_repointing_to_both_household_references_stores_both(store) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields() + ) + + copied = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id, + legacy_household_id=MAPPED_LEGACY_HOUSEHOLD_ID, + ), + ) + assert (copied.household_id, copied.legacy_household_id) == ( + selected.household_id, + MAPPED_LEGACY_HOUSEHOLD_ID, + ) + + # v1 household 11 has no v2 copy, so nothing can contradict the v2 + # reference: both are stored, as creation stores them. + uncopied = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id, legacy_household_id=11 + ), + ) + assert (uncopied.household_id, uncopied.legacy_household_id) == ( + selected.household_id, + 11, + ) + assert uncopied.id == created.id + + +def test_repointing_to_a_missing_foreign_or_conflicting_household_changes_nothing( + store, +) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields(name="Family") + ) + with selected.sessions.begin() as session: + other_us_household = Household( + country_id="us", + default_year=2026, + household_data={"people": [], "household": []}, + canonicalization_version=1, + content_hash="e" * 64, + ) + session.add(other_us_household) + session.flush() + other_us_household_id = other_us_household.id + + rejected_updates = ( + ( + AssociationHouseholdNotFoundError, + UserHouseholdUpdateInput( + household_id=UUID("00000000-0000-0000-0000-000000000099"), + name="Lost", + ), + ), + ( + AssociationCountryConflictError, + UserHouseholdUpdateInput( + household_id=selected.uk_household_id, name="Lost" + ), + ), + ( + AssociationReferenceConflictError, + UserHouseholdUpdateInput( + household_id=other_us_household_id, + legacy_household_id=MAPPED_LEGACY_HOUSEHOLD_ID, + name="Lost", + ), + ), + ( + LegacyReferenceNotFoundError, + UserHouseholdUpdateInput( + household_id=selected.household_id, + legacy_household_id=404, + name="Lost", + ), + ), + ) + for expected_error, rejected_update in rejected_updates: + with pytest.raises(expected_error): + service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=rejected_update, + ) + + unchanged = service.get_user_household( + principal=ALICE, country_id="us", association_id=created.id + ) + assert ( + unchanged.household_id, + unchanged.legacy_household_id, + unchanged.name, + ) == (None, 10, "Family") + assert unchanged.updated_at == created.updated_at + + def test_patch_input_rejects_reassignment_and_empty_changes() -> None: - with pytest.raises(ValueError, match="At least one of legacy_household_id"): - UserHouseholdUpdateInput() with pytest.raises( - ValueError, match=r"household_id\n Extra inputs are not permitted" + ValueError, match="At least one of household_id, legacy_household_id" ): + UserHouseholdUpdateInput() + with pytest.raises(ValueError, match=r"user_id\n Extra inputs are not permitted"): UserHouseholdUpdateInput.model_validate( - {"household_id": "00000000-0000-0000-0000-000000000001"} + {"user_id": "00000000-0000-0000-0000-0000000000a1"} ) with pytest.raises(ValueError, match="legacy_household_id must not be null"): UserHouseholdUpdateInput(legacy_household_id=None) + with pytest.raises(ValueError, match="household_id must not be null"): + UserHouseholdUpdateInput(household_id=None) + + +def test_patch_input_accepts_a_v2_household_reference() -> None: + repoint = UserHouseholdUpdateInput.model_validate( + {"household_id": "00000000-0000-0000-0000-000000000001"} + ) + + assert repoint.household_id == UUID("00000000-0000-0000-0000-000000000001") + assert repoint.model_fields_set == {"household_id"} diff --git a/tests/unit/v2/user_service_support.py b/tests/unit/v2/user_service_support.py index ce54237e3..61f0f3f4b 100644 --- a/tests/unit/v2/user_service_support.py +++ b/tests/unit/v2/user_service_support.py @@ -106,6 +106,7 @@ class V2Store: policy_id: UUID uk_policy_id: UUID household_id: UUID + uk_household_id: UUID @contextmanager @@ -166,7 +167,14 @@ def begin_transaction(connection) -> None: canonicalization_version=1, content_hash="c" * 64, ) - session.add_all([value, uk_policy, household]) + uk_household = Household( + country_id="uk", + default_year=2026, + household_data={"people": [], "household": []}, + canonicalization_version=1, + content_hash="f" * 64, + ) + session.add_all([value, uk_policy, household, uk_household]) session.flush() session.add_all( [ @@ -192,6 +200,7 @@ def begin_transaction(connection) -> None: policy_id=policy.id, uk_policy_id=uk_policy.id, household_id=household.id, + uk_household_id=uk_household.id, ) try: yield store From 15f15aaa961401da2cba8948bf3f50dc3c4c3d5c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:24:27 +0400 Subject: [PATCH 009/118] Pin the status codes of a refused household repoint A household repoint can now fail with a reference conflict, and the household route tests sent only DELETE requests through the failure mapping. Send a repointing PATCH through it and pin each refusal's status: 400 for a country or reference conflict, 404 for a missing household or saved household, and 422 for a missing v1 household. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_user_household_routes.py | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unit/v2/test_user_household_routes.py b/tests/unit/v2/test_user_household_routes.py index 2e1553bcb..3f92084e4 100644 --- a/tests/unit/v2/test_user_household_routes.py +++ b/tests/unit/v2/test_user_household_routes.py @@ -9,6 +9,9 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.associations.validators import ( + AssociationReferenceConflictError, +) from policyengine_api.services.v2.legacy_refs.validators import ( LegacyReferenceNotFoundError, ) @@ -221,6 +224,37 @@ def test_failures_map_to_secret_safe_typed_errors( assert "credential" not in response.text +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (AssociationCountryConflictError("different country"), 400, "country"), + ( + AssociationReferenceConflictError("household_id mismatch"), + 400, + "mismatch", + ), + (AssociationHouseholdNotFoundError("household was not found"), 404, "not"), + (UserHouseholdNotFoundError("association was not found"), 404, "not"), + (LegacyReferenceNotFoundError("v1 household 9 was not found"), 422, "v1"), + ], +) +def test_repoint_failures_map_to_typed_errors( + error: Exception, status: int, message: str +) -> None: + service = RecordingService(_result) + service.error = error + client, _service, _flask_calls = _client(service) + + response = client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json={"household_id": str(HOUSEHOLD_ID), "legacy_household_id": 9}, + ) + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + + def test_patch_preflight_and_native_routes_avoid_cloud_sql_and_flask( monkeypatch, ) -> None: From 3d0b10533f922e4d9d9998c737da28e480217cf8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:05:51 +0400 Subject: [PATCH 010/118] Count saved policies by table name in the migration qualification commands The policy and user-association qualification commands counted saved policies through the UserPolicy model. Both run before every v2 upgrade, so once the model takes a new table name they would query a table the database does not have yet. The policy qualification counts only while its revision is pending, when the table is still named user_policies, so it names that table. The user-association qualification counts on every run, so it counts the table under whichever name the database has: user_policy_associations once renamed, otherwise user_policies. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/policy_migration_qualification.py | 19 ++++++-- ...ser_association_migration_qualification.py | 23 +++++++--- .../v2/test_policy_migration_qualification.py | 26 +++++++++++ ...ser_association_migration_qualification.py | 45 +++++++++++++++++++ 4 files changed, 104 insertions(+), 9 deletions(-) diff --git a/policyengine_api/data/v2/policy_migration_qualification.py b/policyengine_api/data/v2/policy_migration_qualification.py index beb03db24..d353226b1 100644 --- a/policyengine_api/data/v2/policy_migration_qualification.py +++ b/policyengine_api/data/v2/policy_migration_qualification.py @@ -10,12 +10,12 @@ from alembic.config import Config from alembic.migration import MigrationContext from alembic.script import ScriptDirectory -from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy import Connection, Engine, func, inspect, select, table from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.pool import NullPool from sqlmodel import create_engine -from policyengine_api.data.v2.models import ParameterValue, Policy, UserPolicy +from policyengine_api.data.v2.models import ParameterValue, Policy from policyengine_api.data.v2.settings import ( V2ConfigurationError, V2DatabaseSettings, @@ -26,6 +26,10 @@ from collections.abc import Callable, Collection, Mapping POLICY_MIGRATION_REVISION = "711ec2f0a5a5" +# The saved-policy table's name while the policy revision is pending. A later +# revision renames it to ``user_policy_associations``, so counting through +# the current model would query a table such a database does not have yet. +SAVED_POLICY_TABLE_AT_POLICY_REVISION = "user_policies" V2_ALEMBIC_CONFIG = "alembic-v2.ini" @@ -123,8 +127,15 @@ def table_exists(name: str) -> bool: if table_exists("parameter_values") else 0, user_policies=( - int(executor.scalar(select(func.count()).select_from(UserPolicy)) or 0) - if table_exists("user_policies") + int( + executor.scalar( + select(func.count()).select_from( + table(SAVED_POLICY_TABLE_AT_POLICY_REVISION) + ) + ) + or 0 + ) + if table_exists(SAVED_POLICY_TABLE_AT_POLICY_REVISION) else 0 ), ) diff --git a/policyengine_api/data/v2/user_association_migration_qualification.py b/policyengine_api/data/v2/user_association_migration_qualification.py index 9d7aad2a5..fac2b0d00 100644 --- a/policyengine_api/data/v2/user_association_migration_qualification.py +++ b/policyengine_api/data/v2/user_association_migration_qualification.py @@ -24,14 +24,13 @@ from alembic.config import Config from alembic.migration import MigrationContext from alembic.script import ScriptDirectory -from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy import Connection, Engine, func, inspect, select, table from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.pool import NullPool from sqlmodel import create_engine from policyengine_api.data.v2.models import ( UserHouseholdAssociation, - UserPolicy, UserReportAssociation, UserSimulationAssociation, ) @@ -44,7 +43,14 @@ if TYPE_CHECKING: from collections.abc import Callable, Collection, Mapping + from sqlalchemy.sql.expression import TableClause + USER_ASSOCIATION_MIGRATION_REVISION = "fbb7488ebe8a" +# The saved-policy table's name at this revision, and the name a later +# revision gives it. The command runs before every upgrade, so it counts the +# table under whichever name the database has. +SAVED_POLICY_TABLE_AT_RESHAPE = "user_policies" +SAVED_POLICY_TABLE_AFTER_RENAME = "user_policy_associations" V2_ALEMBIC_CONFIG = "alembic-v2.ini" @@ -122,13 +128,20 @@ def read_user_association_counts( ) -> UserAssociationCounts: """Count every row in the four user-association tables.""" - def count(model: type, table_name: str) -> int: + def count(source: type | TableClause, table_name: str) -> int: if existing_tables is not None and table_name not in existing_tables: return 0 - return int(executor.scalar(select(func.count()).select_from(model)) or 0) + return int(executor.scalar(select(func.count()).select_from(source)) or 0) + + saved_policy_table = ( + SAVED_POLICY_TABLE_AFTER_RENAME + if existing_tables is not None + and SAVED_POLICY_TABLE_AFTER_RENAME in existing_tables + else SAVED_POLICY_TABLE_AT_RESHAPE + ) return UserAssociationCounts( - user_policies=count(UserPolicy, "user_policies"), + user_policies=count(table(saved_policy_table), saved_policy_table), user_household_associations=count( UserHouseholdAssociation, "user_household_associations" ), diff --git a/tests/unit/v2/test_policy_migration_qualification.py b/tests/unit/v2/test_policy_migration_qualification.py index 54a711ef4..dc28306a5 100644 --- a/tests/unit/v2/test_policy_migration_qualification.py +++ b/tests/unit/v2/test_policy_migration_qualification.py @@ -175,3 +175,29 @@ def fail() -> None: assert "qualification failed unexpectedly" in captured.err assert "secret" not in captured.err assert "private-host" not in captured.err + + +class RecordingExecutor: + def __init__(self) -> None: + self.statements: list[str] = [] + + def scalar(self, statement: object) -> int: + self.statements.append(str(statement)) + return 0 + + +def test_saved_policies_are_counted_under_the_name_of_the_pending_revision() -> None: + """The policy revision is pending only before the rename of ``user_policies``. + + Counting through the current model would query ``user_policy_associations``, + which a database still waiting for the policy revision does not have yet. + """ + executor = RecordingExecutor() + + qualification.read_policy_data_counts( + executor, existing_tables={"policies", "parameter_values", "user_policies"} + ) + + saved_policy_count = executor.statements[2] + assert "FROM user_policies" in saved_policy_count + assert "user_policy_associations" not in saved_policy_count diff --git a/tests/unit/v2/test_user_association_migration_qualification.py b/tests/unit/v2/test_user_association_migration_qualification.py index 463a073f9..ac3997cac 100644 --- a/tests/unit/v2/test_user_association_migration_qualification.py +++ b/tests/unit/v2/test_user_association_migration_qualification.py @@ -143,3 +143,48 @@ def fail(): payload = json.loads(capsys.readouterr().err) assert payload["outcome"] == "error" assert "secret" not in json.dumps(payload) + + +def test_saved_policies_are_counted_under_the_name_of_the_pending_revision() -> None: + """The reshape is pending only before the later rename of ``user_policies``. + + Counting through the current model would query ``user_policy_associations``, + which a database still waiting for the reshape does not have yet. + """ + _evidence, connection, _engine = _qualify((0, 0, 0, 0), pending=True) + + saved_policy_count = connection.statements[0] + assert "FROM user_policies" in saved_policy_count + assert "user_policy_associations" not in saved_policy_count + + +class RecordingExecutor: + def __init__(self) -> None: + self.statements: list[str] = [] + + def scalar(self, statement: object) -> int: + self.statements.append(str(statement)) + return 3 + + +def test_saved_policies_are_counted_under_their_name_after_the_rename() -> None: + """Once ``user_policies`` is renamed, its rows are counted under the new name. + + The command runs before every upgrade, so it meets the table under either + name; reading only the old name would report no saved policies. + """ + executor = RecordingExecutor() + + counts = qualification.read_user_association_counts( + executor, + existing_tables={ + "user_policy_associations", + "user_household_associations", + "user_simulation_associations", + "user_report_associations", + }, + ) + + assert counts.user_policies == 3 + saved_policy_count = executor.statements[0] + assert "FROM user_policy_associations" in saved_policy_count From 320c5d20d85086d6c46d025163c2c9124829c023 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:32:39 +0400 Subject: [PATCH 011/118] Rename the v2 table user_policies to user_policy_associations The saved-policy table now follows the naming of the other user-association tables. Revision fa7fa1035444 was autogenerated from the renamed model. Its one documented post-generation correction replaces the generated create-and-drop pair, which would have deleted every saved policy, with an in-place rename; each constraint named after the table is dropped under its old name and created under the generated name, and the generated changes to the two convention-named indexes and the three foreign keys that refer to the table are kept around the rename. The composite listing indexes keep their ix_user_policies_* names, as the other association tables keep their short index names. The privilege declarations and the head revision constant use the new name. Dual-write's saved-policy copy goes through the model and follows it. Unchanged: the route /v2/user-policies, the user_policy_id columns, legacy_user_policy_mappings and the v1 Cloud SQL table user_policies. Not included: a temporary view named user_policies for the release still running during the deploy. A view is not autogenerated, so it needs a human migration decision; a strict expected-failure test records the gap. Co-Authored-By: Claude Opus 5.5 --- ...lan-1f-122-rename-user-policies.changed.md | 1 + docs/engineering/skills/alembic-migrations.md | 27 ++ docs/migration/stage-12a-user-login.md | 8 +- ...44_rename_user_policies_to_user_policy_.py | 292 ++++++++++++++ .../data/v2/metadata_validation.py | 4 +- policyengine_api/data/v2/migration_target.py | 2 +- .../data/v2/models/associations.py | 16 +- .../data/v2/models/policy_mappings.py | 2 +- policyengine_api/data/v2/privileges.py | 2 +- .../reads_good.py | 4 +- .../integration/test_alembic_v2_lifecycle.py | 356 +++++++++++++++++- tests/unit/v2/test_alembic_v2.py | 40 +- tests/unit/v2/test_models.py | 55 ++- tests/unit/v2/test_privilege_manifest.py | 2 +- tests/unit/v2/test_privilege_requirements.py | 8 +- .../v2/test_user_policy_legacy_persistence.py | 16 +- 16 files changed, 795 insertions(+), 40 deletions(-) create mode 100644 changelog.d/v2plan-1f-122-rename-user-policies.changed.md create mode 100644 migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py diff --git a/changelog.d/v2plan-1f-122-rename-user-policies.changed.md b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md new file mode 100644 index 000000000..1141ffd75 --- /dev/null +++ b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md @@ -0,0 +1 @@ +The v2 table user_policies is renamed to user_policy_associations, matching the other user-association tables; its constraints and convention-named indexes take the new name. Saved policies, their v1 mappings and the reports that name them are kept. The route /v2/user-policies, the user_policy_id columns, legacy_user_policy_mappings and the v1 Cloud SQL table user_policies are unchanged. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index d4a043d91..3b7a13f36 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -294,3 +294,30 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### API v2 saved-policy table rename + +Revision `fa7fa1035444` was autogenerated after renaming the `UserPolicy` +model's table from `user_policies` to `user_policy_associations`, with its +constraints and convention-named indexes. Autogeneration cannot detect a +rename: it created an empty `user_policy_associations` table and dropped +`user_policies`, which would delete every saved policy, and its copies of the +three composite indexes collided with the old table's indexes of the same +names. Its only post-generation correction replaces that pair with +`op.rename_table` and, for each constraint the generated table named after +`user_policy_associations` (primary key, unique, three checks and two foreign +keys), drops it under its old name and creates it under the generated name +with the generated definition. The generated drops and re-creations of the +two convention-named indexes and of the three foreign keys that refer to the +table are kept, ordered around the rename. The downgrade is the exact reverse. +The v2 Postgres lifecycle test stores a saved policy with a v1 mapping and a +report that names it, and checks that the rows, the references, the +cascades, a table grant and every constraint and index name survive the +upgrade and the downgrade. + +The plan also asks for a temporary view named `user_policies` over the renamed +table, so the release still running during the deploy keeps working. A view +is not autogenerated, and adding one would be a hand-written schema operation, +so the revision leaves it out pending a human migration decision. +`test_previous_release_still_finds_user_policies` records the gap as a strict +expected failure. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index 3fc391392..2831a4a60 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -324,7 +324,13 @@ from the final SQLModel metadata. It has no post-generation corrections. "Rollback"). The models drop the `user_associations` relationships from v2 simulations and -reports. `V2_ALEMBIC_HEAD_REVISION` is `fbb7488ebe8a`. +reports. `V2_ALEMBIC_HEAD_REVISION` was `fbb7488ebe8a` when Stage 12A shipped. + +Revision `fa7fa1035444` later renamed `user_policies` to +`user_policy_associations`, with its constraints and its convention-named +indexes, matching the other association tables; the composite listing +indexes keep their `ix_user_policies_*` names. The route `/v2/user-policies`, +the `user_policy_id` columns and `legacy_user_policy_mappings` are unchanged. ### Reshape safety diff --git a/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py new file mode 100644 index 000000000..d2aa619c5 --- /dev/null +++ b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py @@ -0,0 +1,292 @@ +"""rename user_policies to user_policy_associations + +Revision ID: fa7fa1035444 +Revises: fbb7488ebe8a +Create Date: 2026-10-22 17:07:03.720031 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +revision: str = "fa7fa1035444" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_legacy_user_policy_mappings_association_country"), + "legacy_user_policy_mappings", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_baseline_user_policy_id_use_7740"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_user_policy_id_user_policies"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") + op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") + # Post-generation correction: autogeneration cannot detect a rename. It + # created an empty user_policy_associations table and dropped user_policies, + # which would delete every saved policy (its three composite indexes would + # also have collided with the old table's indexes of the same names). The + # table is renamed in place instead, keeping its rows, grants and composite + # indexes. Each constraint the generated table named after + # user_policy_associations is dropped under its old name and created under + # the generated name with the generated definition. The foreign keys that + # refer to this table were dropped above, before its primary key and unique + # constraint. + op.rename_table("user_policies", "user_policy_associations") + op.drop_constraint("pk_user_policies", "user_policy_associations", type_="primary") + op.create_primary_key( + op.f("pk_user_policy_associations"), "user_policy_associations", ["id"] + ) + op.drop_constraint( + "uq_user_policies_id_country", "user_policy_associations", type_="unique" + ) + op.create_unique_constraint( + "uq_user_policy_associations_id_country", + "user_policy_associations", + ["id", "country_id"], + ) + op.drop_constraint( + "ck_user_policies_baseline", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_baseline"), + "user_policy_associations", + "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", + ) + op.drop_constraint( + "ck_user_policies_country", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + "country_id IN ('us', 'uk')", + ) + op.drop_constraint( + "ck_user_policies_policy_ref", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_policy_ref"), + "user_policy_associations", + "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", + ) + op.drop_constraint( + "fk_user_policies_policy_country", + "user_policy_associations", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_user_policy_associations_policy_country", + "user_policy_associations", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + "fk_user_policies_user_id_users", + "user_policy_associations", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_user_policy_associations_user_id_users"), + "user_policy_associations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + op.f("ix_user_policy_associations_policy_id"), + "user_policy_associations", + ["policy_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_policy_associations_user_id"), + "user_policy_associations", + ["user_id"], + unique=False, + ) + op.create_foreign_key( + "fk_legacy_user_policy_mappings_association_country", + "legacy_user_policy_mappings", + "user_policy_associations", + ["user_policy_id", "country_id"], + ["id", "country_id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_user_policy_id_user_policy_associations"), + "user_report_associations", + "user_policy_associations", + ["user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f( + "fk_user_report_associations_baseline_user_policy_id_user_policy_associations" + ), + "user_report_associations", + "user_policy_associations", + ["baseline_user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f( + "fk_user_report_associations_baseline_user_policy_id_user_policy_associations" + ), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_user_policy_id_user_policy_associations"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + "fk_legacy_user_policy_mappings_association_country", + "legacy_user_policy_mappings", + type_="foreignkey", + ) + op.drop_index( + op.f("ix_user_policy_associations_user_id"), + table_name="user_policy_associations", + ) + op.drop_index( + op.f("ix_user_policy_associations_policy_id"), + table_name="user_policy_associations", + ) + # Post-generation correction: the reverse of the upgrade's rename. The + # generated downgrade created an empty user_policies table and dropped + # user_policy_associations, which would delete every saved policy. The + # table is renamed back in place, and each constraint is dropped under its + # new name and created under the name and definition it has at revision + # fbb7488ebe8a. + op.drop_constraint( + op.f("fk_user_policy_associations_user_id_users"), + "user_policy_associations", + type_="foreignkey", + ) + op.drop_constraint( + "fk_user_policy_associations_policy_country", + "user_policy_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_policy_ref"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_baseline"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + "uq_user_policy_associations_id_country", + "user_policy_associations", + type_="unique", + ) + op.drop_constraint( + op.f("pk_user_policy_associations"), + "user_policy_associations", + type_="primary", + ) + op.rename_table("user_policy_associations", "user_policies") + op.create_primary_key(op.f("pk_user_policies"), "user_policies", ["id"]) + op.create_unique_constraint( + op.f("uq_user_policies_id_country"), "user_policies", ["id", "country_id"] + ) + op.create_check_constraint( + op.f("ck_user_policies_baseline"), + "user_policies", + "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), + "user_policies", + "country_id IN ('us', 'uk')", + ) + op.create_check_constraint( + op.f("ck_user_policies_policy_ref"), + "user_policies", + "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", + ) + op.create_foreign_key( + op.f("fk_user_policies_policy_country"), + "user_policies", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + op.f("ix_user_policies_user_id"), "user_policies", ["user_id"], unique=False + ) + op.create_index( + op.f("ix_user_policies_policy_id"), + "user_policies", + ["policy_id"], + unique=False, + ) + op.create_foreign_key( + op.f("fk_user_report_associations_user_policy_id_user_policies"), + "user_report_associations", + "user_policies", + ["user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_baseline_user_policy_id_use_7740"), + "user_report_associations", + "user_policies", + ["baseline_user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f("fk_legacy_user_policy_mappings_association_country"), + "legacy_user_policy_mappings", + "user_policies", + ["user_policy_id", "country_id"], + ["id", "country_id"], + ondelete="CASCADE", + ) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/metadata_validation.py b/policyengine_api/data/v2/metadata_validation.py index 7445269f1..ab60ab93a 100644 --- a/policyengine_api/data/v2/metadata_validation.py +++ b/policyengine_api/data/v2/metadata_validation.py @@ -3,8 +3,8 @@ from collections.abc import Iterable # These names belong only to the existing v1 database domain. Names that are -# intentionally shared between v1 and v2, such as `simulations` and -# `user_policies`, are excluded from this targeted rejection set. +# intentionally shared between v1 and v2, such as `simulations`, are excluded +# from this targeted rejection set. V1_ONLY_TABLES = frozenset( { "analysis", diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..a57c2419c 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "fa7fa1035444" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index 2c5cd21b1..4823e87fb 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -106,29 +106,29 @@ class UserPolicy(TimestampedModel, table=True): policy that users have saved cannot be deleted. """ - __tablename__ = "user_policies" + __tablename__ = "user_policy_associations" __table_args__ = ( sa.UniqueConstraint( "id", "country_id", - name="uq_user_policies_id_country", + name="uq_user_policy_associations_id_country", ), sa.CheckConstraint( "country_id IN ('us', 'uk')", - name="ck_user_policies_country", + name="ck_user_policy_associations_country", ), sa.CheckConstraint( "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", - name="ck_user_policies_policy_ref", + name="ck_user_policy_associations_policy_ref", ), sa.CheckConstraint( "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", - name="ck_user_policies_baseline", + name="ck_user_policy_associations_baseline", ), sa.ForeignKeyConstraint( ["policy_id", "country_id"], ["policies.id", "policies.country_id"], - name="fk_user_policies_policy_country", + name="fk_user_policy_associations_policy_country", ondelete="RESTRICT", ), sa.Index( @@ -278,7 +278,7 @@ class UserReportAssociation(TimestampedModel, table=True): description: str | None = Field(default=None, sa_type=sa.Text) user_policy_id: UUID | None = Field( default=None, - foreign_key="user_policies.id", + foreign_key="user_policy_associations.id", ondelete="SET NULL", index=True, ) @@ -286,7 +286,7 @@ class UserReportAssociation(TimestampedModel, table=True): # comparison was not current law; its name is the baseline's label. baseline_user_policy_id: UUID | None = Field( default=None, - foreign_key="user_policies.id", + foreign_key="user_policy_associations.id", ondelete="SET NULL", index=True, ) diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index 7cdbc91b9..69aef5642 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -104,7 +104,7 @@ class LegacyUserPolicyMapping(TimestampedModel, table=True): ), sa.ForeignKeyConstraint( ["user_policy_id", "country_id"], - ["user_policies.id", "user_policies.country_id"], + ["user_policy_associations.id", "user_policy_associations.country_id"], name="fk_legacy_user_policy_mappings_association_country", ondelete="CASCADE", ), diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 8f8c34515..dc965e36e 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -135,7 +135,7 @@ def _access( # Users and their saved work (Stage 12A). "user_household_associations": _access(runtime=READ_WRITE), "user_identities": _access(runtime=READ_WRITE), - "user_policies": _access(runtime=READ_WRITE), + "user_policy_associations": _access(runtime=READ_WRITE), "user_report_associations": _access(runtime=READ_WRITE), "user_simulation_associations": _access(runtime=READ_WRITE), "users": _access(runtime=READ_WRITE), diff --git a/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py b/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py index 5437a95c0..0f7567704 100644 --- a/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py +++ b/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py @@ -1,7 +1,7 @@ """Good fixture: a runtime read of only the tables the runtime role is granted. -The user-policy lookup reads ``user_policies``, which the runtime role holds -SELECT on, and nothing else. +The user-policy lookup reads ``user_policy_associations``, which the runtime +role holds SELECT on, and nothing else. """ from __future__ import annotations diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 814971236..694fe0363 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -30,6 +30,7 @@ STAGE_12_PREVIOUS_REVISION = "724b1b11a33e" STAGE_12_COMPARISON_PREVIOUS_REVISION = "439303be14fe" STAGE_12A_PREVIOUS_REVISION = "91af4fc8d8c1" +SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -147,7 +148,7 @@ def _assert_stage_12a_association_shape(engine) -> None: "subject", ] for table_name, reference in ( - ("user_policies", "policy"), + ("user_policy_associations", "policy"), ("user_household_associations", "household"), ("user_simulation_associations", "simulation"), ("user_report_associations", "report"), @@ -171,7 +172,7 @@ def _assert_stage_12a_association_shape(engine) -> None: report_policy_key = next( foreign_key for foreign_key in inspect(engine).get_foreign_keys("user_report_associations") - if foreign_key["referred_table"] == "user_policies" + if foreign_key["referred_table"] == "user_policy_associations" ) assert report_policy_key["options"]["ondelete"] == "SET NULL" for table_name in ("user_simulation_associations", "user_report_associations"): @@ -408,6 +409,11 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_simulation_associations", "user_report_associations", ) +# Tables head renamed, keyed by their name at the parent revision. The +# previous release's statements name the old table, which head does not have +# until a view under that name exists (test_previous_release_still_finds_ +# user_policies), so its column usage runs against the renamed table here. +RENAMED_AT_HEAD = {"user_policies": "user_policy_associations"} COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) # Nullable references that a parent-revision check constraint requires. REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) @@ -587,7 +593,7 @@ def _assert_user_deletion_cascades( assert result.rowcount == 1 for table_name in (*USER_ASSOCIATION_TABLES, "user_identities"): - table = head.tables[table_name] + table = head.tables[RENAMED_AT_HEAD.get(table_name, table_name)] remaining = connection.execute( sa.select(sa.func.count()) .select_from(table) @@ -604,7 +610,8 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: requires), read the row back with every mapped column, rewrite those columns, delete it, and delete a user who owns Stage 12A rows so the cascades run. Head may not give any table the parent revision writes a - new required column without a server default. + new required column without a server default. A table head renamed is + used under its new name (``RENAMED_AT_HEAD``). """ database_url = _disposable_url() config = _config() @@ -614,6 +621,8 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) parent = sa.MetaData() parent.reflect(engine) + for parent_name, head_name in RENAMED_AT_HEAD.items(): + parent.tables[parent_name].name = head_name command.upgrade(config, "head") head = sa.MetaData() head.reflect(engine) @@ -957,3 +966,342 @@ def reflected(name: str) -> sa.Table: connection.execute(sa.delete(table).where(table.c.id == row_id)) command.upgrade(config, "head") engine.dispose() + + +@pytest.mark.xfail( + strict=True, + raises=sa.exc.ProgrammingError, + reason=( + "Head has no view named user_policies yet. Autogeneration cannot " + "create one, and a hand-written view needs a human migration decision " + "(see the saved-policy table rename in the Alembic migrations skill); " + "until then the release running during a deploy cannot find the table." + ), +) +def test_previous_release_still_finds_user_policies() -> None: + """The release running during the deploy reads saved policies by the old name.""" + engine = create_engine(_disposable_url()) + try: + with engine.connect() as connection: + connection.execute(text("SELECT count(*) FROM user_policies")).scalar_one() + finally: + engine.dispose() + + +def _constraint_names(engine, table_name: str) -> set[str]: + with engine.connect() as connection: + return set( + connection.execute( + text( + "SELECT conname FROM pg_constraint " + "WHERE conrelid = to_regclass(:table_name)" + ), + {"table_name": f"public.{table_name}"}, + ).scalars() + ) + + +def _index_names(engine, table_name: str) -> set[str]: + with engine.connect() as connection: + return set( + connection.execute( + text( + "SELECT indexname FROM pg_indexes " + "WHERE schemaname = 'public' AND tablename = :table_name" + ), + {"table_name": table_name}, + ).scalars() + ) + + +def _referring_foreign_keys(engine, table_name: str) -> set[tuple[str, str]]: + with engine.connect() as connection: + return { + (row.on_table, row.conname) + for row in connection.execute( + text( + "SELECT conrelid::regclass::text AS on_table, conname " + "FROM pg_constraint WHERE contype = 'f' " + "AND confrelid = to_regclass(:table_name)" + ), + {"table_name": f"public.{table_name}"}, + ) + } + + +def _closure_of_referred_tables( + metadata: sa.MetaData, table_names: set[str] +) -> set[str]: + closure: set[str] = set() + pending = list(table_names) + while pending: + table_name = pending.pop() + if table_name in closure: + continue + closure.add(table_name) + pending.extend( + foreign_key.referred_table.name + for foreign_key in metadata.tables[table_name].foreign_key_constraints + ) + return closure + + +def _saved_policy_with_references(connection, parent: sa.MetaData) -> dict: + """Store a user, a policy, a saved policy, its v1 mapping and a report.""" + sequence = itertools.count() + stored_rows: dict[str, dict] = {} + owner_tables = _closure_of_referred_tables(parent, {"users", "policies"}) + for table in parent.sorted_tables: + if table.name in owner_tables: + values = _row_values(table, stored_rows, sequence) + if table.name == "users": + # A standard user needs a country (ck_users_standard_primary_country). + values["primary_country"] = "us" + stored_rows[table.name] = _insert_and_read_back(connection, table, values) + saved_policy_id = uuid4() + connection.execute( + parent.tables["user_policies"] + .insert() + .values( + id=saved_policy_id, + user_id=stored_rows["users"]["id"], + policy_id=stored_rows["policies"]["id"], + country_id=stored_rows["policies"]["country_id"], + name="Saved reform", + baseline="current-law", + ) + ) + connection.execute( + parent.tables["legacy_user_policy_mappings"] + .insert() + .values( + id=uuid4(), + country_id=stored_rows["policies"]["country_id"], + legacy_user_policy_id=41, + user_policy_id=saved_policy_id, + fingerprint_version=1, + fingerprint_sha256="a" * 64, + ) + ) + report_id = uuid4() + connection.execute( + parent.tables["user_report_associations"] + .insert() + .values( + id=report_id, + user_id=stored_rows["users"]["id"], + country_id=stored_rows["policies"]["country_id"], + legacy_report_id=42, + user_policy_id=saved_policy_id, + baseline_user_policy_id=saved_policy_id, + ) + ) + return { + "stored_rows": stored_rows, + "saved_policy_id": saved_policy_id, + "report_id": report_id, + } + + +def _saved_policy_row(connection, table_name: str, saved_policy_id: UUID) -> dict: + return dict( + connection.execute( + sa.select(sa.text("*")) + .select_from(sa.table(table_name)) + .where(sa.column("id") == saved_policy_id) + ) + .mappings() + .one() + ) + + +def _delete_saved_policy_rows(engine, parent: sa.MetaData, saved: dict) -> None: + """Remove the stored rows at head, children first, so later tests start clean.""" + with engine.begin() as connection: + connection.execute( + text("DELETE FROM user_report_associations WHERE id = :id"), + {"id": saved["report_id"]}, + ) + # Deleting the saved policy cascades to its v1 mapping. + connection.execute( + text("DELETE FROM user_policy_associations WHERE id = :id"), + {"id": saved["saved_policy_id"]}, + ) + for table in reversed(parent.sorted_tables): + stored_row = saved["stored_rows"].get(table.name) + if stored_row is not None: + connection.execute( + table.delete().where(_primary_key_clause(table, stored_row)) + ) + + +# The names head gives the renamed table's constraints, indexes and the +# foreign keys that refer to it. Names longer than PostgreSQL's 63 characters +# end in a hash, as SQLAlchemy shortens the model's convention names. +SAVED_POLICY_SCHEMA_NAMES_AT_HEAD = { + "constraints": { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + "ck_user_policy_associations_country", + "ck_user_policy_associations_policy_ref", + "ck_user_policy_associations_baseline", + "fk_user_policy_associations_policy_country", + "fk_user_policy_associations_user_id_users", + }, + "indexes": { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + "ix_user_policy_associations_user_id", + "ix_user_policy_associations_policy_id", + "ix_user_policies_country_user_created_id", + "ix_user_policies_country_policy", + "ix_user_policies_country_legacy_policy", + }, + "referring_foreign_keys": { + ( + "legacy_user_policy_mappings", + "fk_legacy_user_policy_mappings_association_country", + ), + ( + "user_report_associations", + "fk_user_report_associations_user_policy_id_user_policy__0820", + ), + ( + "user_report_associations", + "fk_user_report_associations_baseline_user_policy_id_use_0a3d", + ), + }, +} +RENAME_PRIVILEGE_PROBE_ROLE = "rename_privilege_probe" + + +def _schema_names(engine, table_name: str) -> dict[str, set]: + return { + "constraints": _constraint_names(engine, table_name), + "indexes": _index_names(engine, table_name), + "referring_foreign_keys": _referring_foreign_keys(engine, table_name), + } + + +def _saved_policy_references(connection, saved: dict) -> tuple: + """Return the saved policy a v1 mapping and a report's two keys point at.""" + mapped = connection.execute( + text( + "SELECT user_policy_id FROM legacy_user_policy_mappings " + "WHERE legacy_user_policy_id = 41" + ) + ).scalar_one_or_none() + report_references = connection.execute( + text( + "SELECT user_policy_id, baseline_user_policy_id " + "FROM user_report_associations WHERE id = :id" + ), + {"id": saved["report_id"]}, + ).one() + return (mapped, *report_references) + + +def _references_after_deleting_saved_policy(engine, saved: dict) -> tuple: + """Delete the saved policy in a rolled-back transaction; return the references. + + The v1 mapping cascades away and the report's keys are set to null. + """ + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.execute( + text("DELETE FROM user_policy_associations WHERE id = :id"), + {"id": saved["saved_policy_id"]}, + ) + return _saved_policy_references(connection, saved) + finally: + transaction.rollback() + + +def _drop_rename_privilege_probe(engine) -> None: + with engine.begin() as connection: + connection.execute( + text( + "DO $$ BEGIN IF EXISTS (SELECT FROM pg_roles WHERE rolname = " + f"'{RENAME_PRIVILEGE_PROBE_ROLE}') THEN " + f"DROP OWNED BY {RENAME_PRIVILEGE_PROBE_ROLE}; " + f"DROP ROLE {RENAME_PRIVILEGE_PROBE_ROLE}; END IF; END $$" + ) + ) + + +def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: + """Upgrading renames the table in place, and downgrading renames it back. + + Saved policies, their v1 mappings and the reports that name them survive + both directions, the references and a table grant follow the table, and + the constraints and indexes named after the table take its name. The + downgrade restores every name the parent revision had. + """ + config = _config() + engine = create_engine(_disposable_url()) + parent = sa.MetaData() + saved = None + try: + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + parent.reflect(engine) + names_at_parent = _schema_names(engine, "user_policies") + _drop_rename_privilege_probe(engine) + with engine.begin() as connection: + saved = _saved_policy_with_references(connection, parent) + connection.execute(text(f"CREATE ROLE {RENAME_PRIVILEGE_PROBE_ROLE}")) + connection.execute( + text( + "GRANT SELECT, UPDATE ON user_policies TO " + f"{RENAME_PRIVILEGE_PROBE_ROLE}" + ) + ) + before = _saved_policy_row( + connection, "user_policies", saved["saved_policy_id"] + ) + saved_policy_id = saved["saved_policy_id"] + + command.upgrade(config, "head") + + table_names = set(inspect(engine).get_table_names(schema="public")) + assert "user_policies" not in table_names + with engine.connect() as connection: + after = _saved_policy_row( + connection, "user_policy_associations", saved_policy_id + ) + references = _saved_policy_references(connection, saved) + granted = connection.execute( + text( + f"SELECT has_table_privilege('{RENAME_PRIVILEGE_PROBE_ROLE}', " + "'public.user_policy_associations', 'SELECT, UPDATE')" + ) + ).scalar_one() + assert after == before + assert references == (saved_policy_id, saved_policy_id, saved_policy_id) + assert granted + assert ( + _schema_names(engine, "user_policy_associations") + == SAVED_POLICY_SCHEMA_NAMES_AT_HEAD + ) + assert _references_after_deleting_saved_policy(engine, saved) == ( + None, + None, + None, + ) + + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + + table_names = set(inspect(engine).get_table_names(schema="public")) + assert "user_policy_associations" not in table_names + with engine.connect() as connection: + restored = _saved_policy_row(connection, "user_policies", saved_policy_id) + restored_references = _saved_policy_references(connection, saved) + assert restored == before + assert restored_references == references + assert _schema_names(engine, "user_policies") == names_at_parent + finally: + command.upgrade(config, "head") + if saved is not None: + _delete_saved_policy_rows(engine, parent, saved) + _drop_rename_privilege_probe(engine) + engine.dispose() diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 4c0e6ee09..f031d04b3 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -54,6 +54,8 @@ f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" "postgres?sslmode=require" ) +# The Stage 12A revision; the saved-policy rename follows it. +SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" f"{DISPOSABLE_DATABASE_NAME}" @@ -235,6 +237,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + SAVED_POLICY_RENAME_PARENT_REVISION, "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -495,7 +498,6 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -739,3 +741,39 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "current_heads != previous_heads" in env_source assert "current_heads == script_heads" in env_source assert "validate_v2_head_schema(connection, target_metadata)" in env_source + + +def _revision_source(revision_id: str) -> str: + (path,) = (REPO / "migrations/v2/versions").glob(f"{revision_id}_*.py") + return path.read_text(encoding="utf-8") + + +def test_saved_policy_rename_revision_renames_the_table_in_place() -> None: + """The rename keeps every saved reform: no table is created or dropped. + + Autogeneration replaces a renamed table with a new empty one; the revision + records its corrections, and every other operation is generated. + """ + revision = _revision_source(V2_ALEMBIC_HEAD_REVISION) + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert V2_ALEMBIC_HEAD_REVISION != SAVED_POLICY_RENAME_PARENT_REVISION + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert ( + f'down_revision: Union[str, None] = "{SAVED_POLICY_RENAME_PARENT_REVISION}"' + ) in revision + assert 'op.rename_table("user_policies", "user_policy_associations")' in upgrade + assert 'op.rename_table("user_policy_associations", "user_policies")' in downgrade + for body in (upgrade, downgrade): + assert "op.create_table(" not in body + assert "op.drop_table(" not in body + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert upgrade.count("Post-generation correction:") == 1 + assert downgrade.count("Post-generation correction:") == 1 diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 20947ffcb..17b87ec03 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -130,7 +130,7 @@ def test_user_owned_associations_have_relational_integrity() -> None: "user_simulation_associations", "simulation_associations", ), - (UserPolicy, "user_policies", "policy_associations"), + (UserPolicy, "user_policy_associations", "policy_associations"), (UserReportAssociation, "user_report_associations", "report_associations"), ) @@ -195,7 +195,7 @@ def test_policy_parameter_values_use_jsonb_and_enforce_period_identity() -> None def test_user_policy_is_an_independent_country_scoped_association() -> None: - associations = V2_METADATA.tables["user_policies"] + associations = V2_METADATA.tables["user_policy_associations"] assert isinstance(associations.c.user_id.type, sa.Uuid) user_foreign_key = next(iter(associations.c.user_id.foreign_keys)) @@ -215,7 +215,7 @@ def test_user_policy_is_an_independent_country_scoped_association() -> None: policy_country = next( constraint for constraint in associations.foreign_key_constraints - if constraint.name == "fk_user_policies_policy_country" + if constraint.name == "fk_user_policy_associations_policy_country" ) assert [column.name for column in policy_country.columns] == [ "policy_id", @@ -571,7 +571,7 @@ def test_user_identity_maps_one_issuer_subject_to_one_user() -> None: def test_associations_are_v1_anchored_without_client_refs() -> None: for table_name, reference in ( - ("user_policies", "policy"), + ("user_policy_associations", "policy"), ("user_household_associations", "household"), ("user_simulation_associations", "simulation"), ("user_report_associations", "report"), @@ -610,7 +610,7 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None } == {"users.id"} assert {foreign_key.target_fullname for foreign_key in reports.foreign_keys} == { "users.id", - "user_policies.id", + "user_policy_associations.id", } user_policy_key = next(iter(reports.c.user_policy_id.foreign_keys)) assert user_policy_key.ondelete == "SET NULL" @@ -623,15 +623,56 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None def test_user_policy_records_v3_reform_fields() -> None: - policies = V2_METADATA.tables["user_policies"] + policies = V2_METADATA.tables["user_policy_associations"] assert policies.c.baseline.type.length == 16 baseline_check = next( constraint for constraint in policies.constraints - if constraint.name == "ck_user_policies_baseline" + if constraint.name == "ck_user_policy_associations_baseline" ) assert "current-law" in str(baseline_check.sqltext) assert "current-policy" in str(baseline_check.sqltext) postgres_type = policies.c.provenance.type.dialect_impl(postgresql.dialect()) assert isinstance(postgres_type, postgresql.JSONB) + + +def test_saved_policy_table_is_named_like_the_other_association_tables() -> None: + """The saved-reform table and its constraints follow the association pattern. + + Composite indexes keep their short ``ix_user_policies_*`` names, as the + household, simulation and report associations keep ``ix_user_households_*`` + and the like; constraints and convention-named indexes take the table name. + """ + assert UserPolicy.__tablename__ == "user_policy_associations" + assert "user_policies" not in V2_METADATA.tables + table = V2_METADATA.tables["user_policy_associations"] + + assert {constraint.name for constraint in table.constraints} == { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + "ck_user_policy_associations_country", + "ck_user_policy_associations_policy_ref", + "ck_user_policy_associations_baseline", + "fk_user_policy_associations_policy_country", + "fk_user_policy_associations_user_id_users", + } + assert {index.name for index in table.indexes} == { + "ix_user_policy_associations_user_id", + "ix_user_policy_associations_policy_id", + "ix_user_policies_country_user_created_id", + "ix_user_policies_country_policy", + "ix_user_policies_country_legacy_policy", + } + referring_keys = { + (foreign_key.parent.table.name, foreign_key.parent.name) + for referring_table in V2_METADATA.tables.values() + for foreign_key in referring_table.foreign_keys + if foreign_key.column.table is table + } + assert referring_keys == { + ("legacy_user_policy_mappings", "user_policy_id"), + ("legacy_user_policy_mappings", "country_id"), + ("user_report_associations", "user_policy_id"), + ("user_report_associations", "baseline_user_policy_id"), + } diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 7a3a76b44..1168d4888 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -133,7 +133,7 @@ def test_rule_proof_runtime_privileges_declared() -> None: reads ``legacy_policy_mappings``, which the runtime role had no grant on. """ runtime_before_3977 = dict.fromkeys(TABLE_ACCESS, NO_ACCESS) | { - "user_policies": READ_WRITE + "user_policy_associations": READ_WRITE } table_by_model = _table_by_model() diff --git a/tests/unit/v2/test_privilege_requirements.py b/tests/unit/v2/test_privilege_requirements.py index 76ba0b5eb..314a85aff 100644 --- a/tests/unit/v2/test_privilege_requirements.py +++ b/tests/unit/v2/test_privilege_requirements.py @@ -20,7 +20,7 @@ TABLE_BY_MODEL = { "User": "users", - "UserPolicy": "user_policies", + "UserPolicy": "user_policy_associations", "LegacyPolicyMapping": "legacy_policy_mappings", } @@ -72,8 +72,8 @@ def create(session, values): ) assert _needed(path) == { - ("user_policies", TablePrivilege.INSERT), - ("user_policies", TablePrivilege.SELECT), + ("user_policy_associations", TablePrivilege.INSERT), + ("user_policy_associations", TablePrivilege.SELECT), } @@ -169,5 +169,5 @@ def read_with_owner(session): """, ) - assert ("user_policies", TablePrivilege.UPDATE) in _needed(path) + assert ("user_policy_associations", TablePrivilege.UPDATE) in _needed(path) assert ("users", TablePrivilege.UPDATE) not in _needed(path) diff --git a/tests/unit/v2/test_user_policy_legacy_persistence.py b/tests/unit/v2/test_user_policy_legacy_persistence.py index e52134e67..a6911a0db 100644 --- a/tests/unit/v2/test_user_policy_legacy_persistence.py +++ b/tests/unit/v2/test_user_policy_legacy_persistence.py @@ -44,13 +44,15 @@ SELECT_MAPPING_FOR_UPDATE = f"{SELECT_MAPPING} FOR UPDATE" MAPPING_LOOKUP = {"country_id_1": "us", "legacy_user_policy_id_1": 10} SELECT_MAPPED_ASSOCIATION = ( - "SELECT user_policies.id, user_policies.created_at, user_policies.updated_at, " - "user_policies.user_id, user_policies.policy_id, " - "user_policies.legacy_policy_id, user_policies.country_id, " - "user_policies.name, user_policies.description, user_policies.baseline, " - "user_policies.provenance \nFROM user_policies \n" - "WHERE user_policies.id = %(id_1)s::UUID AND " - "user_policies.country_id = %(country_id_1)s::VARCHAR" + "SELECT user_policy_associations.id, user_policy_associations.created_at, " + "user_policy_associations.updated_at, user_policy_associations.user_id, " + "user_policy_associations.policy_id, " + "user_policy_associations.legacy_policy_id, " + "user_policy_associations.country_id, user_policy_associations.name, " + "user_policy_associations.description, user_policy_associations.baseline, " + "user_policy_associations.provenance \nFROM user_policy_associations \n" + "WHERE user_policy_associations.id = %(id_1)s::UUID AND " + "user_policy_associations.country_id = %(country_id_1)s::VARCHAR" ) INSERT_MAPPING = ( "INSERT INTO legacy_user_policy_mappings (id, created_at, updated_at, " From 54bef66a19d81508f09daac354985824c1b8d39a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:54:39 +0400 Subject: [PATCH 012/118] Tighten the saved-policy rename tests - The revision test finds fa7fa1035444 by its ID instead of through the head constant, so a later revision no longer makes it read the wrong file, and it checks the revision's parent through the script directory. - The lifecycle test checks each privilege of the probe role on its own (has_table_privilege with a list is true when any one is held), and checks grants and delete cascades after the downgrade as well as after the upgrade. - The test for the missing user_policies view now stores a saved policy and upgrades to head before reading it under the old name, and only a missing relation counts as the expected failure, so an empty database or a failure while setting up no longer satisfies it. Co-Authored-By: Claude Opus 5.5 --- .../integration/test_alembic_v2_lifecycle.py | 135 +++++++++++++----- tests/unit/v2/test_alembic_v2.py | 17 +-- 2 files changed, 107 insertions(+), 45 deletions(-) diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 694fe0363..6a2b4b129 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 +import psycopg import pytest import sqlalchemy as sa from alembic import command @@ -968,26 +969,6 @@ def reflected(name: str) -> sa.Table: engine.dispose() -@pytest.mark.xfail( - strict=True, - raises=sa.exc.ProgrammingError, - reason=( - "Head has no view named user_policies yet. Autogeneration cannot " - "create one, and a hand-written view needs a human migration decision " - "(see the saved-policy table rename in the Alembic migrations skill); " - "until then the release running during a deploy cannot find the table." - ), -) -def test_previous_release_still_finds_user_policies() -> None: - """The release running during the deploy reads saved policies by the old name.""" - engine = create_engine(_disposable_url()) - try: - with engine.connect() as connection: - connection.execute(text("SELECT count(*) FROM user_policies")).scalar_one() - finally: - engine.dispose() - - def _constraint_names(engine, table_name: str) -> set[str]: with engine.connect() as connection: return set( @@ -1201,7 +1182,9 @@ def _saved_policy_references(connection, saved: dict) -> tuple: return (mapped, *report_references) -def _references_after_deleting_saved_policy(engine, saved: dict) -> tuple: +def _references_after_deleting_saved_policy( + engine, table_name: str, saved: dict +) -> tuple: """Delete the saved policy in a rolled-back transaction; return the references. The v1 mapping cascades away and the report's keys are set to null. @@ -1210,14 +1193,38 @@ def _references_after_deleting_saved_policy(engine, saved: dict) -> tuple: transaction = connection.begin() try: connection.execute( - text("DELETE FROM user_policy_associations WHERE id = :id"), - {"id": saved["saved_policy_id"]}, + sa.delete(sa.table(table_name, sa.column("id"))).where( + sa.column("id") == saved["saved_policy_id"] + ) ) return _saved_policy_references(connection, saved) finally: transaction.rollback() +def _probe_privileges(connection, table_name: str) -> dict[str, bool]: + """Return which of three privileges the probe role holds, one at a time. + + ``has_table_privilege`` with a list of privileges is true when any one is + held, so each is checked separately. + """ + return { + privilege: connection.execute( + text("SELECT has_table_privilege(:role, :table, :privilege)"), + { + "role": RENAME_PRIVILEGE_PROBE_ROLE, + "table": f"public.{table_name}", + "privilege": privilege, + }, + ).scalar_one() + for privilege in ("SELECT", "UPDATE", "INSERT") + } + + +GRANTED_TO_PROBE = {"SELECT": True, "UPDATE": True, "INSERT": False} +CLEARED_REFERENCES = (None, None, None) + + def _drop_rename_privilege_probe(engine) -> None: with engine.begin() as connection: connection.execute( @@ -1270,23 +1277,19 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: connection, "user_policy_associations", saved_policy_id ) references = _saved_policy_references(connection, saved) - granted = connection.execute( - text( - f"SELECT has_table_privilege('{RENAME_PRIVILEGE_PROBE_ROLE}', " - "'public.user_policy_associations', 'SELECT, UPDATE')" - ) - ).scalar_one() + privileges = _probe_privileges(connection, "user_policy_associations") assert after == before assert references == (saved_policy_id, saved_policy_id, saved_policy_id) - assert granted + assert privileges == GRANTED_TO_PROBE assert ( _schema_names(engine, "user_policy_associations") == SAVED_POLICY_SCHEMA_NAMES_AT_HEAD ) - assert _references_after_deleting_saved_policy(engine, saved) == ( - None, - None, - None, + assert ( + _references_after_deleting_saved_policy( + engine, "user_policy_associations", saved + ) + == CLEARED_REFERENCES ) command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) @@ -1296,12 +1299,76 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: with engine.connect() as connection: restored = _saved_policy_row(connection, "user_policies", saved_policy_id) restored_references = _saved_policy_references(connection, saved) + restored_privileges = _probe_privileges(connection, "user_policies") assert restored == before assert restored_references == references + assert restored_privileges == GRANTED_TO_PROBE assert _schema_names(engine, "user_policies") == names_at_parent + assert ( + _references_after_deleting_saved_policy(engine, "user_policies", saved) + == CLEARED_REFERENCES + ) finally: command.upgrade(config, "head") if saved is not None: _delete_saved_policy_rows(engine, parent, saved) _drop_rename_privilege_probe(engine) engine.dispose() + + +class OldTableNameMissingError(Exception): + """Raised when head has no relation named ``user_policies`` to read from.""" + + +def _saved_policy_row_by_old_name(connection, saved_policy_id: UUID) -> dict: + """Read a saved policy as the previous release does, under the old name. + + Only the missing relation is turned into ``OldTableNameMissingError``, so + any other failure in the test is reported as a failure. + """ + try: + return _saved_policy_row(connection, "user_policies", saved_policy_id) + except sa.exc.ProgrammingError as error: + if isinstance(error.orig, psycopg.errors.UndefinedTable): + raise OldTableNameMissingError from error + raise + + +@pytest.mark.xfail( + strict=True, + raises=OldTableNameMissingError, + reason=( + "Head has no view named user_policies yet. Autogeneration cannot " + "create one, and a hand-written view needs a human migration decision " + "(see the saved-policy table rename in the Alembic migrations skill); " + "until then the release running during a deploy cannot find the table." + ), +) +def test_previous_release_still_finds_user_policies() -> None: + """The release running during the deploy reads saved policies by the old name. + + A saved policy stored before the rename must still be found under + ``user_policies`` at head, as the previous release's statements name it. + """ + config = _config() + engine = create_engine(_disposable_url()) + parent = sa.MetaData() + saved = None + try: + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + parent.reflect(engine) + with engine.begin() as connection: + saved = _saved_policy_with_references(connection, parent) + command.upgrade(config, "head") + with engine.connect() as connection: + stored = _saved_policy_row( + connection, "user_policy_associations", saved["saved_policy_id"] + ) + with engine.connect() as connection: + found = _saved_policy_row_by_old_name(connection, saved["saved_policy_id"]) + assert found == stored + finally: + command.upgrade(config, "head") + if saved is not None: + _delete_saved_policy_rows(engine, parent, saved) + engine.dispose() diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index f031d04b3..1be9bc669 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -54,7 +54,8 @@ f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" "postgres?sslmode=require" ) -# The Stage 12A revision; the saved-policy rename follows it. +# The saved-policy rename and its parent, the Stage 12A revision. +SAVED_POLICY_RENAME_REVISION = "fa7fa1035444" SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" @@ -743,31 +744,25 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "validate_v2_head_schema(connection, target_metadata)" in env_source -def _revision_source(revision_id: str) -> str: - (path,) = (REPO / "migrations/v2/versions").glob(f"{revision_id}_*.py") - return path.read_text(encoding="utf-8") - - def test_saved_policy_rename_revision_renames_the_table_in_place() -> None: """The rename keeps every saved reform: no table is created or dropped. Autogeneration replaces a renamed table with a new empty one; the revision records its corrections, and every other operation is generated. """ - revision = _revision_source(V2_ALEMBIC_HEAD_REVISION) + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + rename = script.get_revision(SAVED_POLICY_RENAME_REVISION) + revision = Path(rename.path).read_text(encoding="utf-8") upgrade = revision[ revision.index("def upgrade()") : revision.index("def downgrade()") ] downgrade = revision[revision.index("def downgrade()") :] - assert V2_ALEMBIC_HEAD_REVISION != SAVED_POLICY_RENAME_PARENT_REVISION + assert rename.down_revision == SAVED_POLICY_RENAME_PARENT_REVISION assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision ) - assert ( - f'down_revision: Union[str, None] = "{SAVED_POLICY_RENAME_PARENT_REVISION}"' - ) in revision assert 'op.rename_table("user_policies", "user_policy_associations")' in upgrade assert 'op.rename_table("user_policy_associations", "user_policies")' in downgrade for body in (upgrade, downgrade): From c747f002ebd24c72df0112795e803800c9729aa4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:54:39 +0400 Subject: [PATCH 013/118] Describe the saved-policy rename in the Stage 12A document The merge table and the rule for Stage 10 mirror rows name the table by its current name. The rollback and schema compatibility sections no longer say head lets earlier releases run: until a view under the old name exists, a release from before the rename fails every statement on saved policies, from the schema upgrade until the new release takes traffic, and after a rollback until the rename is downgraded. They also say that downgrading the rename alone keeps every row, and that the Stage 12A compatibility test now maps the renamed table and no longer proves the parent release runs against head. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-12a-user-login.md | 72 ++++++++++++++++++-------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index 2831a4a60..cd4251781 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -126,7 +126,7 @@ unauthenticated: a standard user's row is never locked. | State | Outcome | What happens | | --- | --- | --- | | The identity has no user yet; the browser's unauthenticated user exists | `converted` | The `user_identities` row (issuer, subject, verified email) is inserted pointing at the browser's `users.id`; the user becomes `standard` with `primary_country = country_id`. The ID is unchanged, so every association stays owned. | -| The identity already belongs to standard user S; the unauthenticated user exists | `merged` | Every row of `user_policies`, `user_household_associations`, `user_simulation_associations`, and `user_report_associations` owned by the unauthenticated user moves to S with its ID, contents, and timestamps unchanged; then the unauthenticated user is deleted. | +| The identity already belongs to standard user S; the unauthenticated user exists | `merged` | Every row of `user_policy_associations`, `user_household_associations`, `user_simulation_associations`, and `user_report_associations` owned by the unauthenticated user moves to S with its ID, contents, and timestamps unchanged; then the unauthenticated user is deleted. | | No unauthenticated user with that ID | `none` | The identity's user is resolved, or provisioned, as on any request. | | The ID names a standard user (already converted, or anyone else's) | `none` | That user is never touched; the identity's user is resolved as on any request. | @@ -194,8 +194,8 @@ Concurrency on Postgres: `legacy_household_id`; with `legacy_household_id`, both are stored, and it must be the copy of that v1 household when one exists. - `user-simulations` and `user-reports` accept only legacy IDs in 12A. -- **Stage 10 mirror rows.** A `user_policies` row with a - `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The +- **Stage 10 mirror rows.** A saved-policy row (`user_policy_associations`) + with a `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The mirror projection now also records `legacy_policy_id`. - **Request bounds.** Every `/v2/me*` and `/v2/user-*` write rejects a body over 256 KiB with 413. FastAPI reads and parses the whole JSON body before @@ -331,6 +331,9 @@ Revision `fa7fa1035444` later renamed `user_policies` to indexes, matching the other association tables; the composite listing indexes keep their `ix_user_policies_*` names. The route `/v2/user-policies`, the `user_policy_id` columns and `legacy_user_policy_mappings` are unchanged. +The bullet points above describe revision `fbb7488ebe8a`, when the table was +still named `user_policies`. "Schema compatibility" below says what the +rename means for earlier Public API releases. ### Reshape safety @@ -353,15 +356,22 @@ and production and confirm both dormant tables are empty. ### Rollback -Roll forward whenever possible. The schema lets the previous Public API -revision run (see "Schema compatibility" below), but that revision serves -`/v2/user-policies` and `/v2/user-households` without authentication: any -caller can list a user's saved items by `user_id`, read, rename or delete one -by record ID, and create rows for any user. Rolling the application back -would therefore hand users' saved work to anyone. +Roll forward whenever possible. Since revision `fa7fa1035444` (see +"Schema compatibility" below), no Public API release from before that +revision can read or write saved policies at head: they name the table +`user_policies`, which no longer exists. Every statement they make on it +fails, including `/v2/user-policies`, user-report reads that join saved +policies, the sign-in merge and the Stage 10 saved-policy copy. Rolling the +application back to such a release therefore also needs revision +`fa7fa1035444` downgraded, which keeps every row (see below). The release +before Stage 12A also serves `/v2/user-policies` and `/v2/user-households` +without authentication: any caller can list a user's saved items by +`user_id`, read, rename or delete one by record ID, and create rows for any +user. Rolling the application back would therefore hand users' saved work to +anyone. - **Before any user-owned record exists, an application rollback is safe - without a database downgrade.** Signed-out saving needs no configuration, so + without a Stage 12A database downgrade.** Signed-out saving needs no configuration, so treat user-owned records as existing from the moment this revision serves traffic in an environment. - **Once any user-owned record exists, signed in or not, an application @@ -395,17 +405,35 @@ would therefore hand users' saved work to anyone. `/v2/user-policies` and `/v2/user-households` responses require a non-null `policy_id` / `household_id`, so any response that includes a legacy-only row (a v1 reference with no v2 copy) fails with 500. -- **A database downgrade is lossy.** It restores NOT NULL on `policy_id`, - `household_id`, `simulation_id`, `report_id`, `country`, and - `users.primary_country`, so it fails while legacy-only rows, any simulation - or report associations, or any unauthenticated user exist. Removing them - first deletes users' saved work. +- **Downgrading the rename alone is not lossy.** Downgrading revision + `fa7fa1035444` renames `user_policy_associations` back to `user_policies` + with every row, reference and grant, so releases from before the rename + can use saved policies again. +- **A database downgrade of Stage 12A is lossy.** Downgrading revision + `fbb7488ebe8a` restores NOT NULL on `policy_id`, `household_id`, + `simulation_id`, `report_id`, `country`, and `users.primary_country`, so it + fails while legacy-only rows, any simulation or report associations, or any + unauthenticated user exist. Removing them first deletes users' saved work. ### Schema compatibility -The revision is additive (expand-only) for everything the previous Public API -revision maps: it drops no table or column, and it adds no required column -without a server default to any table that revision writes. The only new +**Since revision `fa7fa1035444`, head does not serve earlier releases.** That +revision renames `user_policies` to `user_policy_associations` without the +temporary view under the old name that the rename was planned with: a view is +not autogenerated, so it waits for a human migration decision (see the +saved-policy table rename in `docs/engineering/skills/alembic-migrations.md`). +Until a view exists, any release from before the rename fails every statement +on saved policies. During a deploy this lasts from the schema upgrade until +the new release takes traffic; after an application rollback it lasts until a +release from the rename or later serves traffic again, or until revision +`fa7fa1035444` is downgraded. +`test_previous_release_still_finds_user_policies` records the gap as a strict +expected failure. + +The rest of this section describes revision `fbb7488ebe8a` alone. It was +additive (expand-only) for everything the previous Public API revision maps: +it drops no table or column, and it adds no required column without a server +default to any table that revision writes. The only new required columns are `country_id` on the simulation and report associations, which that revision maps but never inserts into, and `users.user_type`, whose server default makes every user the previous revision creates (its Stage 10 @@ -418,9 +446,11 @@ row with a NULL `primary_country` does not fail there. The lifecycle test `test_stage_12a_schema_serves_the_previous_revision` exercises this rather than only comparing columns. It reflects the schema at -the parent revision with SQLAlchemy and upgrades to head. Then, through the -reflected parent tables, it does the following for every table the parent -revision maps: +the parent revision with SQLAlchemy and upgrades to head. Since the rename, it +uses the parent's `user_policies` under its head name (`RENAMED_AT_HEAD`), so +it checks the parent's columns and constraints but no longer proves that the +parent release runs against head. Then, through the reflected parent tables, +it does the following for every table the parent revision maps: - inserts a row that supplies only the parent's required columns (plus the nullable values that the parent's own check constraints demand); From d31f13df213495e7e0f0eb76a1c595f8866c7172 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:49:39 +0400 Subject: [PATCH 014/118] Let the privilege reconciliation find, grant and verify views The reconciliation read only ordinary and partitioned tables when it checked which declared relations exist, so a view declared in the privilege manifest counted as missing and blocked every grant. It now reads views too. GRANT ... ON TABLE and has_table_privilege already work on views, and the granted-privilege query already read them. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/privilege_reconciliation.py | 9 +++-- .../test_v2_privilege_reconciliation.py | 36 ++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index d9d0cacec..bb5d673ab 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -276,7 +276,8 @@ def render_summary( FROM pg_class AS table_class JOIN pg_namespace AS table_namespace ON table_namespace.oid = table_class.relnamespace - WHERE table_namespace.nspname = :schema AND table_class.relkind IN ('r', 'p') + WHERE table_namespace.nspname = :schema + AND table_class.relkind IN ('r', 'p', 'v') """ ) @@ -312,7 +313,11 @@ def read_existing_roles(connection: Connection, roles: Collection[str]) -> set[s def read_public_tables(connection: Connection) -> set[str]: - """Return every table in the public schema.""" + """Return every table and view in the public schema. + + Views count because the manifest declares privileges on temporary views as + it does on tables, and ``GRANT ... ON TABLE`` grants on a view too. + """ return set(connection.execute(_PUBLIC_TABLES, {"schema": SCHEMA}).scalars()) diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index f720c3fb9..38db72d94 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -23,7 +23,7 @@ check_declared_privileges, run, ) -from policyengine_api.data.v2.privileges import READ, TablePrivilege +from policyengine_api.data.v2.privileges import READ, READ_WRITE, TablePrivilege from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL if TYPE_CHECKING: @@ -62,6 +62,21 @@ def create_role() -> str: connection.execute(text(f'DROP ROLE "{role}"')) +@pytest.fixture +def temporary_view(superuser_engine) -> Iterator[str]: + """Create a view over ``regions`` for one test, then drop it.""" + view = f"policyengine_privilege_test_view_{uuid4().hex[:12]}" + with superuser_engine.begin() as connection: + connection.execute( + text(f'CREATE VIEW public."{view}" AS SELECT * FROM public.regions') + ) + try: + yield view + finally: + with superuser_engine.begin() as connection: + connection.execute(text(f'DROP VIEW public."{view}"')) + + def _has_privilege(engine: Engine, role: str, table: str, privilege: str) -> bool: with engine.connect() as connection: return bool( @@ -137,3 +152,22 @@ def test_a_missing_role_blocks_every_grant_and_fails_verification( PrivilegeEntry(role, "regions", "SELECT"), ] ) + + +def test_apply_grants_and_verify_checks_privileges_on_a_declared_view( + superuser_engine, temporary_role, temporary_view +) -> None: + """A view declared like a table is found, granted and verified.""" + role = temporary_role() + declared = {role: {temporary_view: READ_WRITE}} + with superuser_engine.begin() as connection: + plan, _defaults = apply_declared_privileges(connection, declared) + missing = check_declared_privileges(connection, declared) + + assert plan.missing_tables == () + assert {entry.privilege for entry in plan.grants} == { + privilege.value for privilege in READ_WRITE + } + assert missing == [] + for privilege in READ_WRITE: + assert _has_privilege(superuser_engine, role, temporary_view, privilege.value) From 035af26e0a96790fd3fcf305a36240cef399a5e3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:00:44 +0400 Subject: [PATCH 015/118] Keep a temporary user_policies view for the release before the rename Revision fa7fa1035444 now creates a view named user_policies with every column of user_policy_associations after the rename, and drops it before the downgrade renames the table back. These two statements are the only hand-written SQL in the v2 chain: a maintainer approved them on 2026-10-22 as an exception to the generated-only rule, and a later revision drops the view. The revision comments and the Alembic skill say so, and the revision test allows exactly these two statements. A single-table view is automatically updatable, so the Public API release still serving during the deploy, or after a rollback to it, keeps reading and writing saved policies through it. The privilege manifest declares the view with its table's privileges, so the reconciliation grants the runtime role SELECT, INSERT, UPDATE and DELETE on it. Autogeneration does not reflect views, so alembic check and the head schema validation stay clean. Tests: the lifecycle test checks the view's columns after the upgrade and its absence after the downgrade, runs the parent revision's inserts, reads, updates and deletes through it, and reads a saved policy stored before the rename under the old name. A new integration test runs the saved-policy routes' services, the sign-in merge, report creation and the Stage 10 saved-policy copy as the runtime role with every statement naming user_policies. The Stage 12A document and the changelog describe the view. Co-Authored-By: Claude Opus 5.5 --- ...lan-1f-122-rename-user-policies.changed.md | 2 +- docs/engineering/skills/alembic-migrations.md | 30 +- docs/migration/stage-12a-user-login.md | 54 ++- ...44_rename_user_policies_to_user_policy_.py | 14 + policyengine_api/data/v2/privileges.py | 34 +- .../integration/test_alembic_v2_lifecycle.py | 138 ++++--- .../integration/test_v2_saved_policy_view.py | 382 ++++++++++++++++++ tests/unit/v2/test_alembic_v2.py | 45 ++- tests/unit/v2/test_privilege_manifest.py | 24 +- 9 files changed, 630 insertions(+), 93 deletions(-) create mode 100644 tests/integration/test_v2_saved_policy_view.py diff --git a/changelog.d/v2plan-1f-122-rename-user-policies.changed.md b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md index 1141ffd75..9c9a77ad8 100644 --- a/changelog.d/v2plan-1f-122-rename-user-policies.changed.md +++ b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md @@ -1 +1 @@ -The v2 table user_policies is renamed to user_policy_associations, matching the other user-association tables; its constraints and convention-named indexes take the new name. Saved policies, their v1 mappings and the reports that name them are kept. The route /v2/user-policies, the user_policy_id columns, legacy_user_policy_mappings and the v1 Cloud SQL table user_policies are unchanged. +The v2 table user_policies is renamed to user_policy_associations, matching the other user-association tables; its constraints and convention-named indexes take the new name. Saved policies, their v1 mappings and the reports that name them are kept. A temporary view named user_policies, with the runtime role's privileges, lets the release from before the rename keep reading and writing saved policies during the deploy and after a rollback; a later change removes it. The privilege reconciliation now finds, grants and verifies privileges on views. The route /v2/user-policies, the user_policy_id columns, legacy_user_policy_mappings and the v1 Cloud SQL table user_policies are unchanged. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 3b7a13f36..a9f6ce77b 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -315,9 +315,27 @@ report that names it, and checks that the rows, the references, the cascades, a table grant and every constraint and index name survive the upgrade and the downgrade. -The plan also asks for a temporary view named `user_policies` over the renamed -table, so the release still running during the deploy keeps working. A view -is not autogenerated, and adding one would be a hand-written schema operation, -so the revision leaves it out pending a human migration decision. -`test_previous_release_still_finds_user_policies` records the gap as a strict -expected failure. +The revision also holds the only approved hand-written exception in the v2 +chain, approved by a maintainer on 2026-10-22: after the rename it creates a +view named `user_policies` with every column of `user_policy_associations`, +and the downgrade drops the view before renaming the table back. The Public +API release still serving during a deploy, or after a rollback to it, names +the table `user_policies`; a single-table view is automatically updatable, so +that release keeps reading and writing saved policies through it. The +privilege manifest declares the view with the table's privileges +(`TEMPORARY_VIEW_TABLES` in `policyengine_api/data/v2/privileges.py`), and the +privilege reconciliation grants and verifies privileges on views as on tables. +Autogeneration does not reflect views, so `alembic check` and the head schema +validation ignore it. A later revision drops the view once no deployable +release names `user_policies`; until then, a revision cannot drop or change +the type of a `user_policy_associations` column the view shows without +dropping the view first. + +Tests: `test_alembic_v2.py` allows exactly these two statements and no other +`op.execute` in any v2 revision. The lifecycle test checks the view's columns +after the upgrade and its absence after the downgrade, and runs the previous +revision's inserts, reads, updates and deletes through it. +`test_v2_saved_policy_view.py` runs today's saved-policy services, the +sign-in merge, report creation and the Stage 10 saved-policy copy as the +runtime role with the table named `user_policies`, so every statement goes +through the view. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index cd4251781..ed952e189 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -357,14 +357,11 @@ and production and confirm both dormant tables are empty. ### Rollback Roll forward whenever possible. Since revision `fa7fa1035444` (see -"Schema compatibility" below), no Public API release from before that -revision can read or write saved policies at head: they name the table -`user_policies`, which no longer exists. Every statement they make on it -fails, including `/v2/user-policies`, user-report reads that join saved -policies, the sign-in merge and the Stage 10 saved-policy copy. Rolling the -application back to such a release therefore also needs revision -`fa7fa1035444` downgraded, which keeps every row (see below). The release -before Stage 12A also serves `/v2/user-policies` and `/v2/user-households` +"Schema compatibility" below), releases from before that revision use saved +policies through the temporary view `user_policies` until a later revision +drops it; after that, rolling the application back to such a release also +needs revision `fa7fa1035444` downgraded, which keeps every row (see below). +The release before Stage 12A also serves `/v2/user-policies` and `/v2/user-households` without authentication: any caller can list a user's saved items by `user_id`, read, rename or delete one by record ID, and create rows for any user. Rolling the application back would therefore hand users' saved work to @@ -406,9 +403,9 @@ anyone. `policy_id` / `household_id`, so any response that includes a legacy-only row (a v1 reference with no v2 copy) fails with 500. - **Downgrading the rename alone is not lossy.** Downgrading revision - `fa7fa1035444` renames `user_policy_associations` back to `user_policies` - with every row, reference and grant, so releases from before the rename - can use saved policies again. + `fa7fa1035444` drops the temporary view and renames + `user_policy_associations` back to `user_policies` with every row, + reference and table grant. - **A database downgrade of Stage 12A is lossy.** Downgrading revision `fbb7488ebe8a` restores NOT NULL on `policy_id`, `household_id`, `simulation_id`, `report_id`, `country`, and `users.primary_country`, so it @@ -417,18 +414,19 @@ anyone. ### Schema compatibility -**Since revision `fa7fa1035444`, head does not serve earlier releases.** That -revision renames `user_policies` to `user_policy_associations` without the -temporary view under the old name that the rename was planned with: a view is -not autogenerated, so it waits for a human migration decision (see the -saved-policy table rename in `docs/engineering/skills/alembic-migrations.md`). -Until a view exists, any release from before the rename fails every statement -on saved policies. During a deploy this lasts from the schema upgrade until -the new release takes traffic; after an application rollback it lasts until a -release from the rename or later serves traffic again, or until revision -`fa7fa1035444` is downgraded. -`test_previous_release_still_finds_user_policies` records the gap as a strict -expected failure. +**Since revision `fa7fa1035444`, earlier releases use saved policies through a +temporary view.** That revision renames `user_policies` to +`user_policy_associations` and creates a view named `user_policies` with every +column of the renamed table, the approved hand-written exception described in +`docs/engineering/skills/alembic-migrations.md` ("API v2 saved-policy table +rename"). A single-table view is automatically updatable, and the runtime role +holds on it what it holds on the table, so a release from before the rename +keeps reading and writing saved policies, during a deploy and after a +rollback: its `/v2/user-policies` routes, report creation that checks the saved +policies a report names, the sign-in merge, and the Stage 10 saved-policy +copy. `test_v2_saved_policy_view.py` runs those operations as the runtime role +through the view. Once a later revision drops the view, such a release fails +every statement on saved policies. The rest of this section describes revision `fbb7488ebe8a` alone. It was additive (expand-only) for everything the previous Public API revision maps: @@ -446,11 +444,11 @@ row with a NULL `primary_country` does not fail there. The lifecycle test `test_stage_12a_schema_serves_the_previous_revision` exercises this rather than only comparing columns. It reflects the schema at -the parent revision with SQLAlchemy and upgrades to head. Since the rename, it -uses the parent's `user_policies` under its head name (`RENAMED_AT_HEAD`), so -it checks the parent's columns and constraints but no longer proves that the -parent release runs against head. Then, through the reflected parent tables, -it does the following for every table the parent revision maps: +the parent revision with SQLAlchemy and upgrades to head. Since the rename, +the parent's `user_policies` is head's temporary view of that name, so the +parent's statements on saved policies run through the view. Then, through the +reflected parent tables, it does the following for every table the parent +revision maps: - inserts a row that supplies only the parent's required columns (plus the nullable values that the parent's own check constraints demand); diff --git a/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py index d2aa619c5..d9dea42da 100644 --- a/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py +++ b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py @@ -149,11 +149,25 @@ def upgrade() -> None: ["id"], ondelete="SET NULL", ) + # Approved hand-written exception: a temporary view under the table's + # earlier name, so the Public API release still serving during the deploy, + # or after a rollback to it, keeps reading and writing saved policies. A + # maintainer approved this hand-written statement on 2026-10-22, and a later + # revision drops the view. A single-table view is automatically updatable; + # the privilege manifest grants on it what it grants on the table. + op.execute( + "CREATE VIEW user_policies AS SELECT id, created_at, updated_at, user_id, " + "policy_id, country_id, name, description, legacy_policy_id, baseline, " + "provenance FROM user_policy_associations" + ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + # Approved hand-written exception: the upgrade's temporary view is dropped + # first, so the table can take its earlier name back. + op.execute("DROP VIEW user_policies") op.drop_constraint( op.f( "fk_user_report_associations_baseline_user_policy_id_user_policy_associations" diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index dc965e36e..5014230f0 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -10,7 +10,9 @@ ``api.v2.runtime-privileges-tested-as-runtime-role``). Every table in the v2 SQLModel metadata, plus ``alembic_version``, has an entry -for every managed role, so "no access" is a decision, not an omission. +for every managed role, so "no access" is a decision, not an omission. A +temporary view under a renamed table's earlier name (``TEMPORARY_VIEW_TABLES``) +is declared with its table's privileges. Roles and what decides their privileges: @@ -145,6 +147,17 @@ def _access( } +# Temporary views under a renamed table's earlier name, each mapped to the +# table it shows. The Public API release that still serves during a deploy, or +# after a rollback to it, reads and writes the table through the view, so every +# role holds on the view what it holds on the table. A later revision drops +# each view, and its entry here goes with it. +TEMPORARY_VIEW_TABLES: Final[dict[str, str]] = { + # Revision fa7fa1035444 renamed user_policies to user_policy_associations. + "user_policies": "user_policy_associations", +} + + class InvalidDeploymentEnvironmentError(ValueError): """Raised for a deployment environment that can't name a Stage 12 role.""" @@ -167,12 +180,23 @@ def stage12_role(deployment_environment: str) -> str: def declared_privileges( deployment_environment: str, ) -> dict[str, dict[str, frozenset[TablePrivilege]]]: - """Return each managed role's declared privileges, keyed by role then table.""" + """Return each managed role's declared privileges, keyed by role then table. + + Each temporary view is declared with its table's privileges. + """ stage12 = stage12_role(deployment_environment) + access_by_relation = TABLE_ACCESS | { + view: TABLE_ACCESS[table] for view, table in TEMPORARY_VIEW_TABLES.items() + } return { - RUNTIME_ROLE: {table: access.runtime for table, access in TABLE_ACCESS.items()}, + RUNTIME_ROLE: { + relation: access.runtime for relation, access in access_by_relation.items() + }, CATALOG_WRITER_ROLE: { - table: access.catalog_writer for table, access in TABLE_ACCESS.items() + relation: access.catalog_writer + for relation, access in access_by_relation.items() + }, + stage12: { + relation: access.stage12 for relation, access in access_by_relation.items() }, - stage12: {table: access.stage12 for table, access in TABLE_ACCESS.items()}, } diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 6a2b4b129..5ef7c6169 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -6,7 +6,6 @@ from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 -import psycopg import pytest import sqlalchemy as sa from alembic import command @@ -22,6 +21,7 @@ V2_ALEMBIC_HEAD_REVISION, V2MigrationTargetError, load_v2_alembic_settings, + validate_v2_head_schema, ) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL @@ -410,11 +410,6 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_simulation_associations", "user_report_associations", ) -# Tables head renamed, keyed by their name at the parent revision. The -# previous release's statements name the old table, which head does not have -# until a view under that name exists (test_previous_release_still_finds_ -# user_policies), so its column usage runs against the renamed table here. -RENAMED_AT_HEAD = {"user_policies": "user_policy_associations"} COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) # Nullable references that a parent-revision check constraint requires. REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) @@ -594,7 +589,7 @@ def _assert_user_deletion_cascades( assert result.rowcount == 1 for table_name in (*USER_ASSOCIATION_TABLES, "user_identities"): - table = head.tables[RENAMED_AT_HEAD.get(table_name, table_name)] + table = head.tables[table_name] remaining = connection.execute( sa.select(sa.func.count()) .select_from(table) @@ -611,8 +606,9 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: requires), read the row back with every mapped column, rewrite those columns, delete it, and delete a user who owns Stage 12A rows so the cascades run. Head may not give any table the parent revision writes a - new required column without a server default. A table head renamed is - used under its new name (``RENAMED_AT_HEAD``). + new required column without a server default. Since the saved-policy + rename, the parent's ``user_policies`` is head's temporary view of that + name, so the parent's statements on it run through the view. """ database_url = _disposable_url() config = _config() @@ -622,11 +618,9 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) parent = sa.MetaData() parent.reflect(engine) - for parent_name, head_name in RENAMED_AT_HEAD.items(): - parent.tables[parent_name].name = head_name command.upgrade(config, "head") head = sa.MetaData() - head.reflect(engine) + head.reflect(engine, views=True) parent_tables = [ table for table in parent.sorted_tables if table.name != "alembic_version" ] @@ -1225,6 +1219,31 @@ def _probe_privileges(connection, table_name: str) -> dict[str, bool]: CLEARED_REFERENCES = (None, None, None) +def _view_names(engine) -> set[str]: + return set(inspect(engine).get_view_names(schema="public")) + + +def _column_names(engine, relation_name: str) -> list[str]: + return [ + column["name"] + for column in inspect(engine).get_columns(relation_name, schema="public") + ] + + +def _assert_view_under_old_name_at_head(engine, config) -> None: + """Head keeps a view named ``user_policies`` with the table's columns. + + Neither the head schema validation nor ``alembic check`` reports it. + """ + assert _view_names(engine) == {"user_policies"} + assert _column_names(engine, "user_policies") == _column_names( + engine, "user_policy_associations" + ) + with engine.connect() as connection: + validate_v2_head_schema(connection, V2_METADATA) + command.check(config) + + def _drop_rename_privilege_probe(engine) -> None: with engine.begin() as connection: connection.execute( @@ -1242,8 +1261,10 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: Saved policies, their v1 mappings and the reports that name them survive both directions, the references and a table grant follow the table, and - the constraints and indexes named after the table take its name. The - downgrade restores every name the parent revision had. + the constraints and indexes named after the table take its name. Head has + a view named ``user_policies`` with the table's columns, which neither the + schema validation nor ``alembic check`` reports. The downgrade drops the + view and restores every name the parent revision had. """ config = _config() engine = create_engine(_disposable_url()) @@ -1272,6 +1293,7 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: table_names = set(inspect(engine).get_table_names(schema="public")) assert "user_policies" not in table_names + _assert_view_under_old_name_at_head(engine, config) with engine.connect() as connection: after = _saved_policy_row( connection, "user_policy_associations", saved_policy_id @@ -1296,6 +1318,8 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: table_names = set(inspect(engine).get_table_names(schema="public")) assert "user_policy_associations" not in table_names + assert "user_policies" in table_names + assert _view_names(engine) == set() with engine.connect() as connection: restored = _saved_policy_row(connection, "user_policies", saved_policy_id) restored_references = _saved_policy_references(connection, saved) @@ -1316,39 +1340,13 @@ def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: engine.dispose() -class OldTableNameMissingError(Exception): - """Raised when head has no relation named ``user_policies`` to read from.""" - - -def _saved_policy_row_by_old_name(connection, saved_policy_id: UUID) -> dict: - """Read a saved policy as the previous release does, under the old name. - - Only the missing relation is turned into ``OldTableNameMissingError``, so - any other failure in the test is reported as a failure. - """ - try: - return _saved_policy_row(connection, "user_policies", saved_policy_id) - except sa.exc.ProgrammingError as error: - if isinstance(error.orig, psycopg.errors.UndefinedTable): - raise OldTableNameMissingError from error - raise - - -@pytest.mark.xfail( - strict=True, - raises=OldTableNameMissingError, - reason=( - "Head has no view named user_policies yet. Autogeneration cannot " - "create one, and a hand-written view needs a human migration decision " - "(see the saved-policy table rename in the Alembic migrations skill); " - "until then the release running during a deploy cannot find the table." - ), -) def test_previous_release_still_finds_user_policies() -> None: - """The release running during the deploy reads saved policies by the old name. + """The release running during the deploy uses saved policies by the old name. - A saved policy stored before the rename must still be found under - ``user_policies`` at head, as the previous release's statements name it. + A saved policy stored before the rename is found under ``user_policies`` + at head, and a row written, changed and deleted through the view is + written, changed and deleted in ``user_policy_associations``. The runtime + role's own use of the view is in ``test_v2_saved_policy_view``. """ config = _config() engine = create_engine(_disposable_url()) @@ -1360,13 +1358,57 @@ def test_previous_release_still_finds_user_policies() -> None: with engine.begin() as connection: saved = _saved_policy_with_references(connection, parent) command.upgrade(config, "head") + saved_policy_id = saved["saved_policy_id"] with engine.connect() as connection: stored = _saved_policy_row( - connection, "user_policy_associations", saved["saved_policy_id"] + connection, "user_policy_associations", saved_policy_id ) - with engine.connect() as connection: - found = _saved_policy_row_by_old_name(connection, saved["saved_policy_id"]) + found = _saved_policy_row(connection, "user_policies", saved_policy_id) assert found == stored + + written_id = uuid4() + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.execute( + text( + "INSERT INTO user_policies " + "(id, user_id, policy_id, country_id, name) " + "VALUES (:id, :user_id, :policy_id, :country_id, 'Written')" + ), + { + "id": written_id, + "user_id": stored["user_id"], + "policy_id": stored["policy_id"], + "country_id": stored["country_id"], + }, + ) + written = _saved_policy_row( + connection, "user_policy_associations", written_id + ) + connection.execute( + text("UPDATE user_policies SET name = 'Changed' WHERE id = :id"), + {"id": written_id}, + ) + changed_name = _saved_policy_row( + connection, "user_policy_associations", written_id + )["name"] + connection.execute( + text("DELETE FROM user_policies WHERE id = :id"), + {"id": written_id}, + ) + remaining = connection.execute( + text( + "SELECT count(*) FROM user_policy_associations WHERE id = :id" + ), + {"id": written_id}, + ).scalar_one() + finally: + transaction.rollback() + assert written["name"] == "Written" + assert written["created_at"] is not None + assert changed_name == "Changed" + assert remaining == 0 finally: command.upgrade(config, "head") if saved is not None: diff --git a/tests/integration/test_v2_saved_policy_view.py b/tests/integration/test_v2_saved_policy_view.py new file mode 100644 index 000000000..7ffb666d5 --- /dev/null +++ b/tests/integration/test_v2_saved_policy_view.py @@ -0,0 +1,382 @@ +"""The release before the saved-policy rename keeps working through the view. + +Revision ``fa7fa1035444`` renames ``user_policies`` to +``user_policy_associations`` and keeps a temporary view under the old name. The +Public API release that still serves during the deploy, or after a rollback to +it, runs the same statements as today's services with the old table name. +These tests run today's services as the runtime role, rewrite each statement's +table name to ``user_policies`` before it is sent, as that release would +compile it, and check the result in the renamed table. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, event, select, text +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserIdentity, +) +from policyengine_api.services.v2.identity.types import ( + UnauthenticatedPrincipal, + UserPrincipal, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_accounts.database_session import ( + AccountDatabaseSession, +) +from policyengine_api.services.v2.user_accounts.services import V2AccountService +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import ( + V2UserPolicyService, + mirror_legacy_user_policy_in_session, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreateFields, + UserPolicyUpdateInput, +) +from policyengine_api.services.v2.user_reports.database_session import ( + UserReportDatabaseSession, +) +from policyengine_api.services.v2.user_reports.services import V2UserReportService +from policyengine_api.services.v2.user_reports.types import UserReportCreateFields +from tests.unit.v2.user_service_support import default_legacy_references + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + +TABLE_BEFORE_THE_RENAME = "user_policies" +TABLE_AFTER_THE_RENAME = "user_policy_associations" +# The renamed table where a statement names it as a table, not inside a label +# such as ``user_policy_associations_id``. +RENAMED_TABLE_REFERENCE = re.compile(rf"\b{TABLE_AFTER_THE_RENAME}\b") +MIRRORED_LEGACY_USER_ID = "auth0|saved-policy-view" + + +@pytest.fixture +def cleanup_engine(disposable_v2_database_url) -> Iterator[Engine]: + engine = create_engine(disposable_v2_database_url) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture +def previous_release_statements() -> list[str]: + """Every statement on saved policies, as sent with the earlier table name.""" + return [] + + +@pytest.fixture +def sessions( + managed_v2_roles, v2_engine_as_role, cleanup_engine, previous_release_statements +) -> Iterator[sessionmaker[Session]]: + """Open sessions as the runtime role that name saved policies as before. + + Each statement that names ``user_policy_associations`` is sent naming + ``user_policies`` instead. Results are matched to columns by position, so + the rewritten labels do not matter. + """ + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime) + + @event.listens_for(runtime_engine, "before_cursor_execute", retval=True) + def name_saved_policies_as_before_the_rename( + _connection, _cursor, statement, parameters, _context, _executemany + ): + if RENAMED_TABLE_REFERENCE.search(statement): + statement = RENAMED_TABLE_REFERENCE.sub(TABLE_BEFORE_THE_RENAME, statement) + previous_release_statements.append(statement) + return statement, parameters + + factory = sessionmaker(runtime_engine, class_=Session, expire_on_commit=False) + issuer = f"https://saved-policy-view-{uuid4().hex[:12]}.example.test/" + factory.kw["info"] = {"issuer": issuer, "browser_user_ids": []} + try: + yield factory + finally: + with cleanup_engine.begin() as connection: + owned = select(UserIdentity.user_id).where(UserIdentity.issuer == issuer) + user_ids = [ + *connection.execute(owned).scalars(), + *factory.kw["info"]["browser_user_ids"], + ] + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) + runtime_engine.dispose() + + +def _browser(sessions) -> UnauthenticatedPrincipal: + user_id = uuid4() + sessions.kw["info"]["browser_user_ids"].append(user_id) + return UnauthenticatedPrincipal(user_id=user_id) + + +def _saved_policy_row(cleanup_engine, association_id: UUID) -> dict | None: + with cleanup_engine.connect() as connection: + row = ( + connection.execute( + text(f"SELECT * FROM {TABLE_AFTER_THE_RENAME} WHERE id = :id"), + {"id": association_id}, + ) + .mappings() + .one_or_none() + ) + return None if row is None else dict(row) + + +def _statement_kinds(statements: list[str]) -> set[str]: + """Return the kind of each statement: its first word.""" + return {statement.split(maxsplit=1)[0].upper() for statement in statements} + + +def test_runtime_role_holds_its_saved_policy_privileges_on_the_view( + sessions, managed_v2_roles +) -> None: + with sessions() as session: + held = { + privilege: session.execute( + text("SELECT has_table_privilege(:role, :view, :privilege)"), + { + "role": managed_v2_roles.runtime, + "view": f"public.{TABLE_BEFORE_THE_RENAME}", + "privilege": privilege, + }, + ).scalar_one() + for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE") + } + + assert held == dict.fromkeys(("SELECT", "INSERT", "UPDATE", "DELETE"), True) + + +def test_previous_release_saves_reads_renames_and_deletes_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + browser = _browser(sessions) + policies = V2UserPolicyService( + UserPolicyDatabaseSession(sessions), default_legacy_references() + ) + + created = policies.create_user_policy( + principal=browser, + association_input=UserPolicyCreateFields( + country_id="us", legacy_policy_id=1, name="Saved" + ), + ) + stored_after_create = _saved_policy_row(cleanup_engine, created.id) + read = policies.get_user_policy( + principal=browser, country_id="us", association_id=created.id + ) + page = policies.list_user_policies(principal=browser, country_id="us") + renamed = policies.patch_user_policy( + principal=browser, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(name="Renamed"), + ) + stored_after_rename = _saved_policy_row(cleanup_engine, created.id) + policies.delete_user_policy( + principal=browser, country_id="us", association_id=created.id + ) + + assert stored_after_create is not None + assert stored_after_create["user_id"] == browser.user_id + assert stored_after_create["created_at"] == created.created_at + assert read.id == created.id + assert [association.id for association in page.items] == [created.id] + assert renamed.name == "Renamed" + assert stored_after_rename is not None + assert stored_after_rename["name"] == "Renamed" + assert _saved_policy_row(cleanup_engine, created.id) is None + assert _statement_kinds(previous_release_statements) == { + "INSERT", + "SELECT", + "UPDATE", + "DELETE", + } + + +def test_previous_release_merges_saved_policies_and_checks_reports_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + """Sign-in moves a browser's saved policies, and reports name them.""" + browser = _browser(sessions) + signed_in = UserPrincipal( + issuer=sessions.kw["info"]["issuer"], subject="google-oauth2|view-merge" + ) + references = default_legacy_references() + policies = V2UserPolicyService(UserPolicyDatabaseSession(sessions), references) + reports = V2UserReportService(UserReportDatabaseSession(sessions), references) + accounts = V2AccountService(AccountDatabaseSession(sessions)) + + account_id = accounts.get_account(principal=signed_in, country_id="us").id + saved = policies.create_user_policy( + principal=browser, + association_input=UserPolicyCreateFields(country_id="us", legacy_policy_id=1), + ) + report = reports.create_user_report( + principal=browser, + association_input=UserReportCreateFields( + country_id="us", + legacy_report_id=615, + user_policy_id=saved.id, + baseline_user_policy_id=saved.id, + ), + ) + accounts.upgrade_account( + principal=signed_in, + unauthenticated_user_id=browser.user_id, + country_id="us", + ) + exported = accounts.export_account(principal=signed_in, country_id="us") + + stored = _saved_policy_row(cleanup_engine, saved.id) + assert stored is not None + assert stored["user_id"] == account_id + assert report.user_policy_id == saved.id + assert [policy.id for policy in exported.policies] == [saved.id] + # The sign-in merge moves every saved policy of the browser's user. + assert any( + re.match( + rf"UPDATE {TABLE_BEFORE_THE_RENAME} SET .*\buser_id=.* WHERE " + rf"{TABLE_BEFORE_THE_RENAME}\.user_id = ", + statement, + ) + for statement in previous_release_statements + ) + + +def _seed_catalog(cleanup_engine) -> tuple[UUID, str]: + parameter_name = f"gov.saved_policy_view.rate_{uuid4().hex[:8]}" + with Session(cleanup_engine) as session, session.begin(): + # The policy copy selects the catalog by this model name. + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + session.add(Parameter(name=parameter_name, tax_benefit_model_version=version)) + session.flush() + return model.id, parameter_name + + +def _delete_mirrored_rows(cleanup_engine, model_id: UUID) -> None: + with cleanup_engine.begin() as connection: + user_ids = select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id == MIRRORED_LEGACY_USER_ID + ) + mirrored_user_ids = connection.execute(user_ids).scalars().all() + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == MIRRORED_LEGACY_USER_ID + ) + ) + # Deleting the user deletes its saved policies and their v1 mappings. + connection.execute(delete(User).where(User.id.in_(mirrored_user_ids))) + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + for model in (LegacyPolicyMapping, ParameterValue): + connection.execute(delete(model).where(model.policy_id.in_(policy_ids))) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def _legacy_saved_policy(reform_label: str) -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot.model_validate( + { + "country_id": "us", + "legacy_user_policy_id": 901, + "reform_id": 801, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": MIRRORED_LEGACY_USER_ID, + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 1, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + ) + + +def test_previous_release_copies_v1_saved_policies_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + """The Stage 10 saved-policy copy inserts and then updates through the view.""" + model_id, parameter_name = _seed_catalog(cleanup_engine) + reform = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=801, + label="Ignored legacy label", + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash=f"view-{uuid4().hex}", + ) + try: + with sessions.begin() as session: + first = mirror_legacy_user_policy_in_session( + session, _legacy_saved_policy("Copied"), reform, source_revision=1 + ) + with sessions.begin() as session: + second = mirror_legacy_user_policy_in_session( + session, + _legacy_saved_policy("Copied again"), + reform, + source_revision=2, + changed_fields=frozenset({"reform_label"}), + ) + stored = _saved_policy_row(cleanup_engine, first.association_id) + finally: + _delete_mirrored_rows(cleanup_engine, model_id) + + assert first.association_created is True + assert second.association_id == first.association_id + assert second.association_created is False + assert stored is not None + assert stored["name"] == "Copied again" + assert {"INSERT", "SELECT", "UPDATE"} <= _statement_kinds( + previous_release_statements + ) diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 1be9bc669..94d79f577 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -57,6 +57,15 @@ # The saved-policy rename and its parent, the Stage 12A revision. SAVED_POLICY_RENAME_REVISION = "fa7fa1035444" SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" +# The only hand-written statements in any v2 revision: the approved temporary +# view under the saved-policy table's earlier name, and its removal on +# downgrade. +SAVED_POLICY_VIEW_CREATION = ( + "CREATE VIEW user_policies AS SELECT id, created_at, updated_at, user_id, " + "policy_id, country_id, name, description, legacy_policy_id, baseline, " + "provenance FROM user_policy_associations" +) +SAVED_POLICY_VIEW_REMOVAL = "DROP VIEW user_policies" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" f"{DISPOSABLE_DATABASE_NAME}" @@ -744,11 +753,21 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "validate_v2_head_schema(connection, target_metadata)" in env_source +def _executed_statements(body: str) -> list[str]: + """Return the SQL each ``op.execute`` call in a revision body runs.""" + return [ + "".join(re.findall(r'"([^"]*)"', call)) + for call in re.findall(r"op\.execute\((.*?)\)\n", body, flags=re.DOTALL) + ] + + def test_saved_policy_rename_revision_renames_the_table_in_place() -> None: """The rename keeps every saved reform: no table is created or dropped. Autogeneration replaces a renamed table with a new empty one; the revision - records its corrections, and every other operation is generated. + records its corrections, and every other operation is generated except the + approved temporary view under the old name, created after the rename and + dropped before the downgrade renames the table back. """ script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) rename = script.get_revision(SAVED_POLICY_RENAME_REVISION) @@ -757,18 +776,36 @@ def test_saved_policy_rename_revision_renames_the_table_in_place() -> None: revision.index("def upgrade()") : revision.index("def downgrade()") ] downgrade = revision[revision.index("def downgrade()") :] + rename_up = 'op.rename_table("user_policies", "user_policy_associations")' + rename_down = 'op.rename_table("user_policy_associations", "user_policies")' assert rename.down_revision == SAVED_POLICY_RENAME_PARENT_REVISION assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision ) - assert 'op.rename_table("user_policies", "user_policy_associations")' in upgrade - assert 'op.rename_table("user_policy_associations", "user_policies")' in downgrade + assert rename_up in upgrade + assert rename_down in downgrade for body in (upgrade, downgrade): assert "op.create_table(" not in body assert "op.drop_table(" not in body - assert "op.execute(" not in body assert "op.bulk_insert(" not in body + assert body.count("Approved hand-written exception:") == 1 + assert _executed_statements(upgrade) == [SAVED_POLICY_VIEW_CREATION] + assert _executed_statements(downgrade) == [SAVED_POLICY_VIEW_REMOVAL] + assert upgrade.index(rename_up) < upgrade.index("op.execute(") + assert downgrade.index("op.execute(") < downgrade.index(rename_down) assert upgrade.count("Post-generation correction:") == 1 assert downgrade.count("Post-generation correction:") == 1 + + +def test_no_other_v2_revision_runs_hand_written_sql() -> None: + """Only the saved-policy rename's approved view uses ``op.execute``.""" + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + revisions_with_sql = { + revision.revision + for revision in script.walk_revisions() + if "op.execute(" in Path(revision.path).read_text(encoding="utf-8") + } + + assert revisions_with_sql == {SAVED_POLICY_RENAME_REVISION} diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 1168d4888..380d9a81e 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -22,6 +22,7 @@ READ_WRITE, RUNTIME_ROLE, TABLE_ACCESS, + TEMPORARY_VIEW_TABLES, InvalidDeploymentEnvironmentError, TablePrivilege, declared_privileges, @@ -72,7 +73,28 @@ def test_every_v2_table_has_an_entry_for_every_managed_role() -> None: assert set(TABLE_ACCESS) == metadata_tables | {"alembic_version"} for role, privileges_by_table in declared_privileges(TEST_ENVIRONMENT).items(): - assert set(privileges_by_table) == set(TABLE_ACCESS), role + assert set(privileges_by_table) == set(TABLE_ACCESS) | set( + TEMPORARY_VIEW_TABLES + ), role + + +def test_each_temporary_view_grants_every_role_what_its_table_grants() -> None: + """A view under a renamed table's earlier name serves the earlier release. + + That release reads and writes the table through the view, so each role + needs on the view exactly what it holds on the table. + """ + metadata_tables = {table.name for table in V2_METADATA.tables.values()} + + assert TEMPORARY_VIEW_TABLES == {"user_policies": "user_policy_associations"} + for view, table in TEMPORARY_VIEW_TABLES.items(): + assert view not in metadata_tables + assert table in metadata_tables + for role, privileges_by_table in declared_privileges(TEST_ENVIRONMENT).items(): + assert privileges_by_table[view] == privileges_by_table[table], role + assert declared_privileges(TEST_ENVIRONMENT)[RUNTIME_ROLE]["user_policies"] == ( + READ_WRITE + ) def test_v2_schema_has_no_sequences_the_manifest_would_need_to_cover() -> None: From eb4c29c1db174f650bc49ec733ed5e9b374e39aa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:56:01 +0400 Subject: [PATCH 016/118] Run the saved-policy view tests in the cross-database integration workflow The workflow runs an explicit list of integration modules, and test_v2_saved_policy_view.py was not in it. It now runs in the step for signed-in identity provisioning and user associations, and rule api.v2.runtime-privileges-tested-as-runtime-role names it, since it runs the saved-policy services as policyengine_v2_runtime. A unit test fails when an integration module that uses the managed v2 roles is missing from the workflow, or one that runs services as a managed role is missing from the rule's scope. No environment variable is added. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/v2-integration-check.yml | 2 +- docs/engineering/skills/alembic-migrations.md | 1 + rules/registry.toml | 2 +- tests/unit/test_alembic_workflows.py | 36 +++++++++++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 3f402ae80..7959a83c2 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -82,7 +82,7 @@ jobs: - name: Test v2 policy persistence and immediate v1 mirroring run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py tests/integration/test_v2_saved_policy_view.py - name: Test comparison-run persistence and privilege reconciliation as the managed roles run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py - name: Test v2 household persistence and immediate v1 copying diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index a9f6ce77b..f656486ab 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -179,6 +179,7 @@ When a pull request adds or changes a v2 table, or a connector's use of one: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles uv run python scripts/reconcile_v2_privileges.py verify uv run pytest -q tests/integration/test_v2_user_identity_persistence.py \ + tests/integration/test_v2_saved_policy_view.py \ tests/integration/test_v2_stage12_comparison_persistence.py \ tests/integration/test_v2_privilege_reconciliation.py ``` diff --git a/rules/registry.toml b/rules/registry.toml index 66b86bb8d..24faf8334 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -671,7 +671,7 @@ id = "api.v2.runtime-privileges-tested-as-runtime-role" category = "migration" statement = "The cross-database integration tests run the v2 user services as policyengine_v2_runtime and the comparison-run services as policyengine_stage12_, holding only their declared privileges, so a statement those privileges don't allow fails CI." rationale = "docs/engineering/skills/alembic-migrations.md#table-privileges" -scope = ["tests/integration/conftest.py", "tests/integration/test_v2_user_identity_persistence.py", "tests/integration/test_v2_stage12_comparison_persistence.py"] +scope = ["tests/integration/conftest.py", "tests/integration/test_v2_user_identity_persistence.py", "tests/integration/test_v2_saved_policy_view.py", "tests/integration/test_v2_stage12_comparison_persistence.py"] tier = "mechanical" status = "enforced" autofix = false diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 797c82848..a0e85c6c5 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -3,6 +3,7 @@ import os import re import subprocess +import tomllib from pathlib import Path import pytest @@ -257,6 +258,41 @@ def test_v2_integration_check_grants_and_verifies_privileges_before_tests(): assert schema_upgrade < grant < verify < first_test +def _integration_modules_using(pattern: str) -> list[str]: + return sorted( + path.relative_to(REPO).as_posix() + for path in (REPO / "tests" / "integration").glob("test_*.py") + if re.search(pattern, path.read_text(encoding="utf-8")) + ) + + +def test_every_managed_role_integration_module_runs_in_ci_and_is_in_its_rule(): + """Integration modules that use the managed v2 roles run in the workflow. + + Those that run services as a managed role (``v2_engine_as_role``) are also + in the scope of rule ``api.v2.runtime-privileges-tested-as-runtime-role``. + """ + workflow = _workflow("v2-integration-check.yml") + rules = tomllib.loads((REPO / "rules" / "registry.toml").read_text()) + (runtime_role_rule,) = ( + rule + for rule in rules["rule"] + if rule["id"] == "api.v2.runtime-privileges-tested-as-runtime-role" + ) + managed_role_modules = _integration_modules_using( + r"\b(managed_v2_roles|v2_engine_as_role)\b" + ) + role_session_modules = _integration_modules_using(r"\bv2_engine_as_role\b") + + assert "tests/integration/test_v2_saved_policy_view.py" in role_session_modules + assert [module for module in managed_role_modules if module not in workflow] == [] + assert [ + module + for module in role_session_modules + if module not in runtime_role_rule["scope"] + ] == [] + + def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): workflow = _workflow("push.yml") orchestration_script = ( From c0794eb2399d847a1fc640d06ac2fc8b9cdb8ad3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:00:05 +0400 Subject: [PATCH 017/118] Require each declared relation to exist with its declared kind The reconciliation read tables and views as one set of names, so a view satisfied a table entry of the same name and a table satisfied a temporary-view entry: before the rename, the user_policies table satisfied the user_policies view's entry. It now reads tables and views separately. A declared table must exist as a table and a TEMPORARY_VIEW_TABLES entry as a view; anything else is reported under "Declared tables missing" or "Declared views missing", nothing is granted, and verify reports every privilege declared on it as missing. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/alembic-migrations.md | 6 +- .../data/v2/privilege_reconciliation.py | 98 ++++++++++++++----- tests/integration/conftest.py | 4 +- .../test_v2_privilege_reconciliation.py | 40 +++++++- .../unit/v2/test_privilege_reconciliation.py | 80 ++++++++++++++- 5 files changed, 192 insertions(+), 36 deletions(-) diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index f656486ab..093358e22 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -155,8 +155,10 @@ check the declarations: `apply` grants each declared privilege a role lacks. It never revokes: it reports privileges the file doesn't declare, and the projects' default -privileges, in the job summary. It grants nothing when a declared role or -table is missing. The roles themselves are provisioned outside the +privileges, in the job summary. It grants nothing when a declared role, +table or view is missing; a declared table must exist as a table and a +temporary view (`TEMPORARY_VIEW_TABLES`) as a view, so a relation of the other +kind counts as missing. The roles themselves are provisioned outside the repository; `--create-missing-roles` is accepted only for the disposable test database. diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index bb5d673ab..33af3896b 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -6,8 +6,10 @@ - ``apply`` reads the managed roles' current grants, grants every declared privilege a role lacks, and reports privileges the manifest doesn't declare. - It never revokes, and it changes nothing when a declared role or table is - missing. Running it again grants nothing more. + It never revokes, and it changes nothing when a declared role, table or view + is missing. A declared table must exist as a table and a declared temporary + view as a view; a relation of the other kind counts as missing. Running it + again grants nothing more. - ``verify`` checks every declared privilege with ``has_table_privilege`` (rule ``api.v2.deployed-privileges-match``) and fails on any that is missing. @@ -42,6 +44,7 @@ ) from policyengine_api.data.v2.privileges import ( PERSISTENT_ENVIRONMENTS, + TEMPORARY_VIEW_TABLES, TablePrivilege, declared_privileges, ) @@ -56,6 +59,8 @@ GITHUB_STEP_SUMMARY = "GITHUB_STEP_SUMMARY" SCHEMA = "public" SAFE_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") +# The manifest's temporary views; every other declared name is a table. +DECLARED_VIEWS: frozenset[str] = frozenset(TEMPORARY_VIEW_TABLES) class PrivilegeReconciliationError(RuntimeError): @@ -94,11 +99,20 @@ class ReconciliationPlan: undeclared: tuple[PrivilegeEntry, ...] missing_roles: tuple[str, ...] missing_tables: tuple[str, ...] + missing_views: tuple[str, ...] = () @property def can_apply(self) -> bool: - """Whether every declared role and table exists, so grants may run.""" - return not self.missing_roles and not self.missing_tables + """Whether every declared role, table and view exists, so grants may run.""" + return not (self.missing_roles or self.missing_tables or self.missing_views) + + +@dataclass(frozen=True) +class PublicRelations: + """The tables and views in the public schema, by name.""" + + tables: frozenset[str] + views: frozenset[str] def plan_reconciliation( @@ -106,21 +120,30 @@ def plan_reconciliation( granted: Mapping[str, Mapping[str, Collection[str]]], *, existing_roles: Collection[str], - existing_tables: Collection[str], + existing_relations: PublicRelations, + declared_views: Collection[str] = (), ) -> ReconciliationPlan: """Compare declared privileges with what the database grants. ``granted`` holds the managed roles' direct grants, by role then table. + Names in ``declared_views`` must exist as views, and every other declared + name as a table. """ missing_roles = tuple(sorted(set(declared) - set(existing_roles))) - declared_tables = {table for tables in declared.values() for table in tables} - missing_tables = tuple(sorted(declared_tables - set(existing_tables))) + declared_names = {table for tables in declared.values() for table in tables} + missing_views = tuple( + sorted((declared_names & set(declared_views)) - existing_relations.views) + ) + missing_tables = tuple( + sorted(declared_names - set(declared_views) - existing_relations.tables) + ) + missing_relations = {*missing_tables, *missing_views} grants = sorted( PrivilegeEntry(role, table, privilege.value) for role, tables in declared.items() if role not in missing_roles for table, privileges in tables.items() - if table not in missing_tables + if table not in missing_relations for privilege in privileges if privilege.value not in granted.get(role, {}).get(table, ()) ) @@ -140,6 +163,7 @@ def plan_reconciliation( undeclared=tuple(undeclared), missing_roles=missing_roles, missing_tables=missing_tables, + missing_views=missing_views, ) @@ -214,6 +238,12 @@ def render_summary( + ", ".join(f"`{table}`" for table in plan.missing_tables), "", ] + if plan.missing_views: + lines += [ + "Declared views missing from the database (nothing was granted): " + + ", ".join(f"`{view}`" for view in plan.missing_views), + "", + ] grant_heading = "Granted" if applied else "Would grant" lines += [f"### {grant_heading}", ""] if plan.grants: @@ -270,16 +300,17 @@ def render_summary( "SELECT rolname FROM pg_roles WHERE rolname IN :roles" ).bindparams(bindparam("roles", expanding=True)) -_PUBLIC_TABLES = text( +_PUBLIC_RELATIONS = text( """ - SELECT table_class.relname - FROM pg_class AS table_class - JOIN pg_namespace AS table_namespace - ON table_namespace.oid = table_class.relnamespace - WHERE table_namespace.nspname = :schema - AND table_class.relkind IN ('r', 'p', 'v') + SELECT relation_class.relname, relation_class.relkind + FROM pg_class AS relation_class + JOIN pg_namespace AS relation_namespace + ON relation_namespace.oid = relation_class.relnamespace + WHERE relation_namespace.nspname = :schema + AND relation_class.relkind IN ('r', 'p', 'v') """ ) +_TABLE_KINDS = frozenset({"r", "p"}) _DEFAULT_PRIVILEGES = text( """ @@ -312,13 +343,17 @@ def read_existing_roles(connection: Connection, roles: Collection[str]) -> set[s return set(connection.execute(_EXISTING_ROLES, {"roles": list(roles)}).scalars()) -def read_public_tables(connection: Connection) -> set[str]: - """Return every table and view in the public schema. +def read_public_relations(connection: Connection) -> PublicRelations: + """Return the tables and the views in the public schema. - Views count because the manifest declares privileges on temporary views as - it does on tables, and ``GRANT ... ON TABLE`` grants on a view too. + The manifest declares privileges on temporary views as it does on tables, + and ``GRANT ... ON TABLE`` grants on a view too. """ - return set(connection.execute(_PUBLIC_TABLES, {"schema": SCHEMA}).scalars()) + rows = connection.execute(_PUBLIC_RELATIONS, {"schema": SCHEMA}).all() + return PublicRelations( + tables=frozenset(name for name, kind in rows if kind in _TABLE_KINDS), + views=frozenset(name for name, kind in rows if kind == "v"), + ) def read_granted_privileges( @@ -368,17 +403,22 @@ def create_missing_roles(connection: Connection, roles: Iterable[str]) -> list[s def apply_declared_privileges( connection: Connection, declared: Mapping[str, Mapping[str, frozenset[TablePrivilege]]], + *, + declared_views: Collection[str] = DECLARED_VIEWS, ) -> tuple[ReconciliationPlan, list[DefaultPrivilege]]: """Grant every declared privilege a role lacks; never revoke. - Grants nothing when a declared role or table is missing. + Grants nothing when a declared role, table or view is missing, including a + declared view that exists as a table or a declared table that exists as a + view. """ roles = list(declared) plan = plan_reconciliation( declared, read_granted_privileges(connection, roles), existing_roles=read_existing_roles(connection, roles), - existing_tables=read_public_tables(connection), + existing_relations=read_public_relations(connection), + declared_views=declared_views, ) if plan.can_apply: for statement in grant_statements( @@ -391,16 +431,24 @@ def apply_declared_privileges( def check_declared_privileges( connection: Connection, declared: Mapping[str, Mapping[str, frozenset[TablePrivilege]]], + *, + declared_views: Collection[str] = DECLARED_VIEWS, ) -> list[PrivilegeEntry]: """Return every declared privilege a role doesn't effectively hold. - A missing role or table counts as missing every privilege declared for it. + A missing role, table or view counts as missing every privilege declared + for it; so does a declared view that exists as a table, or the reverse. """ existing_roles = read_existing_roles(connection, list(declared)) - existing_tables = read_public_tables(connection) + relations = read_public_relations(connection) + + def exists_as_declared(name: str) -> bool: + if name in declared_views: + return name in relations.views + return name in relations.tables def has_privilege(role: str, table: str, privilege: str) -> bool: - if role not in existing_roles or table not in existing_tables: + if role not in existing_roles or not exists_as_declared(table): return False return bool( connection.execute( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b90efebd2..eb855ebb2 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -105,8 +105,8 @@ def managed_v2_roles(disposable_v2_database_url: str) -> ManagedV2Roles: engine.dispose() if not plan.can_apply: pytest.fail( - "the disposable v2 database lacks declared roles or tables: " - f"{plan.missing_roles} {plan.missing_tables}" + "the disposable v2 database lacks declared roles, tables or views: " + f"{plan.missing_roles} {plan.missing_tables} {plan.missing_views}" ) return ManagedV2Roles( deployment_environment=deployment_environment, diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index 38db72d94..9596a5966 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -161,13 +161,47 @@ def test_apply_grants_and_verify_checks_privileges_on_a_declared_view( role = temporary_role() declared = {role: {temporary_view: READ_WRITE}} with superuser_engine.begin() as connection: - plan, _defaults = apply_declared_privileges(connection, declared) - missing = check_declared_privileges(connection, declared) + plan, _defaults = apply_declared_privileges( + connection, declared, declared_views={temporary_view} + ) + missing = check_declared_privileges( + connection, declared, declared_views={temporary_view} + ) - assert plan.missing_tables == () + assert plan.missing_tables == plan.missing_views == () assert {entry.privilege for entry in plan.grants} == { privilege.value for privilege in READ_WRITE } assert missing == [] for privilege in READ_WRITE: assert _has_privilege(superuser_engine, role, temporary_view, privilege.value) + + +def test_a_view_declared_as_a_table_or_a_table_declared_as_a_view_blocks_grants( + superuser_engine, temporary_role, temporary_view +) -> None: + """Each declared name must exist with its declared kind. + + ``regions`` is a table, declared here as a view, and the temporary view is + declared as a table: neither is granted, and both fail verification. + """ + role = temporary_role() + declared = {role: {"regions": READ, temporary_view: READ}} + with superuser_engine.begin() as connection: + plan, _defaults = apply_declared_privileges( + connection, declared, declared_views={"regions"} + ) + missing = check_declared_privileges( + connection, declared, declared_views={"regions"} + ) + + assert plan.missing_views == ("regions",) + assert plan.missing_tables == (temporary_view,) + assert not plan.can_apply + assert not _has_privilege(superuser_engine, role, "regions", "SELECT") + assert missing == sorted( + [ + PrivilegeEntry(role, "regions", "SELECT"), + PrivilegeEntry(role, temporary_view, "SELECT"), + ] + ) diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index 73ca30e19..a6d9590dc 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -11,6 +11,7 @@ from policyengine_api.data.v2.privilege_reconciliation import ( DefaultPrivilege, PrivilegeEntry, + PublicRelations, ReconciliationPlan, grant_statements, plan_reconciliation, @@ -48,12 +49,16 @@ def _declared() -> dict[str, dict[str, frozenset[TablePrivilege]]]: } +def _tables(names: set[str]) -> PublicRelations: + return PublicRelations(tables=frozenset(names), views=frozenset()) + + def test_missing_declared_privileges_become_grants() -> None: plan = plan_reconciliation( _declared(), granted={RUNTIME: {"users": {"SELECT", "INSERT"}}}, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports"}, + existing_relations=_tables({"users", "legacy_policy_mappings", "reports"}), ) assert plan.grants == ( @@ -78,7 +83,7 @@ def test_privileges_beyond_the_manifest_are_reported_not_granted_or_revoked() -> STAGE12: {"users": {"SELECT"}}, }, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports"}, + existing_relations=_tables({"users", "legacy_policy_mappings", "reports"}), ) assert plan.grants == () @@ -96,7 +101,9 @@ def test_privileges_on_tables_outside_the_manifest_are_reported() -> None: _declared(), granted={RUNTIME: {"scratch_table": {"SELECT"}}}, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports", "scratch_table"}, + existing_relations=_tables( + {"users", "legacy_policy_mappings", "reports", "scratch_table"} + ), ) assert PrivilegeEntry(RUNTIME, "scratch_table", "SELECT") in plan.undeclared @@ -107,7 +114,7 @@ def test_missing_roles_and_tables_are_recorded_and_block_the_plan() -> None: _declared(), granted={}, existing_roles={RUNTIME}, - existing_tables={"users", "reports"}, + existing_relations=_tables({"users", "reports"}), ) assert plan.missing_roles == (STAGE12,) @@ -120,6 +127,71 @@ def test_missing_roles_and_tables_are_recorded_and_block_the_plan() -> None: ) +def test_a_relation_of_the_other_kind_does_not_satisfy_a_declaration() -> None: + """A declared table must be a table, and a declared view must be a view. + + Before the rename, ``user_policies`` is a table, which must not satisfy the + temporary view of that name; a view named like a declared table must not + satisfy the table either. + """ + declared = { + RUNTIME: {"users": READ_WRITE, "user_policies": READ_WRITE, "reports": READ} + } + + plan = plan_reconciliation( + declared, + granted={}, + existing_roles={RUNTIME}, + existing_relations=PublicRelations( + tables=frozenset({"user_policies", "reports"}), views=frozenset({"users"}) + ), + declared_views={"user_policies"}, + ) + + assert plan.missing_tables == ("users",) + assert plan.missing_views == ("user_policies",) + assert not plan.can_apply + + +def test_declared_views_that_exist_as_views_are_granted() -> None: + declared = {RUNTIME: {"users": READ, "user_policies": READ_WRITE}} + + plan = plan_reconciliation( + declared, + granted={}, + existing_roles={RUNTIME}, + existing_relations=PublicRelations( + tables=frozenset({"users"}), views=frozenset({"user_policies"}) + ), + declared_views={"user_policies"}, + ) + + assert plan.can_apply + assert plan.missing_tables == plan.missing_views == () + assert {entry.table for entry in plan.grants} == {"users", "user_policies"} + + +def test_summary_names_missing_views_separately_from_missing_tables() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=("users",), + missing_views=("user_policies",), + ) + + summary = render_summary(plan, (), deployment_environment="staging", applied=False) + + assert ( + "Declared tables missing from the database (nothing was granted): `users`" + in summary + ) + assert ( + "Declared views missing from the database (nothing was granted): " + "`user_policies`" in summary + ) + + def test_grant_statements_group_privileges_per_role_and_table() -> None: plan = ReconciliationPlan( grants=( From fa682e2929f2e54a0cbb0fa444211d1450dbc237 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:54:33 +0400 Subject: [PATCH 018/118] Add checked-in region reference files for the US and the UK Each file lists every region of one country with its code, label, type, parent, US state, the value that selects its households, and the v1 identifier it replaces. The UK file lists 1,016 regions: the United Kingdom, its four nations (Northern Ireland as country/northern_ireland, replacing country/ni), 650 constituencies and 361 local authorities by ONS code (constituency/E14001063 replaces constituency/Aldershot), each under its nation. The 361 include Ards and North Down (N09000011), which the Public API's own list now carries. The US file lists the United States, 51 states, 436 congressional districts with unchanged codes, and 333 places. The constituency and local-authority rows come from the Public API's own lookup files; the US states, districts and places, and every filter value, match the region registry of PolicyEngine.py 6.7.5. A loader validates each file: unique codes and v1 identifiers, codes that start with their type, and parents that lead to the national region. Nothing reads the files yet. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/data/v2/catalog/errors.py | 7 + .../data/v2/catalog/extraction.py | 25 +- .../v2/catalog/region_reference_files/uk.csv | 1017 +++++++++++++++++ .../v2/catalog/region_reference_files/us.csv | 822 +++++++++++++ .../data/v2/catalog/region_references.py | 251 ++++ tests/unit/v2/test_region_references.py | 318 ++++++ 6 files changed, 2420 insertions(+), 20 deletions(-) create mode 100644 policyengine_api/data/v2/catalog/errors.py create mode 100644 policyengine_api/data/v2/catalog/region_reference_files/uk.csv create mode 100644 policyengine_api/data/v2/catalog/region_reference_files/us.csv create mode 100644 policyengine_api/data/v2/catalog/region_references.py create mode 100644 tests/unit/v2/test_region_references.py diff --git a/policyengine_api/data/v2/catalog/errors.py b/policyengine_api/data/v2/catalog/errors.py new file mode 100644 index 000000000..53be90f51 --- /dev/null +++ b/policyengine_api/data/v2/catalog/errors.py @@ -0,0 +1,7 @@ +"""The error raised when catalog sources cannot be normalized.""" + +from __future__ import annotations + + +class CatalogExtractionError(RuntimeError): + """Raised before database access when source metadata cannot be normalized.""" diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 75b0b213f..99ac8086b 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -14,6 +14,7 @@ from urllib.parse import urlsplit from uuid import NAMESPACE_URL, UUID, uuid5 +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.records import ( CountryCatalog, DatasetRecord, @@ -27,6 +28,10 @@ RegionRecord, VariableRecord, ) +from policyengine_api.data.v2.catalog.region_references import ( + MAX_REGION_FILTER_VALUE_LENGTH, + SUPPORTED_REGION_TYPES, +) if TYPE_CHECKING: from policyengine_api.json_types import JSONValue @@ -46,26 +51,10 @@ "us": tuple(range(2035, 2021, -1)), "uk": tuple(range(2024, 2031)), } -SUPPORTED_REGION_TYPES = frozenset( - { - "national", - "country", - "state", - "congressional_district", - "constituency", - "local_authority", - "city", - "place", - } -) PLACEHOLDER_VERSIONS = frozenset({"", "0", "0.0", "0.0.0", "unknown"}) DATASET_YEAR_PATTERN = re.compile(r"(?:19|20)\d{2}") -class CatalogExtractionError(RuntimeError): - """Raised before database access when source metadata cannot be normalized.""" - - def _identifier(*parts: object) -> UUID: value = "/".join(str(part) for part in parts) return uuid5(NAMESPACE_URL, f"https://api.policyengine.org/v2/catalog/{value}") @@ -99,10 +88,6 @@ def _optional_text( return value -# The catalog's region filter_value column holds at most this many characters. -MAX_REGION_FILTER_VALUE_LENGTH = 255 - - def _serialize_region_filter_value(value: object, *, region_code: str) -> str: """Keep scalar text or encode flat membership values in the existing column. diff --git a/policyengine_api/data/v2/catalog/region_reference_files/uk.csv b/policyengine_api/data/v2/catalog/region_reference_files/uk.csv new file mode 100644 index 000000000..e20f450de --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_reference_files/uk.csv @@ -0,0 +1,1017 @@ +code,label,region_type,parent_code,state_code,state_name,filter_value,v1_code +uk,United Kingdom,national,,,,,uk +country/england,England,country,uk,,,"[""NORTH_EAST"",""NORTH_WEST"",""YORKSHIRE"",""EAST_MIDLANDS"",""WEST_MIDLANDS"",""EAST_OF_ENGLAND"",""LONDON"",""SOUTH_EAST"",""SOUTH_WEST""]",country/england +country/scotland,Scotland,country,uk,,,SCOTLAND,country/scotland +country/wales,Wales,country,uk,,,WALES,country/wales +country/northern_ireland,Northern Ireland,country,uk,,,NORTHERN_IRELAND,country/ni +constituency/E14001063,Aldershot,constituency,country/england,,,E14001063,constituency/Aldershot +constituency/E14001064,Aldridge-Brownhills,constituency,country/england,,,E14001064,constituency/Aldridge-Brownhills +constituency/E14001065,Altrincham and Sale West,constituency,country/england,,,E14001065,constituency/Altrincham and Sale West +constituency/E14001066,Amber Valley,constituency,country/england,,,E14001066,constituency/Amber Valley +constituency/E14001067,Arundel and South Downs,constituency,country/england,,,E14001067,constituency/Arundel and South Downs +constituency/E14001068,Ashfield,constituency,country/england,,,E14001068,constituency/Ashfield +constituency/E14001069,Ashford,constituency,country/england,,,E14001069,constituency/Ashford +constituency/E14001070,Ashton-under-Lyne,constituency,country/england,,,E14001070,constituency/Ashton-under-Lyne +constituency/E14001071,Aylesbury,constituency,country/england,,,E14001071,constituency/Aylesbury +constituency/E14001072,Banbury,constituency,country/england,,,E14001072,constituency/Banbury +constituency/E14001073,Barking,constituency,country/england,,,E14001073,constituency/Barking +constituency/E14001074,Barnsley North,constituency,country/england,,,E14001074,constituency/Barnsley North +constituency/E14001075,Barnsley South,constituency,country/england,,,E14001075,constituency/Barnsley South +constituency/E14001076,Barrow and Furness,constituency,country/england,,,E14001076,constituency/Barrow and Furness +constituency/E14001077,Basildon and Billericay,constituency,country/england,,,E14001077,constituency/Basildon and Billericay +constituency/E14001078,Basingstoke,constituency,country/england,,,E14001078,constituency/Basingstoke +constituency/E14001079,Bassetlaw,constituency,country/england,,,E14001079,constituency/Bassetlaw +constituency/E14001080,Bath,constituency,country/england,,,E14001080,constituency/Bath +constituency/E14001081,Battersea,constituency,country/england,,,E14001081,constituency/Battersea +constituency/E14001082,Beaconsfield,constituency,country/england,,,E14001082,constituency/Beaconsfield +constituency/E14001083,Beckenham and Penge,constituency,country/england,,,E14001083,constituency/Beckenham and Penge +constituency/E14001084,Bedford,constituency,country/england,,,E14001084,constituency/Bedford +constituency/E14001085,Bermondsey and Old Southwark,constituency,country/england,,,E14001085,constituency/Bermondsey and Old Southwark +constituency/E14001086,Bethnal Green and Stepney,constituency,country/england,,,E14001086,constituency/Bethnal Green and Stepney +constituency/E14001087,Beverley and Holderness,constituency,country/england,,,E14001087,constituency/Beverley and Holderness +constituency/E14001088,Bexhill and Battle,constituency,country/england,,,E14001088,constituency/Bexhill and Battle +constituency/E14001089,Bexleyheath and Crayford,constituency,country/england,,,E14001089,constituency/Bexleyheath and Crayford +constituency/E14001090,Bicester and Woodstock,constituency,country/england,,,E14001090,constituency/Bicester and Woodstock +constituency/E14001091,Birkenhead,constituency,country/england,,,E14001091,constituency/Birkenhead +constituency/E14001092,Birmingham Edgbaston,constituency,country/england,,,E14001092,constituency/Birmingham Edgbaston +constituency/E14001093,Birmingham Erdington,constituency,country/england,,,E14001093,constituency/Birmingham Erdington +constituency/E14001094,Birmingham Hall Green and Moseley,constituency,country/england,,,E14001094,constituency/Birmingham Hall Green and Moseley +constituency/E14001095,Birmingham Hodge Hill and Solihull North,constituency,country/england,,,E14001095,constituency/Birmingham Hodge Hill and Solihull North +constituency/E14001096,Birmingham Ladywood,constituency,country/england,,,E14001096,constituency/Birmingham Ladywood +constituency/E14001097,Birmingham Northfield,constituency,country/england,,,E14001097,constituency/Birmingham Northfield +constituency/E14001098,Birmingham Perry Barr,constituency,country/england,,,E14001098,constituency/Birmingham Perry Barr +constituency/E14001099,Birmingham Selly Oak,constituency,country/england,,,E14001099,constituency/Birmingham Selly Oak +constituency/E14001100,Birmingham Yardley,constituency,country/england,,,E14001100,constituency/Birmingham Yardley +constituency/E14001101,Bishop Auckland,constituency,country/england,,,E14001101,constituency/Bishop Auckland +constituency/E14001102,Blackburn,constituency,country/england,,,E14001102,constituency/Blackburn +constituency/E14001103,Blackley and Middleton South,constituency,country/england,,,E14001103,constituency/Blackley and Middleton South +constituency/E14001104,Blackpool North and Fleetwood,constituency,country/england,,,E14001104,constituency/Blackpool North and Fleetwood +constituency/E14001105,Blackpool South,constituency,country/england,,,E14001105,constituency/Blackpool South +constituency/E14001106,Blaydon and Consett,constituency,country/england,,,E14001106,constituency/Blaydon and Consett +constituency/E14001107,Blyth and Ashington,constituency,country/england,,,E14001107,constituency/Blyth and Ashington +constituency/E14001108,Bognor Regis and Littlehampton,constituency,country/england,,,E14001108,constituency/Bognor Regis and Littlehampton +constituency/E14001109,Bolsover,constituency,country/england,,,E14001109,constituency/Bolsover +constituency/E14001110,Bolton North East,constituency,country/england,,,E14001110,constituency/Bolton North East +constituency/E14001111,Bolton South and Walkden,constituency,country/england,,,E14001111,constituency/Bolton South and Walkden +constituency/E14001112,Bolton West,constituency,country/england,,,E14001112,constituency/Bolton West +constituency/E14001113,Bootle,constituency,country/england,,,E14001113,constituency/Bootle +constituency/E14001114,Boston and Skegness,constituency,country/england,,,E14001114,constituency/Boston and Skegness +constituency/E14001115,Bournemouth East,constituency,country/england,,,E14001115,constituency/Bournemouth East +constituency/E14001116,Bournemouth West,constituency,country/england,,,E14001116,constituency/Bournemouth West +constituency/E14001117,Bracknell,constituency,country/england,,,E14001117,constituency/Bracknell +constituency/E14001118,Bradford East,constituency,country/england,,,E14001118,constituency/Bradford East +constituency/E14001119,Bradford South,constituency,country/england,,,E14001119,constituency/Bradford South +constituency/E14001120,Bradford West,constituency,country/england,,,E14001120,constituency/Bradford West +constituency/E14001121,Braintree,constituency,country/england,,,E14001121,constituency/Braintree +constituency/E14001122,Brent East,constituency,country/england,,,E14001122,constituency/Brent East +constituency/E14001123,Brent West,constituency,country/england,,,E14001123,constituency/Brent West +constituency/E14001124,Brentford and Isleworth,constituency,country/england,,,E14001124,constituency/Brentford and Isleworth +constituency/E14001125,Brentwood and Ongar,constituency,country/england,,,E14001125,constituency/Brentwood and Ongar +constituency/E14001126,Bridgwater,constituency,country/england,,,E14001126,constituency/Bridgwater +constituency/E14001127,Bridlington and The Wolds,constituency,country/england,,,E14001127,constituency/Bridlington and The Wolds +constituency/E14001128,Brigg and Immingham,constituency,country/england,,,E14001128,constituency/Brigg and Immingham +constituency/E14001129,Brighton Kemptown and Peacehaven,constituency,country/england,,,E14001129,constituency/Brighton Kemptown and Peacehaven +constituency/E14001130,Brighton Pavilion,constituency,country/england,,,E14001130,constituency/Brighton Pavilion +constituency/E14001131,Bristol Central,constituency,country/england,,,E14001131,constituency/Bristol Central +constituency/E14001132,Bristol East,constituency,country/england,,,E14001132,constituency/Bristol East +constituency/E14001133,Bristol North East,constituency,country/england,,,E14001133,constituency/Bristol North East +constituency/E14001134,Bristol North West,constituency,country/england,,,E14001134,constituency/Bristol North West +constituency/E14001135,Bristol South,constituency,country/england,,,E14001135,constituency/Bristol South +constituency/E14001136,Broadland and Fakenham,constituency,country/england,,,E14001136,constituency/Broadland and Fakenham +constituency/E14001137,Bromley and Biggin Hill,constituency,country/england,,,E14001137,constituency/Bromley and Biggin Hill +constituency/E14001138,Bromsgrove,constituency,country/england,,,E14001138,constituency/Bromsgrove +constituency/E14001139,Broxbourne,constituency,country/england,,,E14001139,constituency/Broxbourne +constituency/E14001140,Broxtowe,constituency,country/england,,,E14001140,constituency/Broxtowe +constituency/E14001141,Buckingham and Bletchley,constituency,country/england,,,E14001141,constituency/Buckingham and Bletchley +constituency/E14001142,Burnley,constituency,country/england,,,E14001142,constituency/Burnley +constituency/E14001143,Burton and Uttoxeter,constituency,country/england,,,E14001143,constituency/Burton and Uttoxeter +constituency/E14001144,Bury North,constituency,country/england,,,E14001144,constituency/Bury North +constituency/E14001145,Bury South,constituency,country/england,,,E14001145,constituency/Bury South +constituency/E14001146,Bury St Edmunds and Stowmarket,constituency,country/england,,,E14001146,constituency/Bury St Edmunds and Stowmarket +constituency/E14001147,Calder Valley,constituency,country/england,,,E14001147,constituency/Calder Valley +constituency/E14001148,Camborne and Redruth,constituency,country/england,,,E14001148,constituency/Camborne and Redruth +constituency/E14001149,Cambridge,constituency,country/england,,,E14001149,constituency/Cambridge +constituency/E14001150,Cannock Chase,constituency,country/england,,,E14001150,constituency/Cannock Chase +constituency/E14001151,Canterbury,constituency,country/england,,,E14001151,constituency/Canterbury +constituency/E14001152,Carlisle,constituency,country/england,,,E14001152,constituency/Carlisle +constituency/E14001153,Carshalton and Wallington,constituency,country/england,,,E14001153,constituency/Carshalton and Wallington +constituency/E14001154,Castle Point,constituency,country/england,,,E14001154,constituency/Castle Point +constituency/E14001155,Central Devon,constituency,country/england,,,E14001155,constituency/Central Devon +constituency/E14001156,Central Suffolk and North Ipswich,constituency,country/england,,,E14001156,constituency/Central Suffolk and North Ipswich +constituency/E14001157,Chatham and Aylesford,constituency,country/england,,,E14001157,constituency/Chatham and Aylesford +constituency/E14001158,Cheadle,constituency,country/england,,,E14001158,constituency/Cheadle +constituency/E14001159,Chelmsford,constituency,country/england,,,E14001159,constituency/Chelmsford +constituency/E14001160,Chelsea and Fulham,constituency,country/england,,,E14001160,constituency/Chelsea and Fulham +constituency/E14001161,Cheltenham,constituency,country/england,,,E14001161,constituency/Cheltenham +constituency/E14001162,Chesham and Amersham,constituency,country/england,,,E14001162,constituency/Chesham and Amersham +constituency/E14001163,Chester North and Neston,constituency,country/england,,,E14001163,constituency/Chester North and Neston +constituency/E14001164,Chester South and Eddisbury,constituency,country/england,,,E14001164,constituency/Chester South and Eddisbury +constituency/E14001165,Chesterfield,constituency,country/england,,,E14001165,constituency/Chesterfield +constituency/E14001166,Chichester,constituency,country/england,,,E14001166,constituency/Chichester +constituency/E14001167,Chingford and Woodford Green,constituency,country/england,,,E14001167,constituency/Chingford and Woodford Green +constituency/E14001168,Chippenham,constituency,country/england,,,E14001168,constituency/Chippenham +constituency/E14001169,Chipping Barnet,constituency,country/england,,,E14001169,constituency/Chipping Barnet +constituency/E14001170,Chorley,constituency,country/england,,,E14001170,constituency/Chorley +constituency/E14001171,Christchurch,constituency,country/england,,,E14001171,constituency/Christchurch +constituency/E14001172,Cities of London and Westminster,constituency,country/england,,,E14001172,constituency/Cities of London and Westminster +constituency/E14001173,City of Durham,constituency,country/england,,,E14001173,constituency/City of Durham +constituency/E14001174,Clacton,constituency,country/england,,,E14001174,constituency/Clacton +constituency/E14001175,Clapham and Brixton Hill,constituency,country/england,,,E14001175,constituency/Clapham and Brixton Hill +constituency/E14001176,Colchester,constituency,country/england,,,E14001176,constituency/Colchester +constituency/E14001177,Colne Valley,constituency,country/england,,,E14001177,constituency/Colne Valley +constituency/E14001178,Congleton,constituency,country/england,,,E14001178,constituency/Congleton +constituency/E14001179,Corby and East Northamptonshire,constituency,country/england,,,E14001179,constituency/Corby and East Northamptonshire +constituency/E14001180,Coventry East,constituency,country/england,,,E14001180,constituency/Coventry East +constituency/E14001181,Coventry North West,constituency,country/england,,,E14001181,constituency/Coventry North West +constituency/E14001182,Coventry South,constituency,country/england,,,E14001182,constituency/Coventry South +constituency/E14001183,Cramlington and Killingworth,constituency,country/england,,,E14001183,constituency/Cramlington and Killingworth +constituency/E14001184,Crawley,constituency,country/england,,,E14001184,constituency/Crawley +constituency/E14001185,Crewe and Nantwich,constituency,country/england,,,E14001185,constituency/Crewe and Nantwich +constituency/E14001186,Croydon East,constituency,country/england,,,E14001186,constituency/Croydon East +constituency/E14001187,Croydon South,constituency,country/england,,,E14001187,constituency/Croydon South +constituency/E14001188,Croydon West,constituency,country/england,,,E14001188,constituency/Croydon West +constituency/E14001189,Dagenham and Rainham,constituency,country/england,,,E14001189,constituency/Dagenham and Rainham +constituency/E14001190,Darlington,constituency,country/england,,,E14001190,constituency/Darlington +constituency/E14001191,Dartford,constituency,country/england,,,E14001191,constituency/Dartford +constituency/E14001192,Daventry,constituency,country/england,,,E14001192,constituency/Daventry +constituency/E14001193,Derby North,constituency,country/england,,,E14001193,constituency/Derby North +constituency/E14001194,Derby South,constituency,country/england,,,E14001194,constituency/Derby South +constituency/E14001195,Derbyshire Dales,constituency,country/england,,,E14001195,constituency/Derbyshire Dales +constituency/E14001196,Dewsbury and Batley,constituency,country/england,,,E14001196,constituency/Dewsbury and Batley +constituency/E14001197,Didcot and Wantage,constituency,country/england,,,E14001197,constituency/Didcot and Wantage +constituency/E14001198,Doncaster Central,constituency,country/england,,,E14001198,constituency/Doncaster Central +constituency/E14001199,Doncaster East and the Isle of Axholme,constituency,country/england,,,E14001199,constituency/Doncaster East and the Isle of Axholme +constituency/E14001200,Doncaster North,constituency,country/england,,,E14001200,constituency/Doncaster North +constituency/E14001201,Dorking and Horley,constituency,country/england,,,E14001201,constituency/Dorking and Horley +constituency/E14001202,Dover and Deal,constituency,country/england,,,E14001202,constituency/Dover and Deal +constituency/E14001203,Droitwich and Evesham,constituency,country/england,,,E14001203,constituency/Droitwich and Evesham +constituency/E14001204,Dudley,constituency,country/england,,,E14001204,constituency/Dudley +constituency/E14001205,Dulwich and West Norwood,constituency,country/england,,,E14001205,constituency/Dulwich and West Norwood +constituency/E14001206,Dunstable and Leighton Buzzard,constituency,country/england,,,E14001206,constituency/Dunstable and Leighton Buzzard +constituency/E14001207,Ealing Central and Acton,constituency,country/england,,,E14001207,constituency/Ealing Central and Acton +constituency/E14001208,Ealing North,constituency,country/england,,,E14001208,constituency/Ealing North +constituency/E14001209,Ealing Southall,constituency,country/england,,,E14001209,constituency/Ealing Southall +constituency/E14001210,Earley and Woodley,constituency,country/england,,,E14001210,constituency/Earley and Woodley +constituency/E14001211,Easington,constituency,country/england,,,E14001211,constituency/Easington +constituency/E14001212,East Grinstead and Uckfield,constituency,country/england,,,E14001212,constituency/East Grinstead and Uckfield +constituency/E14001213,East Ham,constituency,country/england,,,E14001213,constituency/East Ham +constituency/E14001214,East Hampshire,constituency,country/england,,,E14001214,constituency/East Hampshire +constituency/E14001215,East Surrey,constituency,country/england,,,E14001215,constituency/East Surrey +constituency/E14001216,East Thanet,constituency,country/england,,,E14001216,constituency/East Thanet +constituency/E14001217,East Wiltshire,constituency,country/england,,,E14001217,constituency/East Wiltshire +constituency/E14001218,East Worthing and Shoreham,constituency,country/england,,,E14001218,constituency/East Worthing and Shoreham +constituency/E14001219,Eastbourne,constituency,country/england,,,E14001219,constituency/Eastbourne +constituency/E14001220,Eastleigh,constituency,country/england,,,E14001220,constituency/Eastleigh +constituency/E14001221,Edmonton and Winchmore Hill,constituency,country/england,,,E14001221,constituency/Edmonton and Winchmore Hill +constituency/E14001222,Ellesmere Port and Bromborough,constituency,country/england,,,E14001222,constituency/Ellesmere Port and Bromborough +constituency/E14001223,Eltham and Chislehurst,constituency,country/england,,,E14001223,constituency/Eltham and Chislehurst +constituency/E14001224,Ely and East Cambridgeshire,constituency,country/england,,,E14001224,constituency/Ely and East Cambridgeshire +constituency/E14001225,Enfield North,constituency,country/england,,,E14001225,constituency/Enfield North +constituency/E14001226,Epping Forest,constituency,country/england,,,E14001226,constituency/Epping Forest +constituency/E14001227,Epsom and Ewell,constituency,country/england,,,E14001227,constituency/Epsom and Ewell +constituency/E14001228,Erewash,constituency,country/england,,,E14001228,constituency/Erewash +constituency/E14001229,Erith and Thamesmead,constituency,country/england,,,E14001229,constituency/Erith and Thamesmead +constituency/E14001230,Esher and Walton,constituency,country/england,,,E14001230,constituency/Esher and Walton +constituency/E14001231,Exeter,constituency,country/england,,,E14001231,constituency/Exeter +constituency/E14001232,Exmouth and Exeter East,constituency,country/england,,,E14001232,constituency/Exmouth and Exeter East +constituency/E14001233,Fareham and Waterlooville,constituency,country/england,,,E14001233,constituency/Fareham and Waterlooville +constituency/E14001234,Farnham and Bordon,constituency,country/england,,,E14001234,constituency/Farnham and Bordon +constituency/E14001235,Faversham and Mid Kent,constituency,country/england,,,E14001235,constituency/Faversham and Mid Kent +constituency/E14001236,Feltham and Heston,constituency,country/england,,,E14001236,constituency/Feltham and Heston +constituency/E14001237,Filton and Bradley Stoke,constituency,country/england,,,E14001237,constituency/Filton and Bradley Stoke +constituency/E14001238,Finchley and Golders Green,constituency,country/england,,,E14001238,constituency/Finchley and Golders Green +constituency/E14001239,Folkestone and Hythe,constituency,country/england,,,E14001239,constituency/Folkestone and Hythe +constituency/E14001240,Forest of Dean,constituency,country/england,,,E14001240,constituency/Forest of Dean +constituency/E14001241,Frome and East Somerset,constituency,country/england,,,E14001241,constituency/Frome and East Somerset +constituency/E14001242,Fylde,constituency,country/england,,,E14001242,constituency/Fylde +constituency/E14001243,Gainsborough,constituency,country/england,,,E14001243,constituency/Gainsborough +constituency/E14001244,Gateshead Central and Whickham,constituency,country/england,,,E14001244,constituency/Gateshead Central and Whickham +constituency/E14001245,Gedling,constituency,country/england,,,E14001245,constituency/Gedling +constituency/E14001246,Gillingham and Rainham,constituency,country/england,,,E14001246,constituency/Gillingham and Rainham +constituency/E14001247,Glastonbury and Somerton,constituency,country/england,,,E14001247,constituency/Glastonbury and Somerton +constituency/E14001248,Gloucester,constituency,country/england,,,E14001248,constituency/Gloucester +constituency/E14001249,Godalming and Ash,constituency,country/england,,,E14001249,constituency/Godalming and Ash +constituency/E14001250,Goole and Pocklington,constituency,country/england,,,E14001250,constituency/Goole and Pocklington +constituency/E14001251,Gorton and Denton,constituency,country/england,,,E14001251,constituency/Gorton and Denton +constituency/E14001252,Gosport,constituency,country/england,,,E14001252,constituency/Gosport +constituency/E14001253,Grantham and Bourne,constituency,country/england,,,E14001253,constituency/Grantham and Bourne +constituency/E14001254,Gravesham,constituency,country/england,,,E14001254,constituency/Gravesham +constituency/E14001255,Great Grimsby and Cleethorpes,constituency,country/england,,,E14001255,constituency/Great Grimsby and Cleethorpes +constituency/E14001256,Great Yarmouth,constituency,country/england,,,E14001256,constituency/Great Yarmouth +constituency/E14001257,Greenwich and Woolwich,constituency,country/england,,,E14001257,constituency/Greenwich and Woolwich +constituency/E14001258,Guildford,constituency,country/england,,,E14001258,constituency/Guildford +constituency/E14001259,Hackney North and Stoke Newington,constituency,country/england,,,E14001259,constituency/Hackney North and Stoke Newington +constituency/E14001260,Hackney South and Shoreditch,constituency,country/england,,,E14001260,constituency/Hackney South and Shoreditch +constituency/E14001261,Halesowen,constituency,country/england,,,E14001261,constituency/Halesowen +constituency/E14001262,Halifax,constituency,country/england,,,E14001262,constituency/Halifax +constituency/E14001263,Hamble Valley,constituency,country/england,,,E14001263,constituency/Hamble Valley +constituency/E14001264,Hammersmith and Chiswick,constituency,country/england,,,E14001264,constituency/Hammersmith and Chiswick +constituency/E14001265,Hampstead and Highgate,constituency,country/england,,,E14001265,constituency/Hampstead and Highgate +constituency/E14001266,"Harborough, Oadby and Wigston",constituency,country/england,,,E14001266,"constituency/Harborough, Oadby and Wigston" +constituency/E14001267,Harlow,constituency,country/england,,,E14001267,constituency/Harlow +constituency/E14001268,Harpenden and Berkhamsted,constituency,country/england,,,E14001268,constituency/Harpenden and Berkhamsted +constituency/E14001269,Harrogate and Knaresborough,constituency,country/england,,,E14001269,constituency/Harrogate and Knaresborough +constituency/E14001270,Harrow East,constituency,country/england,,,E14001270,constituency/Harrow East +constituency/E14001271,Harrow West,constituency,country/england,,,E14001271,constituency/Harrow West +constituency/E14001272,Hartlepool,constituency,country/england,,,E14001272,constituency/Hartlepool +constituency/E14001273,Harwich and North Essex,constituency,country/england,,,E14001273,constituency/Harwich and North Essex +constituency/E14001274,Hastings and Rye,constituency,country/england,,,E14001274,constituency/Hastings and Rye +constituency/E14001275,Havant,constituency,country/england,,,E14001275,constituency/Havant +constituency/E14001276,Hayes and Harlington,constituency,country/england,,,E14001276,constituency/Hayes and Harlington +constituency/E14001277,Hazel Grove,constituency,country/england,,,E14001277,constituency/Hazel Grove +constituency/E14001278,Hemel Hempstead,constituency,country/england,,,E14001278,constituency/Hemel Hempstead +constituency/E14001279,Hendon,constituency,country/england,,,E14001279,constituency/Hendon +constituency/E14001280,Henley and Thame,constituency,country/england,,,E14001280,constituency/Henley and Thame +constituency/E14001281,Hereford and South Herefordshire,constituency,country/england,,,E14001281,constituency/Hereford and South Herefordshire +constituency/E14001282,Herne Bay and Sandwich,constituency,country/england,,,E14001282,constituency/Herne Bay and Sandwich +constituency/E14001283,Hertford and Stortford,constituency,country/england,,,E14001283,constituency/Hertford and Stortford +constituency/E14001284,Hertsmere,constituency,country/england,,,E14001284,constituency/Hertsmere +constituency/E14001285,Hexham,constituency,country/england,,,E14001285,constituency/Hexham +constituency/E14001286,Heywood and Middleton North,constituency,country/england,,,E14001286,constituency/Heywood and Middleton North +constituency/E14001287,High Peak,constituency,country/england,,,E14001287,constituency/High Peak +constituency/E14001288,Hinckley and Bosworth,constituency,country/england,,,E14001288,constituency/Hinckley and Bosworth +constituency/E14001289,Hitchin,constituency,country/england,,,E14001289,constituency/Hitchin +constituency/E14001290,Holborn and St Pancras,constituency,country/england,,,E14001290,constituency/Holborn and St Pancras +constituency/E14001291,Honiton and Sidmouth,constituency,country/england,,,E14001291,constituency/Honiton and Sidmouth +constituency/E14001292,Hornchurch and Upminster,constituency,country/england,,,E14001292,constituency/Hornchurch and Upminster +constituency/E14001293,Hornsey and Friern Barnet,constituency,country/england,,,E14001293,constituency/Hornsey and Friern Barnet +constituency/E14001294,Horsham,constituency,country/england,,,E14001294,constituency/Horsham +constituency/E14001295,Houghton and Sunderland South,constituency,country/england,,,E14001295,constituency/Houghton and Sunderland South +constituency/E14001296,Hove and Portslade,constituency,country/england,,,E14001296,constituency/Hove and Portslade +constituency/E14001297,Huddersfield,constituency,country/england,,,E14001297,constituency/Huddersfield +constituency/E14001298,Huntingdon,constituency,country/england,,,E14001298,constituency/Huntingdon +constituency/E14001299,Hyndburn,constituency,country/england,,,E14001299,constituency/Hyndburn +constituency/E14001300,Ilford North,constituency,country/england,,,E14001300,constituency/Ilford North +constituency/E14001301,Ilford South,constituency,country/england,,,E14001301,constituency/Ilford South +constituency/E14001302,Ipswich,constituency,country/england,,,E14001302,constituency/Ipswich +constituency/E14001303,Isle of Wight East,constituency,country/england,,,E14001303,constituency/Isle of Wight East +constituency/E14001304,Isle of Wight West,constituency,country/england,,,E14001304,constituency/Isle of Wight West +constituency/E14001305,Islington North,constituency,country/england,,,E14001305,constituency/Islington North +constituency/E14001306,Islington South and Finsbury,constituency,country/england,,,E14001306,constituency/Islington South and Finsbury +constituency/E14001307,Jarrow and Gateshead East,constituency,country/england,,,E14001307,constituency/Jarrow and Gateshead East +constituency/E14001308,Keighley and Ilkley,constituency,country/england,,,E14001308,constituency/Keighley and Ilkley +constituency/E14001309,Kenilworth and Southam,constituency,country/england,,,E14001309,constituency/Kenilworth and Southam +constituency/E14001310,Kensington and Bayswater,constituency,country/england,,,E14001310,constituency/Kensington and Bayswater +constituency/E14001311,Kettering,constituency,country/england,,,E14001311,constituency/Kettering +constituency/E14001312,Kingston and Surbiton,constituency,country/england,,,E14001312,constituency/Kingston and Surbiton +constituency/E14001313,Kingston upon Hull East,constituency,country/england,,,E14001313,constituency/Kingston upon Hull East +constituency/E14001314,Kingston upon Hull North and Cottingham,constituency,country/england,,,E14001314,constituency/Kingston upon Hull North and Cottingham +constituency/E14001315,Kingston upon Hull West and Haltemprice,constituency,country/england,,,E14001315,constituency/Kingston upon Hull West and Haltemprice +constituency/E14001316,Kingswinford and South Staffordshire,constituency,country/england,,,E14001316,constituency/Kingswinford and South Staffordshire +constituency/E14001317,Knowsley,constituency,country/england,,,E14001317,constituency/Knowsley +constituency/E14001318,Lancaster and Wyre,constituency,country/england,,,E14001318,constituency/Lancaster and Wyre +constituency/E14001319,Leeds Central and Headingley,constituency,country/england,,,E14001319,constituency/Leeds Central and Headingley +constituency/E14001320,Leeds East,constituency,country/england,,,E14001320,constituency/Leeds East +constituency/E14001321,Leeds North East,constituency,country/england,,,E14001321,constituency/Leeds North East +constituency/E14001322,Leeds North West,constituency,country/england,,,E14001322,constituency/Leeds North West +constituency/E14001323,Leeds South,constituency,country/england,,,E14001323,constituency/Leeds South +constituency/E14001324,Leeds South West and Morley,constituency,country/england,,,E14001324,constituency/Leeds South West and Morley +constituency/E14001325,Leeds West and Pudsey,constituency,country/england,,,E14001325,constituency/Leeds West and Pudsey +constituency/E14001326,Leicester East,constituency,country/england,,,E14001326,constituency/Leicester East +constituency/E14001327,Leicester South,constituency,country/england,,,E14001327,constituency/Leicester South +constituency/E14001328,Leicester West,constituency,country/england,,,E14001328,constituency/Leicester West +constituency/E14001329,Leigh and Atherton,constituency,country/england,,,E14001329,constituency/Leigh and Atherton +constituency/E14001330,Lewes,constituency,country/england,,,E14001330,constituency/Lewes +constituency/E14001331,Lewisham East,constituency,country/england,,,E14001331,constituency/Lewisham East +constituency/E14001332,Lewisham North,constituency,country/england,,,E14001332,constituency/Lewisham North +constituency/E14001333,Lewisham West and East Dulwich,constituency,country/england,,,E14001333,constituency/Lewisham West and East Dulwich +constituency/E14001334,Leyton and Wanstead,constituency,country/england,,,E14001334,constituency/Leyton and Wanstead +constituency/E14001335,Lichfield,constituency,country/england,,,E14001335,constituency/Lichfield +constituency/E14001336,Lincoln,constituency,country/england,,,E14001336,constituency/Lincoln +constituency/E14001337,Liverpool Garston,constituency,country/england,,,E14001337,constituency/Liverpool Garston +constituency/E14001338,Liverpool Riverside,constituency,country/england,,,E14001338,constituency/Liverpool Riverside +constituency/E14001339,Liverpool Walton,constituency,country/england,,,E14001339,constituency/Liverpool Walton +constituency/E14001340,Liverpool Wavertree,constituency,country/england,,,E14001340,constituency/Liverpool Wavertree +constituency/E14001341,Liverpool West Derby,constituency,country/england,,,E14001341,constituency/Liverpool West Derby +constituency/E14001342,Loughborough,constituency,country/england,,,E14001342,constituency/Loughborough +constituency/E14001343,Louth and Horncastle,constituency,country/england,,,E14001343,constituency/Louth and Horncastle +constituency/E14001344,Lowestoft,constituency,country/england,,,E14001344,constituency/Lowestoft +constituency/E14001345,Luton North,constituency,country/england,,,E14001345,constituency/Luton North +constituency/E14001346,Luton South and South Bedfordshire,constituency,country/england,,,E14001346,constituency/Luton South and South Bedfordshire +constituency/E14001347,Macclesfield,constituency,country/england,,,E14001347,constituency/Macclesfield +constituency/E14001348,Maidenhead,constituency,country/england,,,E14001348,constituency/Maidenhead +constituency/E14001349,Maidstone and Malling,constituency,country/england,,,E14001349,constituency/Maidstone and Malling +constituency/E14001350,Makerfield,constituency,country/england,,,E14001350,constituency/Makerfield +constituency/E14001351,Maldon,constituency,country/england,,,E14001351,constituency/Maldon +constituency/E14001352,Manchester Central,constituency,country/england,,,E14001352,constituency/Manchester Central +constituency/E14001353,Manchester Rusholme,constituency,country/england,,,E14001353,constituency/Manchester Rusholme +constituency/E14001354,Manchester Withington,constituency,country/england,,,E14001354,constituency/Manchester Withington +constituency/E14001355,Mansfield,constituency,country/england,,,E14001355,constituency/Mansfield +constituency/E14001356,Melksham and Devizes,constituency,country/england,,,E14001356,constituency/Melksham and Devizes +constituency/E14001357,Melton and Syston,constituency,country/england,,,E14001357,constituency/Melton and Syston +constituency/E14001358,Meriden and Solihull East,constituency,country/england,,,E14001358,constituency/Meriden and Solihull East +constituency/E14001359,Mid Bedfordshire,constituency,country/england,,,E14001359,constituency/Mid Bedfordshire +constituency/E14001360,Mid Buckinghamshire,constituency,country/england,,,E14001360,constituency/Mid Buckinghamshire +constituency/E14001361,Mid Cheshire,constituency,country/england,,,E14001361,constituency/Mid Cheshire +constituency/E14001362,Mid Derbyshire,constituency,country/england,,,E14001362,constituency/Mid Derbyshire +constituency/E14001363,Mid Dorset and North Poole,constituency,country/england,,,E14001363,constituency/Mid Dorset and North Poole +constituency/E14001364,Mid Leicestershire,constituency,country/england,,,E14001364,constituency/Mid Leicestershire +constituency/E14001365,Mid Norfolk,constituency,country/england,,,E14001365,constituency/Mid Norfolk +constituency/E14001366,Mid Sussex,constituency,country/england,,,E14001366,constituency/Mid Sussex +constituency/E14001367,Middlesbrough and Thornaby East,constituency,country/england,,,E14001367,constituency/Middlesbrough and Thornaby East +constituency/E14001368,Middlesbrough South and East Cleveland,constituency,country/england,,,E14001368,constituency/Middlesbrough South and East Cleveland +constituency/E14001369,Milton Keynes Central,constituency,country/england,,,E14001369,constituency/Milton Keynes Central +constituency/E14001370,Milton Keynes North,constituency,country/england,,,E14001370,constituency/Milton Keynes North +constituency/E14001371,Mitcham and Morden,constituency,country/england,,,E14001371,constituency/Mitcham and Morden +constituency/E14001372,Morecambe and Lunesdale,constituency,country/england,,,E14001372,constituency/Morecambe and Lunesdale +constituency/E14001373,New Forest East,constituency,country/england,,,E14001373,constituency/New Forest East +constituency/E14001374,New Forest West,constituency,country/england,,,E14001374,constituency/New Forest West +constituency/E14001375,Newark,constituency,country/england,,,E14001375,constituency/Newark +constituency/E14001376,Newbury,constituency,country/england,,,E14001376,constituency/Newbury +constituency/E14001377,Newcastle upon Tyne Central and West,constituency,country/england,,,E14001377,constituency/Newcastle upon Tyne Central and West +constituency/E14001378,Newcastle upon Tyne East and Wallsend,constituency,country/england,,,E14001378,constituency/Newcastle upon Tyne East and Wallsend +constituency/E14001379,Newcastle upon Tyne North,constituency,country/england,,,E14001379,constituency/Newcastle upon Tyne North +constituency/E14001380,Newcastle-under-Lyme,constituency,country/england,,,E14001380,constituency/Newcastle-under-Lyme +constituency/E14001381,Newton Abbot,constituency,country/england,,,E14001381,constituency/Newton Abbot +constituency/E14001382,Newton Aycliffe and Spennymoor,constituency,country/england,,,E14001382,constituency/Newton Aycliffe and Spennymoor +constituency/E14001383,Normanton and Hemsworth,constituency,country/england,,,E14001383,constituency/Normanton and Hemsworth +constituency/E14001384,North Bedfordshire,constituency,country/england,,,E14001384,constituency/North Bedfordshire +constituency/E14001385,North Cornwall,constituency,country/england,,,E14001385,constituency/North Cornwall +constituency/E14001386,North Cotswolds,constituency,country/england,,,E14001386,constituency/North Cotswolds +constituency/E14001387,North Devon,constituency,country/england,,,E14001387,constituency/North Devon +constituency/E14001388,North Dorset,constituency,country/england,,,E14001388,constituency/North Dorset +constituency/E14001389,North Durham,constituency,country/england,,,E14001389,constituency/North Durham +constituency/E14001390,North East Cambridgeshire,constituency,country/england,,,E14001390,constituency/North East Cambridgeshire +constituency/E14001391,North East Derbyshire,constituency,country/england,,,E14001391,constituency/North East Derbyshire +constituency/E14001392,North East Hampshire,constituency,country/england,,,E14001392,constituency/North East Hampshire +constituency/E14001393,North East Hertfordshire,constituency,country/england,,,E14001393,constituency/North East Hertfordshire +constituency/E14001394,North East Somerset and Hanham,constituency,country/england,,,E14001394,constituency/North East Somerset and Hanham +constituency/E14001395,North Herefordshire,constituency,country/england,,,E14001395,constituency/North Herefordshire +constituency/E14001396,North Norfolk,constituency,country/england,,,E14001396,constituency/North Norfolk +constituency/E14001397,North Northumberland,constituency,country/england,,,E14001397,constituency/North Northumberland +constituency/E14001398,North Shropshire,constituency,country/england,,,E14001398,constituency/North Shropshire +constituency/E14001399,North Somerset,constituency,country/england,,,E14001399,constituency/North Somerset +constituency/E14001400,North Warwickshire and Bedworth,constituency,country/england,,,E14001400,constituency/North Warwickshire and Bedworth +constituency/E14001401,North West Cambridgeshire,constituency,country/england,,,E14001401,constituency/North West Cambridgeshire +constituency/E14001402,North West Essex,constituency,country/england,,,E14001402,constituency/North West Essex +constituency/E14001403,North West Hampshire,constituency,country/england,,,E14001403,constituency/North West Hampshire +constituency/E14001404,North West Leicestershire,constituency,country/england,,,E14001404,constituency/North West Leicestershire +constituency/E14001405,North West Norfolk,constituency,country/england,,,E14001405,constituency/North West Norfolk +constituency/E14001406,Northampton North,constituency,country/england,,,E14001406,constituency/Northampton North +constituency/E14001407,Northampton South,constituency,country/england,,,E14001407,constituency/Northampton South +constituency/E14001408,Norwich North,constituency,country/england,,,E14001408,constituency/Norwich North +constituency/E14001409,Norwich South,constituency,country/england,,,E14001409,constituency/Norwich South +constituency/E14001410,Nottingham East,constituency,country/england,,,E14001410,constituency/Nottingham East +constituency/E14001411,Nottingham North and Kimberley,constituency,country/england,,,E14001411,constituency/Nottingham North and Kimberley +constituency/E14001412,Nottingham South,constituency,country/england,,,E14001412,constituency/Nottingham South +constituency/E14001413,Nuneaton,constituency,country/england,,,E14001413,constituency/Nuneaton +constituency/E14001414,Old Bexley and Sidcup,constituency,country/england,,,E14001414,constituency/Old Bexley and Sidcup +constituency/E14001415,Oldham East and Saddleworth,constituency,country/england,,,E14001415,constituency/Oldham East and Saddleworth +constituency/E14001416,"Oldham West, Chadderton and Royton",constituency,country/england,,,E14001416,"constituency/Oldham West, Chadderton and Royton" +constituency/E14001417,Orpington,constituency,country/england,,,E14001417,constituency/Orpington +constituency/E14001418,Ossett and Denby Dale,constituency,country/england,,,E14001418,constituency/Ossett and Denby Dale +constituency/E14001419,Oxford East,constituency,country/england,,,E14001419,constituency/Oxford East +constituency/E14001420,Oxford West and Abingdon,constituency,country/england,,,E14001420,constituency/Oxford West and Abingdon +constituency/E14001421,Peckham,constituency,country/england,,,E14001421,constituency/Peckham +constituency/E14001422,Pendle and Clitheroe,constituency,country/england,,,E14001422,constituency/Pendle and Clitheroe +constituency/E14001423,Penistone and Stocksbridge,constituency,country/england,,,E14001423,constituency/Penistone and Stocksbridge +constituency/E14001424,Penrith and Solway,constituency,country/england,,,E14001424,constituency/Penrith and Solway +constituency/E14001425,Peterborough,constituency,country/england,,,E14001425,constituency/Peterborough +constituency/E14001426,Plymouth Moor View,constituency,country/england,,,E14001426,constituency/Plymouth Moor View +constituency/E14001427,Plymouth Sutton and Devonport,constituency,country/england,,,E14001427,constituency/Plymouth Sutton and Devonport +constituency/E14001428,"Pontefract, Castleford and Knottingley",constituency,country/england,,,E14001428,"constituency/Pontefract, Castleford and Knottingley" +constituency/E14001429,Poole,constituency,country/england,,,E14001429,constituency/Poole +constituency/E14001430,Poplar and Limehouse,constituency,country/england,,,E14001430,constituency/Poplar and Limehouse +constituency/E14001431,Portsmouth North,constituency,country/england,,,E14001431,constituency/Portsmouth North +constituency/E14001432,Portsmouth South,constituency,country/england,,,E14001432,constituency/Portsmouth South +constituency/E14001433,Preston,constituency,country/england,,,E14001433,constituency/Preston +constituency/E14001434,Putney,constituency,country/england,,,E14001434,constituency/Putney +constituency/E14001435,Queen's Park and Maida Vale,constituency,country/england,,,E14001435,constituency/Queen's Park and Maida Vale +constituency/E14001436,Rawmarsh and Conisbrough,constituency,country/england,,,E14001436,constituency/Rawmarsh and Conisbrough +constituency/E14001437,Rayleigh and Wickford,constituency,country/england,,,E14001437,constituency/Rayleigh and Wickford +constituency/E14001438,Reading Central,constituency,country/england,,,E14001438,constituency/Reading Central +constituency/E14001439,Reading West and Mid Berkshire,constituency,country/england,,,E14001439,constituency/Reading West and Mid Berkshire +constituency/E14001440,Redcar,constituency,country/england,,,E14001440,constituency/Redcar +constituency/E14001441,Redditch,constituency,country/england,,,E14001441,constituency/Redditch +constituency/E14001442,Reigate,constituency,country/england,,,E14001442,constituency/Reigate +constituency/E14001443,Ribble Valley,constituency,country/england,,,E14001443,constituency/Ribble Valley +constituency/E14001444,Richmond and Northallerton,constituency,country/england,,,E14001444,constituency/Richmond and Northallerton +constituency/E14001445,Richmond Park,constituency,country/england,,,E14001445,constituency/Richmond Park +constituency/E14001446,Rochdale,constituency,country/england,,,E14001446,constituency/Rochdale +constituency/E14001447,Rochester and Strood,constituency,country/england,,,E14001447,constituency/Rochester and Strood +constituency/E14001448,Romford,constituency,country/england,,,E14001448,constituency/Romford +constituency/E14001449,Romsey and Southampton North,constituency,country/england,,,E14001449,constituency/Romsey and Southampton North +constituency/E14001450,Rossendale and Darwen,constituency,country/england,,,E14001450,constituency/Rossendale and Darwen +constituency/E14001451,Rother Valley,constituency,country/england,,,E14001451,constituency/Rother Valley +constituency/E14001452,Rotherham,constituency,country/england,,,E14001452,constituency/Rotherham +constituency/E14001453,Rugby,constituency,country/england,,,E14001453,constituency/Rugby +constituency/E14001454,"Ruislip, Northwood and Pinner",constituency,country/england,,,E14001454,"constituency/Ruislip, Northwood and Pinner" +constituency/E14001455,Runcorn and Helsby,constituency,country/england,,,E14001455,constituency/Runcorn and Helsby +constituency/E14001456,Runnymede and Weybridge,constituency,country/england,,,E14001456,constituency/Runnymede and Weybridge +constituency/E14001457,Rushcliffe,constituency,country/england,,,E14001457,constituency/Rushcliffe +constituency/E14001458,Rutland and Stamford,constituency,country/england,,,E14001458,constituency/Rutland and Stamford +constituency/E14001459,Salford,constituency,country/england,,,E14001459,constituency/Salford +constituency/E14001460,Salisbury,constituency,country/england,,,E14001460,constituency/Salisbury +constituency/E14001461,Scarborough and Whitby,constituency,country/england,,,E14001461,constituency/Scarborough and Whitby +constituency/E14001462,Scunthorpe,constituency,country/england,,,E14001462,constituency/Scunthorpe +constituency/E14001463,Sefton Central,constituency,country/england,,,E14001463,constituency/Sefton Central +constituency/E14001464,Selby,constituency,country/england,,,E14001464,constituency/Selby +constituency/E14001465,Sevenoaks,constituency,country/england,,,E14001465,constituency/Sevenoaks +constituency/E14001466,Sheffield Brightside and Hillsborough,constituency,country/england,,,E14001466,constituency/Sheffield Brightside and Hillsborough +constituency/E14001467,Sheffield Central,constituency,country/england,,,E14001467,constituency/Sheffield Central +constituency/E14001468,Sheffield Hallam,constituency,country/england,,,E14001468,constituency/Sheffield Hallam +constituency/E14001469,Sheffield Heeley,constituency,country/england,,,E14001469,constituency/Sheffield Heeley +constituency/E14001470,Sheffield South East,constituency,country/england,,,E14001470,constituency/Sheffield South East +constituency/E14001471,Sherwood Forest,constituency,country/england,,,E14001471,constituency/Sherwood Forest +constituency/E14001472,Shipley,constituency,country/england,,,E14001472,constituency/Shipley +constituency/E14001473,Shrewsbury,constituency,country/england,,,E14001473,constituency/Shrewsbury +constituency/E14001474,Sittingbourne and Sheppey,constituency,country/england,,,E14001474,constituency/Sittingbourne and Sheppey +constituency/E14001475,Skipton and Ripon,constituency,country/england,,,E14001475,constituency/Skipton and Ripon +constituency/E14001476,Sleaford and North Hykeham,constituency,country/england,,,E14001476,constituency/Sleaford and North Hykeham +constituency/E14001477,Slough,constituency,country/england,,,E14001477,constituency/Slough +constituency/E14001478,Smethwick,constituency,country/england,,,E14001478,constituency/Smethwick +constituency/E14001479,Solihull West and Shirley,constituency,country/england,,,E14001479,constituency/Solihull West and Shirley +constituency/E14001480,South Basildon and East Thurrock,constituency,country/england,,,E14001480,constituency/South Basildon and East Thurrock +constituency/E14001481,South Cambridgeshire,constituency,country/england,,,E14001481,constituency/South Cambridgeshire +constituency/E14001482,South Cotswolds,constituency,country/england,,,E14001482,constituency/South Cotswolds +constituency/E14001483,South Derbyshire,constituency,country/england,,,E14001483,constituency/South Derbyshire +constituency/E14001484,South Devon,constituency,country/england,,,E14001484,constituency/South Devon +constituency/E14001485,South Dorset,constituency,country/england,,,E14001485,constituency/South Dorset +constituency/E14001486,South East Cornwall,constituency,country/england,,,E14001486,constituency/South East Cornwall +constituency/E14001487,South Holland and The Deepings,constituency,country/england,,,E14001487,constituency/South Holland and The Deepings +constituency/E14001488,South Leicestershire,constituency,country/england,,,E14001488,constituency/South Leicestershire +constituency/E14001489,South Norfolk,constituency,country/england,,,E14001489,constituency/South Norfolk +constituency/E14001490,South Northamptonshire,constituency,country/england,,,E14001490,constituency/South Northamptonshire +constituency/E14001491,South Ribble,constituency,country/england,,,E14001491,constituency/South Ribble +constituency/E14001492,South Shields,constituency,country/england,,,E14001492,constituency/South Shields +constituency/E14001493,South Shropshire,constituency,country/england,,,E14001493,constituency/South Shropshire +constituency/E14001494,South Suffolk,constituency,country/england,,,E14001494,constituency/South Suffolk +constituency/E14001495,South West Devon,constituency,country/england,,,E14001495,constituency/South West Devon +constituency/E14001496,South West Hertfordshire,constituency,country/england,,,E14001496,constituency/South West Hertfordshire +constituency/E14001497,South West Norfolk,constituency,country/england,,,E14001497,constituency/South West Norfolk +constituency/E14001498,South West Wiltshire,constituency,country/england,,,E14001498,constituency/South West Wiltshire +constituency/E14001499,Southampton Itchen,constituency,country/england,,,E14001499,constituency/Southampton Itchen +constituency/E14001500,Southampton Test,constituency,country/england,,,E14001500,constituency/Southampton Test +constituency/E14001501,Southend East and Rochford,constituency,country/england,,,E14001501,constituency/Southend East and Rochford +constituency/E14001502,Southend West and Leigh,constituency,country/england,,,E14001502,constituency/Southend West and Leigh +constituency/E14001503,Southgate and Wood Green,constituency,country/england,,,E14001503,constituency/Southgate and Wood Green +constituency/E14001504,Southport,constituency,country/england,,,E14001504,constituency/Southport +constituency/E14001505,Spelthorne,constituency,country/england,,,E14001505,constituency/Spelthorne +constituency/E14001506,Spen Valley,constituency,country/england,,,E14001506,constituency/Spen Valley +constituency/E14001507,St Albans,constituency,country/england,,,E14001507,constituency/St Albans +constituency/E14001508,St Austell and Newquay,constituency,country/england,,,E14001508,constituency/St Austell and Newquay +constituency/E14001509,St Helens North,constituency,country/england,,,E14001509,constituency/St Helens North +constituency/E14001510,St Helens South and Whiston,constituency,country/england,,,E14001510,constituency/St Helens South and Whiston +constituency/E14001511,St Ives,constituency,country/england,,,E14001511,constituency/St Ives +constituency/E14001512,St Neots and Mid Cambridgeshire,constituency,country/england,,,E14001512,constituency/St Neots and Mid Cambridgeshire +constituency/E14001513,Stafford,constituency,country/england,,,E14001513,constituency/Stafford +constituency/E14001514,Staffordshire Moorlands,constituency,country/england,,,E14001514,constituency/Staffordshire Moorlands +constituency/E14001515,Stalybridge and Hyde,constituency,country/england,,,E14001515,constituency/Stalybridge and Hyde +constituency/E14001516,Stevenage,constituency,country/england,,,E14001516,constituency/Stevenage +constituency/E14001517,Stockport,constituency,country/england,,,E14001517,constituency/Stockport +constituency/E14001518,Stockton North,constituency,country/england,,,E14001518,constituency/Stockton North +constituency/E14001519,Stockton West,constituency,country/england,,,E14001519,constituency/Stockton West +constituency/E14001520,Stoke-on-Trent Central,constituency,country/england,,,E14001520,constituency/Stoke-on-Trent Central +constituency/E14001521,Stoke-on-Trent North,constituency,country/england,,,E14001521,constituency/Stoke-on-Trent North +constituency/E14001522,Stoke-on-Trent South,constituency,country/england,,,E14001522,constituency/Stoke-on-Trent South +constituency/E14001523,"Stone, Great Wyrley and Penkridge",constituency,country/england,,,E14001523,"constituency/Stone, Great Wyrley and Penkridge" +constituency/E14001524,Stourbridge,constituency,country/england,,,E14001524,constituency/Stourbridge +constituency/E14001525,Stratford and Bow,constituency,country/england,,,E14001525,constituency/Stratford and Bow +constituency/E14001526,Stratford-on-Avon,constituency,country/england,,,E14001526,constituency/Stratford-on-Avon +constituency/E14001527,Streatham and Croydon North,constituency,country/england,,,E14001527,constituency/Streatham and Croydon North +constituency/E14001528,Stretford and Urmston,constituency,country/england,,,E14001528,constituency/Stretford and Urmston +constituency/E14001529,Stroud,constituency,country/england,,,E14001529,constituency/Stroud +constituency/E14001530,Suffolk Coastal,constituency,country/england,,,E14001530,constituency/Suffolk Coastal +constituency/E14001531,Sunderland Central,constituency,country/england,,,E14001531,constituency/Sunderland Central +constituency/E14001532,Surrey Heath,constituency,country/england,,,E14001532,constituency/Surrey Heath +constituency/E14001533,Sussex Weald,constituency,country/england,,,E14001533,constituency/Sussex Weald +constituency/E14001534,Sutton and Cheam,constituency,country/england,,,E14001534,constituency/Sutton and Cheam +constituency/E14001535,Sutton Coldfield,constituency,country/england,,,E14001535,constituency/Sutton Coldfield +constituency/E14001536,Swindon North,constituency,country/england,,,E14001536,constituency/Swindon North +constituency/E14001537,Swindon South,constituency,country/england,,,E14001537,constituency/Swindon South +constituency/E14001538,Tamworth,constituency,country/england,,,E14001538,constituency/Tamworth +constituency/E14001539,Tatton,constituency,country/england,,,E14001539,constituency/Tatton +constituency/E14001540,Taunton and Wellington,constituency,country/england,,,E14001540,constituency/Taunton and Wellington +constituency/E14001541,Telford,constituency,country/england,,,E14001541,constituency/Telford +constituency/E14001542,Tewkesbury,constituency,country/england,,,E14001542,constituency/Tewkesbury +constituency/E14001543,The Wrekin,constituency,country/england,,,E14001543,constituency/The Wrekin +constituency/E14001544,Thirsk and Malton,constituency,country/england,,,E14001544,constituency/Thirsk and Malton +constituency/E14001545,Thornbury and Yate,constituency,country/england,,,E14001545,constituency/Thornbury and Yate +constituency/E14001546,Thurrock,constituency,country/england,,,E14001546,constituency/Thurrock +constituency/E14001547,Tipton and Wednesbury,constituency,country/england,,,E14001547,constituency/Tipton and Wednesbury +constituency/E14001548,Tiverton and Minehead,constituency,country/england,,,E14001548,constituency/Tiverton and Minehead +constituency/E14001549,Tonbridge,constituency,country/england,,,E14001549,constituency/Tonbridge +constituency/E14001550,Tooting,constituency,country/england,,,E14001550,constituency/Tooting +constituency/E14001551,Torbay,constituency,country/england,,,E14001551,constituency/Torbay +constituency/E14001552,Torridge and Tavistock,constituency,country/england,,,E14001552,constituency/Torridge and Tavistock +constituency/E14001553,Tottenham,constituency,country/england,,,E14001553,constituency/Tottenham +constituency/E14001554,Truro and Falmouth,constituency,country/england,,,E14001554,constituency/Truro and Falmouth +constituency/E14001555,Tunbridge Wells,constituency,country/england,,,E14001555,constituency/Tunbridge Wells +constituency/E14001556,Twickenham,constituency,country/england,,,E14001556,constituency/Twickenham +constituency/E14001557,Tynemouth,constituency,country/england,,,E14001557,constituency/Tynemouth +constituency/E14001558,Uxbridge and South Ruislip,constituency,country/england,,,E14001558,constituency/Uxbridge and South Ruislip +constituency/E14001559,Vauxhall and Camberwell Green,constituency,country/england,,,E14001559,constituency/Vauxhall and Camberwell Green +constituency/E14001560,Wakefield and Rothwell,constituency,country/england,,,E14001560,constituency/Wakefield and Rothwell +constituency/E14001561,Wallasey,constituency,country/england,,,E14001561,constituency/Wallasey +constituency/E14001562,Walsall and Bloxwich,constituency,country/england,,,E14001562,constituency/Walsall and Bloxwich +constituency/E14001563,Walthamstow,constituency,country/england,,,E14001563,constituency/Walthamstow +constituency/E14001564,Warrington North,constituency,country/england,,,E14001564,constituency/Warrington North +constituency/E14001565,Warrington South,constituency,country/england,,,E14001565,constituency/Warrington South +constituency/E14001566,Warwick and Leamington,constituency,country/england,,,E14001566,constituency/Warwick and Leamington +constituency/E14001567,Washington and Gateshead South,constituency,country/england,,,E14001567,constituency/Washington and Gateshead South +constituency/E14001568,Watford,constituency,country/england,,,E14001568,constituency/Watford +constituency/E14001569,Waveney Valley,constituency,country/england,,,E14001569,constituency/Waveney Valley +constituency/E14001570,Weald of Kent,constituency,country/england,,,E14001570,constituency/Weald of Kent +constituency/E14001571,Wellingborough and Rushden,constituency,country/england,,,E14001571,constituency/Wellingborough and Rushden +constituency/E14001572,Wells and Mendip Hills,constituency,country/england,,,E14001572,constituency/Wells and Mendip Hills +constituency/E14001573,Welwyn Hatfield,constituency,country/england,,,E14001573,constituency/Welwyn Hatfield +constituency/E14001574,West Bromwich,constituency,country/england,,,E14001574,constituency/West Bromwich +constituency/E14001575,West Dorset,constituency,country/england,,,E14001575,constituency/West Dorset +constituency/E14001576,West Ham and Beckton,constituency,country/england,,,E14001576,constituency/West Ham and Beckton +constituency/E14001577,West Lancashire,constituency,country/england,,,E14001577,constituency/West Lancashire +constituency/E14001578,West Suffolk,constituency,country/england,,,E14001578,constituency/West Suffolk +constituency/E14001579,West Worcestershire,constituency,country/england,,,E14001579,constituency/West Worcestershire +constituency/E14001580,Westmorland and Lonsdale,constituency,country/england,,,E14001580,constituency/Westmorland and Lonsdale +constituency/E14001581,Weston-super-Mare,constituency,country/england,,,E14001581,constituency/Weston-super-Mare +constituency/E14001582,Wetherby and Easingwold,constituency,country/england,,,E14001582,constituency/Wetherby and Easingwold +constituency/E14001583,Whitehaven and Workington,constituency,country/england,,,E14001583,constituency/Whitehaven and Workington +constituency/E14001584,Widnes and Halewood,constituency,country/england,,,E14001584,constituency/Widnes and Halewood +constituency/E14001585,Wigan,constituency,country/england,,,E14001585,constituency/Wigan +constituency/E14001586,Wimbledon,constituency,country/england,,,E14001586,constituency/Wimbledon +constituency/E14001587,Winchester,constituency,country/england,,,E14001587,constituency/Winchester +constituency/E14001588,Windsor,constituency,country/england,,,E14001588,constituency/Windsor +constituency/E14001589,Wirral West,constituency,country/england,,,E14001589,constituency/Wirral West +constituency/E14001590,Witham,constituency,country/england,,,E14001590,constituency/Witham +constituency/E14001591,Witney,constituency,country/england,,,E14001591,constituency/Witney +constituency/E14001592,Woking,constituency,country/england,,,E14001592,constituency/Woking +constituency/E14001593,Wokingham,constituency,country/england,,,E14001593,constituency/Wokingham +constituency/E14001594,Wolverhampton North East,constituency,country/england,,,E14001594,constituency/Wolverhampton North East +constituency/E14001595,Wolverhampton South East,constituency,country/england,,,E14001595,constituency/Wolverhampton South East +constituency/E14001596,Wolverhampton West,constituency,country/england,,,E14001596,constituency/Wolverhampton West +constituency/E14001597,Worcester,constituency,country/england,,,E14001597,constituency/Worcester +constituency/E14001598,Worsley and Eccles,constituency,country/england,,,E14001598,constituency/Worsley and Eccles +constituency/E14001599,Worthing West,constituency,country/england,,,E14001599,constituency/Worthing West +constituency/E14001600,Wycombe,constituency,country/england,,,E14001600,constituency/Wycombe +constituency/E14001601,Wyre Forest,constituency,country/england,,,E14001601,constituency/Wyre Forest +constituency/E14001602,Wythenshawe and Sale East,constituency,country/england,,,E14001602,constituency/Wythenshawe and Sale East +constituency/E14001603,Yeovil,constituency,country/england,,,E14001603,constituency/Yeovil +constituency/E14001604,York Central,constituency,country/england,,,E14001604,constituency/York Central +constituency/E14001605,York Outer,constituency,country/england,,,E14001605,constituency/York Outer +constituency/N05000001,Belfast East,constituency,country/northern_ireland,,,N05000001,constituency/Belfast East +constituency/N05000002,Belfast North,constituency,country/northern_ireland,,,N05000002,constituency/Belfast North +constituency/N05000003,Belfast South and Mid Down,constituency,country/northern_ireland,,,N05000003,constituency/Belfast South and Mid Down +constituency/N05000004,Belfast West,constituency,country/northern_ireland,,,N05000004,constituency/Belfast West +constituency/N05000005,East Antrim,constituency,country/northern_ireland,,,N05000005,constituency/East Antrim +constituency/N05000006,East Londonderry,constituency,country/northern_ireland,,,N05000006,constituency/East Londonderry +constituency/N05000007,Fermanagh and South Tyrone,constituency,country/northern_ireland,,,N05000007,constituency/Fermanagh and South Tyrone +constituency/N05000008,Foyle,constituency,country/northern_ireland,,,N05000008,constituency/Foyle +constituency/N05000009,Lagan Valley,constituency,country/northern_ireland,,,N05000009,constituency/Lagan Valley +constituency/N05000010,Mid Ulster,constituency,country/northern_ireland,,,N05000010,constituency/Mid Ulster +constituency/N05000011,Newry and Armagh,constituency,country/northern_ireland,,,N05000011,constituency/Newry and Armagh +constituency/N05000012,North Antrim,constituency,country/northern_ireland,,,N05000012,constituency/North Antrim +constituency/N05000013,North Down,constituency,country/northern_ireland,,,N05000013,constituency/North Down +constituency/N05000014,South Antrim,constituency,country/northern_ireland,,,N05000014,constituency/South Antrim +constituency/N05000015,South Down,constituency,country/northern_ireland,,,N05000015,constituency/South Down +constituency/N05000016,Strangford,constituency,country/northern_ireland,,,N05000016,constituency/Strangford +constituency/N05000017,Upper Bann,constituency,country/northern_ireland,,,N05000017,constituency/Upper Bann +constituency/N05000018,West Tyrone,constituency,country/northern_ireland,,,N05000018,constituency/West Tyrone +constituency/S14000021,East Renfrewshire,constituency,country/scotland,,,S14000021,constituency/East Renfrewshire +constituency/S14000027,Na h-Eileanan an Iar,constituency,country/scotland,,,S14000027,constituency/Na h-Eileanan an Iar +constituency/S14000045,Midlothian,constituency,country/scotland,,,S14000045,constituency/Midlothian +constituency/S14000048,North Ayrshire and Arran,constituency,country/scotland,,,S14000048,constituency/North Ayrshire and Arran +constituency/S14000051,Orkney and Shetland,constituency,country/scotland,,,S14000051,constituency/Orkney and Shetland +constituency/S14000060,Aberdeen North,constituency,country/scotland,,,S14000060,constituency/Aberdeen North +constituency/S14000061,Aberdeen South,constituency,country/scotland,,,S14000061,constituency/Aberdeen South +constituency/S14000062,Aberdeenshire North and Moray East,constituency,country/scotland,,,S14000062,constituency/Aberdeenshire North and Moray East +constituency/S14000063,Airdrie and Shotts,constituency,country/scotland,,,S14000063,constituency/Airdrie and Shotts +constituency/S14000064,Alloa and Grangemouth,constituency,country/scotland,,,S14000064,constituency/Alloa and Grangemouth +constituency/S14000065,Angus and Perthshire Glens,constituency,country/scotland,,,S14000065,constituency/Angus and Perthshire Glens +constituency/S14000066,Arbroath and Broughty Ferry,constituency,country/scotland,,,S14000066,constituency/Arbroath and Broughty Ferry +constituency/S14000067,"Argyll, Bute and South Lochaber",constituency,country/scotland,,,S14000067,"constituency/Argyll, Bute and South Lochaber" +constituency/S14000068,Bathgate and Linlithgow,constituency,country/scotland,,,S14000068,constituency/Bathgate and Linlithgow +constituency/S14000069,"Caithness, Sutherland and Easter Ross",constituency,country/scotland,,,S14000069,"constituency/Caithness, Sutherland and Easter Ross" +constituency/S14000070,Coatbridge and Bellshill,constituency,country/scotland,,,S14000070,constituency/Coatbridge and Bellshill +constituency/S14000071,Cowdenbeath and Kirkcaldy,constituency,country/scotland,,,S14000071,constituency/Cowdenbeath and Kirkcaldy +constituency/S14000072,Cumbernauld and Kirkintilloch,constituency,country/scotland,,,S14000072,constituency/Cumbernauld and Kirkintilloch +constituency/S14000073,Dumfries and Galloway,constituency,country/scotland,,,S14000073,constituency/Dumfries and Galloway +constituency/S14000074,"Dumfriesshire, Clydesdale and Tweeddale",constituency,country/scotland,,,S14000074,"constituency/Dumfriesshire, Clydesdale and Tweeddale" +constituency/S14000075,Dundee Central,constituency,country/scotland,,,S14000075,constituency/Dundee Central +constituency/S14000076,Dunfermline and Dollar,constituency,country/scotland,,,S14000076,constituency/Dunfermline and Dollar +constituency/S14000077,East Kilbride and Strathaven,constituency,country/scotland,,,S14000077,constituency/East Kilbride and Strathaven +constituency/S14000078,Edinburgh East and Musselburgh,constituency,country/scotland,,,S14000078,constituency/Edinburgh East and Musselburgh +constituency/S14000079,Edinburgh North and Leith,constituency,country/scotland,,,S14000079,constituency/Edinburgh North and Leith +constituency/S14000080,Edinburgh South,constituency,country/scotland,,,S14000080,constituency/Edinburgh South +constituency/S14000081,Edinburgh South West,constituency,country/scotland,,,S14000081,constituency/Edinburgh South West +constituency/S14000082,Edinburgh West,constituency,country/scotland,,,S14000082,constituency/Edinburgh West +constituency/S14000083,Falkirk,constituency,country/scotland,,,S14000083,constituency/Falkirk +constituency/S14000084,Glasgow East,constituency,country/scotland,,,S14000084,constituency/Glasgow East +constituency/S14000085,Glasgow North,constituency,country/scotland,,,S14000085,constituency/Glasgow North +constituency/S14000086,Glasgow North East,constituency,country/scotland,,,S14000086,constituency/Glasgow North East +constituency/S14000087,Glasgow South,constituency,country/scotland,,,S14000087,constituency/Glasgow South +constituency/S14000088,Glasgow South West,constituency,country/scotland,,,S14000088,constituency/Glasgow South West +constituency/S14000089,Glasgow West,constituency,country/scotland,,,S14000089,constituency/Glasgow West +constituency/S14000090,Glenrothes and Mid Fife,constituency,country/scotland,,,S14000090,constituency/Glenrothes and Mid Fife +constituency/S14000091,Gordon and Buchan,constituency,country/scotland,,,S14000091,constituency/Gordon and Buchan +constituency/S14000092,Hamilton and Clyde Valley,constituency,country/scotland,,,S14000092,constituency/Hamilton and Clyde Valley +constituency/S14000093,Inverclyde and Renfrewshire West,constituency,country/scotland,,,S14000093,constituency/Inverclyde and Renfrewshire West +constituency/S14000094,"Inverness, Skye and West Ross-shire",constituency,country/scotland,,,S14000094,"constituency/Inverness, Skye and West Ross-shire" +constituency/S14000095,Livingston,constituency,country/scotland,,,S14000095,constituency/Livingston +constituency/S14000096,Lothian East,constituency,country/scotland,,,S14000096,constituency/Lothian East +constituency/S14000097,Mid Dunbartonshire,constituency,country/scotland,,,S14000097,constituency/Mid Dunbartonshire +constituency/S14000098,"Moray West, Nairn and Strathspey",constituency,country/scotland,,,S14000098,"constituency/Moray West, Nairn and Strathspey" +constituency/S14000099,"Motherwell, Wishaw and Carluke",constituency,country/scotland,,,S14000099,"constituency/Motherwell, Wishaw and Carluke" +constituency/S14000100,North East Fife,constituency,country/scotland,,,S14000100,constituency/North East Fife +constituency/S14000101,Paisley and Renfrewshire North,constituency,country/scotland,,,S14000101,constituency/Paisley and Renfrewshire North +constituency/S14000102,Paisley and Renfrewshire South,constituency,country/scotland,,,S14000102,constituency/Paisley and Renfrewshire South +constituency/S14000103,Perth and Kinross-shire,constituency,country/scotland,,,S14000103,constituency/Perth and Kinross-shire +constituency/S14000104,Rutherglen,constituency,country/scotland,,,S14000104,constituency/Rutherglen +constituency/S14000105,Stirling and Strathallan,constituency,country/scotland,,,S14000105,constituency/Stirling and Strathallan +constituency/S14000106,West Dunbartonshire,constituency,country/scotland,,,S14000106,constituency/West Dunbartonshire +constituency/S14000107,"Ayr, Carrick and Cumnock",constituency,country/scotland,,,S14000107,"constituency/Ayr, Carrick and Cumnock" +constituency/S14000108,"Berwickshire, Roxburgh and Selkirk",constituency,country/scotland,,,S14000108,"constituency/Berwickshire, Roxburgh and Selkirk" +constituency/S14000109,Central Ayrshire,constituency,country/scotland,,,S14000109,constituency/Central Ayrshire +constituency/S14000110,Kilmarnock and Loudoun,constituency,country/scotland,,,S14000110,constituency/Kilmarnock and Loudoun +constituency/S14000111,West Aberdeenshire and Kincardine,constituency,country/scotland,,,S14000111,constituency/West Aberdeenshire and Kincardine +constituency/W07000081,Aberafan Maesteg,constituency,country/wales,,,W07000081,constituency/Aberafan Maesteg +constituency/W07000082,Alyn and Deeside,constituency,country/wales,,,W07000082,constituency/Alyn and Deeside +constituency/W07000083,Bangor Aberconwy,constituency,country/wales,,,W07000083,constituency/Bangor Aberconwy +constituency/W07000084,Blaenau Gwent and Rhymney,constituency,country/wales,,,W07000084,constituency/Blaenau Gwent and Rhymney +constituency/W07000085,"Brecon, Radnor and Cwm Tawe",constituency,country/wales,,,W07000085,"constituency/Brecon, Radnor and Cwm Tawe" +constituency/W07000086,Bridgend,constituency,country/wales,,,W07000086,constituency/Bridgend +constituency/W07000087,Caerfyrddin,constituency,country/wales,,,W07000087,constituency/Caerfyrddin +constituency/W07000088,Caerphilly,constituency,country/wales,,,W07000088,constituency/Caerphilly +constituency/W07000089,Cardiff East,constituency,country/wales,,,W07000089,constituency/Cardiff East +constituency/W07000090,Cardiff North,constituency,country/wales,,,W07000090,constituency/Cardiff North +constituency/W07000091,Cardiff South and Penarth,constituency,country/wales,,,W07000091,constituency/Cardiff South and Penarth +constituency/W07000092,Cardiff West,constituency,country/wales,,,W07000092,constituency/Cardiff West +constituency/W07000093,Ceredigion Preseli,constituency,country/wales,,,W07000093,constituency/Ceredigion Preseli +constituency/W07000094,Clwyd East,constituency,country/wales,,,W07000094,constituency/Clwyd East +constituency/W07000095,Clwyd North,constituency,country/wales,,,W07000095,constituency/Clwyd North +constituency/W07000096,Dwyfor Meirionnydd,constituency,country/wales,,,W07000096,constituency/Dwyfor Meirionnydd +constituency/W07000097,Gower,constituency,country/wales,,,W07000097,constituency/Gower +constituency/W07000098,Llanelli,constituency,country/wales,,,W07000098,constituency/Llanelli +constituency/W07000099,Merthyr Tydfil and Aberdare,constituency,country/wales,,,W07000099,constituency/Merthyr Tydfil and Aberdare +constituency/W07000100,Mid and South Pembrokeshire,constituency,country/wales,,,W07000100,constituency/Mid and South Pembrokeshire +constituency/W07000101,Monmouthshire,constituency,country/wales,,,W07000101,constituency/Monmouthshire +constituency/W07000102,Montgomeryshire and Glyndwr,constituency,country/wales,,,W07000102,constituency/Montgomeryshire and Glyndwr +constituency/W07000103,Neath and Swansea East,constituency,country/wales,,,W07000103,constituency/Neath and Swansea East +constituency/W07000104,Newport East,constituency,country/wales,,,W07000104,constituency/Newport East +constituency/W07000105,Newport West and Islwyn,constituency,country/wales,,,W07000105,constituency/Newport West and Islwyn +constituency/W07000106,Pontypridd,constituency,country/wales,,,W07000106,constituency/Pontypridd +constituency/W07000107,Rhondda and Ogmore,constituency,country/wales,,,W07000107,constituency/Rhondda and Ogmore +constituency/W07000108,Swansea West,constituency,country/wales,,,W07000108,constituency/Swansea West +constituency/W07000109,Torfaen,constituency,country/wales,,,W07000109,constituency/Torfaen +constituency/W07000110,Vale of Glamorgan,constituency,country/wales,,,W07000110,constituency/Vale of Glamorgan +constituency/W07000111,Wrexham,constituency,country/wales,,,W07000111,constituency/Wrexham +constituency/W07000112,Ynys Môn,constituency,country/wales,,,W07000112,constituency/Ynys Môn +local_authority/E06000001,Hartlepool,local_authority,country/england,,,E06000001,local_authority/Hartlepool +local_authority/E06000002,Middlesbrough,local_authority,country/england,,,E06000002,local_authority/Middlesbrough +local_authority/E06000003,Redcar and Cleveland,local_authority,country/england,,,E06000003,local_authority/Redcar and Cleveland +local_authority/E06000004,Stockton-on-Tees,local_authority,country/england,,,E06000004,local_authority/Stockton-on-Tees +local_authority/E06000005,Darlington,local_authority,country/england,,,E06000005,local_authority/Darlington +local_authority/E06000006,Halton,local_authority,country/england,,,E06000006,local_authority/Halton +local_authority/E06000007,Warrington,local_authority,country/england,,,E06000007,local_authority/Warrington +local_authority/E06000008,Blackburn with Darwen,local_authority,country/england,,,E06000008,local_authority/Blackburn with Darwen +local_authority/E06000009,Blackpool,local_authority,country/england,,,E06000009,local_authority/Blackpool +local_authority/E06000010,"Kingston upon Hull, City of",local_authority,country/england,,,E06000010,"local_authority/Kingston upon Hull, City of" +local_authority/E06000011,East Riding of Yorkshire,local_authority,country/england,,,E06000011,local_authority/East Riding of Yorkshire +local_authority/E06000012,North East Lincolnshire,local_authority,country/england,,,E06000012,local_authority/North East Lincolnshire +local_authority/E06000013,North Lincolnshire,local_authority,country/england,,,E06000013,local_authority/North Lincolnshire +local_authority/E06000014,York,local_authority,country/england,,,E06000014,local_authority/York +local_authority/E06000015,Derby,local_authority,country/england,,,E06000015,local_authority/Derby +local_authority/E06000016,Leicester,local_authority,country/england,,,E06000016,local_authority/Leicester +local_authority/E06000017,Rutland,local_authority,country/england,,,E06000017,local_authority/Rutland +local_authority/E06000018,Nottingham,local_authority,country/england,,,E06000018,local_authority/Nottingham +local_authority/E06000019,"Herefordshire, County of",local_authority,country/england,,,E06000019,"local_authority/Herefordshire, County of" +local_authority/E06000020,Telford and Wrekin,local_authority,country/england,,,E06000020,local_authority/Telford and Wrekin +local_authority/E06000021,Stoke-on-Trent,local_authority,country/england,,,E06000021,local_authority/Stoke-on-Trent +local_authority/E06000022,Bath and North East Somerset,local_authority,country/england,,,E06000022,local_authority/Bath and North East Somerset +local_authority/E06000023,"Bristol, City of",local_authority,country/england,,,E06000023,"local_authority/Bristol, City of" +local_authority/E06000024,North Somerset,local_authority,country/england,,,E06000024,local_authority/North Somerset +local_authority/E06000025,South Gloucestershire,local_authority,country/england,,,E06000025,local_authority/South Gloucestershire +local_authority/E06000026,Plymouth,local_authority,country/england,,,E06000026,local_authority/Plymouth +local_authority/E06000027,Torbay,local_authority,country/england,,,E06000027,local_authority/Torbay +local_authority/E06000030,Swindon,local_authority,country/england,,,E06000030,local_authority/Swindon +local_authority/E06000031,Peterborough,local_authority,country/england,,,E06000031,local_authority/Peterborough +local_authority/E06000032,Luton,local_authority,country/england,,,E06000032,local_authority/Luton +local_authority/E06000033,Southend-on-Sea,local_authority,country/england,,,E06000033,local_authority/Southend-on-Sea +local_authority/E06000034,Thurrock,local_authority,country/england,,,E06000034,local_authority/Thurrock +local_authority/E06000035,Medway,local_authority,country/england,,,E06000035,local_authority/Medway +local_authority/E06000036,Bracknell Forest,local_authority,country/england,,,E06000036,local_authority/Bracknell Forest +local_authority/E06000037,West Berkshire,local_authority,country/england,,,E06000037,local_authority/West Berkshire +local_authority/E06000038,Reading,local_authority,country/england,,,E06000038,local_authority/Reading +local_authority/E06000039,Slough,local_authority,country/england,,,E06000039,local_authority/Slough +local_authority/E06000040,Windsor and Maidenhead,local_authority,country/england,,,E06000040,local_authority/Windsor and Maidenhead +local_authority/E06000041,Wokingham,local_authority,country/england,,,E06000041,local_authority/Wokingham +local_authority/E06000042,Milton Keynes,local_authority,country/england,,,E06000042,local_authority/Milton Keynes +local_authority/E06000043,Brighton and Hove,local_authority,country/england,,,E06000043,local_authority/Brighton and Hove +local_authority/E06000044,Portsmouth,local_authority,country/england,,,E06000044,local_authority/Portsmouth +local_authority/E06000045,Southampton,local_authority,country/england,,,E06000045,local_authority/Southampton +local_authority/E06000046,Isle of Wight,local_authority,country/england,,,E06000046,local_authority/Isle of Wight +local_authority/E06000047,County Durham,local_authority,country/england,,,E06000047,local_authority/County Durham +local_authority/E06000049,Cheshire East,local_authority,country/england,,,E06000049,local_authority/Cheshire East +local_authority/E06000050,Cheshire West and Chester,local_authority,country/england,,,E06000050,local_authority/Cheshire West and Chester +local_authority/E06000051,Shropshire,local_authority,country/england,,,E06000051,local_authority/Shropshire +local_authority/E06000052,Cornwall,local_authority,country/england,,,E06000052,local_authority/Cornwall +local_authority/E06000053,Isles of Scilly,local_authority,country/england,,,E06000053,local_authority/Isles of Scilly +local_authority/E06000054,Wiltshire,local_authority,country/england,,,E06000054,local_authority/Wiltshire +local_authority/E06000055,Bedford,local_authority,country/england,,,E06000055,local_authority/Bedford +local_authority/E06000056,Central Bedfordshire,local_authority,country/england,,,E06000056,local_authority/Central Bedfordshire +local_authority/E06000057,Northumberland,local_authority,country/england,,,E06000057,local_authority/Northumberland +local_authority/E06000058,"Bournemouth, Christchurch and Poole",local_authority,country/england,,,E06000058,"local_authority/Bournemouth, Christchurch and Poole" +local_authority/E06000059,Dorset,local_authority,country/england,,,E06000059,local_authority/Dorset +local_authority/E06000060,Buckinghamshire,local_authority,country/england,,,E06000060,local_authority/Buckinghamshire +local_authority/E06000061,North Northamptonshire,local_authority,country/england,,,E06000061,local_authority/North Northamptonshire +local_authority/E06000062,West Northamptonshire,local_authority,country/england,,,E06000062,local_authority/West Northamptonshire +local_authority/E06000063,Cumberland,local_authority,country/england,,,E06000063,local_authority/Cumberland +local_authority/E06000064,Westmorland and Furness,local_authority,country/england,,,E06000064,local_authority/Westmorland and Furness +local_authority/E06000065,North Yorkshire,local_authority,country/england,,,E06000065,local_authority/North Yorkshire +local_authority/E06000066,Somerset,local_authority,country/england,,,E06000066,local_authority/Somerset +local_authority/E07000008,Cambridge,local_authority,country/england,,,E07000008,local_authority/Cambridge +local_authority/E07000009,East Cambridgeshire,local_authority,country/england,,,E07000009,local_authority/East Cambridgeshire +local_authority/E07000010,Fenland,local_authority,country/england,,,E07000010,local_authority/Fenland +local_authority/E07000011,Huntingdonshire,local_authority,country/england,,,E07000011,local_authority/Huntingdonshire +local_authority/E07000012,South Cambridgeshire,local_authority,country/england,,,E07000012,local_authority/South Cambridgeshire +local_authority/E07000032,Amber Valley,local_authority,country/england,,,E07000032,local_authority/Amber Valley +local_authority/E07000033,Bolsover,local_authority,country/england,,,E07000033,local_authority/Bolsover +local_authority/E07000034,Chesterfield,local_authority,country/england,,,E07000034,local_authority/Chesterfield +local_authority/E07000035,Derbyshire Dales,local_authority,country/england,,,E07000035,local_authority/Derbyshire Dales +local_authority/E07000036,Erewash,local_authority,country/england,,,E07000036,local_authority/Erewash +local_authority/E07000037,High Peak,local_authority,country/england,,,E07000037,local_authority/High Peak +local_authority/E07000038,North East Derbyshire,local_authority,country/england,,,E07000038,local_authority/North East Derbyshire +local_authority/E07000039,South Derbyshire,local_authority,country/england,,,E07000039,local_authority/South Derbyshire +local_authority/E07000040,East Devon,local_authority,country/england,,,E07000040,local_authority/East Devon +local_authority/E07000041,Exeter,local_authority,country/england,,,E07000041,local_authority/Exeter +local_authority/E07000042,Mid Devon,local_authority,country/england,,,E07000042,local_authority/Mid Devon +local_authority/E07000043,North Devon,local_authority,country/england,,,E07000043,local_authority/North Devon +local_authority/E07000044,South Hams,local_authority,country/england,,,E07000044,local_authority/South Hams +local_authority/E07000045,Teignbridge,local_authority,country/england,,,E07000045,local_authority/Teignbridge +local_authority/E07000046,Torridge,local_authority,country/england,,,E07000046,local_authority/Torridge +local_authority/E07000047,West Devon,local_authority,country/england,,,E07000047,local_authority/West Devon +local_authority/E07000061,Eastbourne,local_authority,country/england,,,E07000061,local_authority/Eastbourne +local_authority/E07000062,Hastings,local_authority,country/england,,,E07000062,local_authority/Hastings +local_authority/E07000063,Lewes,local_authority,country/england,,,E07000063,local_authority/Lewes +local_authority/E07000064,Rother,local_authority,country/england,,,E07000064,local_authority/Rother +local_authority/E07000065,Wealden,local_authority,country/england,,,E07000065,local_authority/Wealden +local_authority/E07000066,Basildon,local_authority,country/england,,,E07000066,local_authority/Basildon +local_authority/E07000067,Braintree,local_authority,country/england,,,E07000067,local_authority/Braintree +local_authority/E07000068,Brentwood,local_authority,country/england,,,E07000068,local_authority/Brentwood +local_authority/E07000069,Castle Point,local_authority,country/england,,,E07000069,local_authority/Castle Point +local_authority/E07000070,Chelmsford,local_authority,country/england,,,E07000070,local_authority/Chelmsford +local_authority/E07000071,Colchester,local_authority,country/england,,,E07000071,local_authority/Colchester +local_authority/E07000072,Epping Forest,local_authority,country/england,,,E07000072,local_authority/Epping Forest +local_authority/E07000073,Harlow,local_authority,country/england,,,E07000073,local_authority/Harlow +local_authority/E07000074,Maldon,local_authority,country/england,,,E07000074,local_authority/Maldon +local_authority/E07000075,Rochford,local_authority,country/england,,,E07000075,local_authority/Rochford +local_authority/E07000076,Tendring,local_authority,country/england,,,E07000076,local_authority/Tendring +local_authority/E07000077,Uttlesford,local_authority,country/england,,,E07000077,local_authority/Uttlesford +local_authority/E07000078,Cheltenham,local_authority,country/england,,,E07000078,local_authority/Cheltenham +local_authority/E07000079,Cotswold,local_authority,country/england,,,E07000079,local_authority/Cotswold +local_authority/E07000080,Forest of Dean,local_authority,country/england,,,E07000080,local_authority/Forest of Dean +local_authority/E07000081,Gloucester,local_authority,country/england,,,E07000081,local_authority/Gloucester +local_authority/E07000082,Stroud,local_authority,country/england,,,E07000082,local_authority/Stroud +local_authority/E07000083,Tewkesbury,local_authority,country/england,,,E07000083,local_authority/Tewkesbury +local_authority/E07000084,Basingstoke and Deane,local_authority,country/england,,,E07000084,local_authority/Basingstoke and Deane +local_authority/E07000085,East Hampshire,local_authority,country/england,,,E07000085,local_authority/East Hampshire +local_authority/E07000086,Eastleigh,local_authority,country/england,,,E07000086,local_authority/Eastleigh +local_authority/E07000087,Fareham,local_authority,country/england,,,E07000087,local_authority/Fareham +local_authority/E07000088,Gosport,local_authority,country/england,,,E07000088,local_authority/Gosport +local_authority/E07000089,Hart,local_authority,country/england,,,E07000089,local_authority/Hart +local_authority/E07000090,Havant,local_authority,country/england,,,E07000090,local_authority/Havant +local_authority/E07000091,New Forest,local_authority,country/england,,,E07000091,local_authority/New Forest +local_authority/E07000092,Rushmoor,local_authority,country/england,,,E07000092,local_authority/Rushmoor +local_authority/E07000093,Test Valley,local_authority,country/england,,,E07000093,local_authority/Test Valley +local_authority/E07000094,Winchester,local_authority,country/england,,,E07000094,local_authority/Winchester +local_authority/E07000095,Broxbourne,local_authority,country/england,,,E07000095,local_authority/Broxbourne +local_authority/E07000096,Dacorum,local_authority,country/england,,,E07000096,local_authority/Dacorum +local_authority/E07000098,Hertsmere,local_authority,country/england,,,E07000098,local_authority/Hertsmere +local_authority/E07000099,North Hertfordshire,local_authority,country/england,,,E07000099,local_authority/North Hertfordshire +local_authority/E07000102,Three Rivers,local_authority,country/england,,,E07000102,local_authority/Three Rivers +local_authority/E07000103,Watford,local_authority,country/england,,,E07000103,local_authority/Watford +local_authority/E07000105,Ashford,local_authority,country/england,,,E07000105,local_authority/Ashford +local_authority/E07000106,Canterbury,local_authority,country/england,,,E07000106,local_authority/Canterbury +local_authority/E07000107,Dartford,local_authority,country/england,,,E07000107,local_authority/Dartford +local_authority/E07000108,Dover,local_authority,country/england,,,E07000108,local_authority/Dover +local_authority/E07000109,Gravesham,local_authority,country/england,,,E07000109,local_authority/Gravesham +local_authority/E07000110,Maidstone,local_authority,country/england,,,E07000110,local_authority/Maidstone +local_authority/E07000111,Sevenoaks,local_authority,country/england,,,E07000111,local_authority/Sevenoaks +local_authority/E07000112,Folkestone and Hythe,local_authority,country/england,,,E07000112,local_authority/Folkestone and Hythe +local_authority/E07000113,Swale,local_authority,country/england,,,E07000113,local_authority/Swale +local_authority/E07000114,Thanet,local_authority,country/england,,,E07000114,local_authority/Thanet +local_authority/E07000115,Tonbridge and Malling,local_authority,country/england,,,E07000115,local_authority/Tonbridge and Malling +local_authority/E07000116,Tunbridge Wells,local_authority,country/england,,,E07000116,local_authority/Tunbridge Wells +local_authority/E07000117,Burnley,local_authority,country/england,,,E07000117,local_authority/Burnley +local_authority/E07000118,Chorley,local_authority,country/england,,,E07000118,local_authority/Chorley +local_authority/E07000119,Fylde,local_authority,country/england,,,E07000119,local_authority/Fylde +local_authority/E07000120,Hyndburn,local_authority,country/england,,,E07000120,local_authority/Hyndburn +local_authority/E07000121,Lancaster,local_authority,country/england,,,E07000121,local_authority/Lancaster +local_authority/E07000122,Pendle,local_authority,country/england,,,E07000122,local_authority/Pendle +local_authority/E07000123,Preston,local_authority,country/england,,,E07000123,local_authority/Preston +local_authority/E07000124,Ribble Valley,local_authority,country/england,,,E07000124,local_authority/Ribble Valley +local_authority/E07000125,Rossendale,local_authority,country/england,,,E07000125,local_authority/Rossendale +local_authority/E07000126,South Ribble,local_authority,country/england,,,E07000126,local_authority/South Ribble +local_authority/E07000127,West Lancashire,local_authority,country/england,,,E07000127,local_authority/West Lancashire +local_authority/E07000128,Wyre,local_authority,country/england,,,E07000128,local_authority/Wyre +local_authority/E07000129,Blaby,local_authority,country/england,,,E07000129,local_authority/Blaby +local_authority/E07000130,Charnwood,local_authority,country/england,,,E07000130,local_authority/Charnwood +local_authority/E07000131,Harborough,local_authority,country/england,,,E07000131,local_authority/Harborough +local_authority/E07000132,Hinckley and Bosworth,local_authority,country/england,,,E07000132,local_authority/Hinckley and Bosworth +local_authority/E07000133,Melton,local_authority,country/england,,,E07000133,local_authority/Melton +local_authority/E07000134,North West Leicestershire,local_authority,country/england,,,E07000134,local_authority/North West Leicestershire +local_authority/E07000135,Oadby and Wigston,local_authority,country/england,,,E07000135,local_authority/Oadby and Wigston +local_authority/E07000136,Boston,local_authority,country/england,,,E07000136,local_authority/Boston +local_authority/E07000137,East Lindsey,local_authority,country/england,,,E07000137,local_authority/East Lindsey +local_authority/E07000138,Lincoln,local_authority,country/england,,,E07000138,local_authority/Lincoln +local_authority/E07000139,North Kesteven,local_authority,country/england,,,E07000139,local_authority/North Kesteven +local_authority/E07000140,South Holland,local_authority,country/england,,,E07000140,local_authority/South Holland +local_authority/E07000141,South Kesteven,local_authority,country/england,,,E07000141,local_authority/South Kesteven +local_authority/E07000142,West Lindsey,local_authority,country/england,,,E07000142,local_authority/West Lindsey +local_authority/E07000143,Breckland,local_authority,country/england,,,E07000143,local_authority/Breckland +local_authority/E07000144,Broadland,local_authority,country/england,,,E07000144,local_authority/Broadland +local_authority/E07000145,Great Yarmouth,local_authority,country/england,,,E07000145,local_authority/Great Yarmouth +local_authority/E07000146,King's Lynn and West Norfolk,local_authority,country/england,,,E07000146,local_authority/King's Lynn and West Norfolk +local_authority/E07000147,North Norfolk,local_authority,country/england,,,E07000147,local_authority/North Norfolk +local_authority/E07000148,Norwich,local_authority,country/england,,,E07000148,local_authority/Norwich +local_authority/E07000149,South Norfolk,local_authority,country/england,,,E07000149,local_authority/South Norfolk +local_authority/E07000170,Ashfield,local_authority,country/england,,,E07000170,local_authority/Ashfield +local_authority/E07000171,Bassetlaw,local_authority,country/england,,,E07000171,local_authority/Bassetlaw +local_authority/E07000172,Broxtowe,local_authority,country/england,,,E07000172,local_authority/Broxtowe +local_authority/E07000173,Gedling,local_authority,country/england,,,E07000173,local_authority/Gedling +local_authority/E07000174,Mansfield,local_authority,country/england,,,E07000174,local_authority/Mansfield +local_authority/E07000175,Newark and Sherwood,local_authority,country/england,,,E07000175,local_authority/Newark and Sherwood +local_authority/E07000176,Rushcliffe,local_authority,country/england,,,E07000176,local_authority/Rushcliffe +local_authority/E07000177,Cherwell,local_authority,country/england,,,E07000177,local_authority/Cherwell +local_authority/E07000178,Oxford,local_authority,country/england,,,E07000178,local_authority/Oxford +local_authority/E07000179,South Oxfordshire,local_authority,country/england,,,E07000179,local_authority/South Oxfordshire +local_authority/E07000180,Vale of White Horse,local_authority,country/england,,,E07000180,local_authority/Vale of White Horse +local_authority/E07000181,West Oxfordshire,local_authority,country/england,,,E07000181,local_authority/West Oxfordshire +local_authority/E07000192,Cannock Chase,local_authority,country/england,,,E07000192,local_authority/Cannock Chase +local_authority/E07000193,East Staffordshire,local_authority,country/england,,,E07000193,local_authority/East Staffordshire +local_authority/E07000194,Lichfield,local_authority,country/england,,,E07000194,local_authority/Lichfield +local_authority/E07000195,Newcastle-under-Lyme,local_authority,country/england,,,E07000195,local_authority/Newcastle-under-Lyme +local_authority/E07000196,South Staffordshire,local_authority,country/england,,,E07000196,local_authority/South Staffordshire +local_authority/E07000197,Stafford,local_authority,country/england,,,E07000197,local_authority/Stafford +local_authority/E07000198,Staffordshire Moorlands,local_authority,country/england,,,E07000198,local_authority/Staffordshire Moorlands +local_authority/E07000199,Tamworth,local_authority,country/england,,,E07000199,local_authority/Tamworth +local_authority/E07000200,Babergh,local_authority,country/england,,,E07000200,local_authority/Babergh +local_authority/E07000202,Ipswich,local_authority,country/england,,,E07000202,local_authority/Ipswich +local_authority/E07000203,Mid Suffolk,local_authority,country/england,,,E07000203,local_authority/Mid Suffolk +local_authority/E07000207,Elmbridge,local_authority,country/england,,,E07000207,local_authority/Elmbridge +local_authority/E07000208,Epsom and Ewell,local_authority,country/england,,,E07000208,local_authority/Epsom and Ewell +local_authority/E07000209,Guildford,local_authority,country/england,,,E07000209,local_authority/Guildford +local_authority/E07000210,Mole Valley,local_authority,country/england,,,E07000210,local_authority/Mole Valley +local_authority/E07000211,Reigate and Banstead,local_authority,country/england,,,E07000211,local_authority/Reigate and Banstead +local_authority/E07000212,Runnymede,local_authority,country/england,,,E07000212,local_authority/Runnymede +local_authority/E07000213,Spelthorne,local_authority,country/england,,,E07000213,local_authority/Spelthorne +local_authority/E07000214,Surrey Heath,local_authority,country/england,,,E07000214,local_authority/Surrey Heath +local_authority/E07000215,Tandridge,local_authority,country/england,,,E07000215,local_authority/Tandridge +local_authority/E07000216,Waverley,local_authority,country/england,,,E07000216,local_authority/Waverley +local_authority/E07000217,Woking,local_authority,country/england,,,E07000217,local_authority/Woking +local_authority/E07000218,North Warwickshire,local_authority,country/england,,,E07000218,local_authority/North Warwickshire +local_authority/E07000219,Nuneaton and Bedworth,local_authority,country/england,,,E07000219,local_authority/Nuneaton and Bedworth +local_authority/E07000220,Rugby,local_authority,country/england,,,E07000220,local_authority/Rugby +local_authority/E07000221,Stratford-on-Avon,local_authority,country/england,,,E07000221,local_authority/Stratford-on-Avon +local_authority/E07000222,Warwick,local_authority,country/england,,,E07000222,local_authority/Warwick +local_authority/E07000223,Adur,local_authority,country/england,,,E07000223,local_authority/Adur +local_authority/E07000224,Arun,local_authority,country/england,,,E07000224,local_authority/Arun +local_authority/E07000225,Chichester,local_authority,country/england,,,E07000225,local_authority/Chichester +local_authority/E07000226,Crawley,local_authority,country/england,,,E07000226,local_authority/Crawley +local_authority/E07000227,Horsham,local_authority,country/england,,,E07000227,local_authority/Horsham +local_authority/E07000228,Mid Sussex,local_authority,country/england,,,E07000228,local_authority/Mid Sussex +local_authority/E07000229,Worthing,local_authority,country/england,,,E07000229,local_authority/Worthing +local_authority/E07000234,Bromsgrove,local_authority,country/england,,,E07000234,local_authority/Bromsgrove +local_authority/E07000235,Malvern Hills,local_authority,country/england,,,E07000235,local_authority/Malvern Hills +local_authority/E07000236,Redditch,local_authority,country/england,,,E07000236,local_authority/Redditch +local_authority/E07000237,Worcester,local_authority,country/england,,,E07000237,local_authority/Worcester +local_authority/E07000238,Wychavon,local_authority,country/england,,,E07000238,local_authority/Wychavon +local_authority/E07000239,Wyre Forest,local_authority,country/england,,,E07000239,local_authority/Wyre Forest +local_authority/E07000240,St Albans,local_authority,country/england,,,E07000240,local_authority/St Albans +local_authority/E07000241,Welwyn Hatfield,local_authority,country/england,,,E07000241,local_authority/Welwyn Hatfield +local_authority/E07000242,East Hertfordshire,local_authority,country/england,,,E07000242,local_authority/East Hertfordshire +local_authority/E07000243,Stevenage,local_authority,country/england,,,E07000243,local_authority/Stevenage +local_authority/E07000244,East Suffolk,local_authority,country/england,,,E07000244,local_authority/East Suffolk +local_authority/E07000245,West Suffolk,local_authority,country/england,,,E07000245,local_authority/West Suffolk +local_authority/E08000001,Bolton,local_authority,country/england,,,E08000001,local_authority/Bolton +local_authority/E08000002,Bury,local_authority,country/england,,,E08000002,local_authority/Bury +local_authority/E08000003,Manchester,local_authority,country/england,,,E08000003,local_authority/Manchester +local_authority/E08000004,Oldham,local_authority,country/england,,,E08000004,local_authority/Oldham +local_authority/E08000005,Rochdale,local_authority,country/england,,,E08000005,local_authority/Rochdale +local_authority/E08000006,Salford,local_authority,country/england,,,E08000006,local_authority/Salford +local_authority/E08000007,Stockport,local_authority,country/england,,,E08000007,local_authority/Stockport +local_authority/E08000008,Tameside,local_authority,country/england,,,E08000008,local_authority/Tameside +local_authority/E08000009,Trafford,local_authority,country/england,,,E08000009,local_authority/Trafford +local_authority/E08000010,Wigan,local_authority,country/england,,,E08000010,local_authority/Wigan +local_authority/E08000011,Knowsley,local_authority,country/england,,,E08000011,local_authority/Knowsley +local_authority/E08000012,Liverpool,local_authority,country/england,,,E08000012,local_authority/Liverpool +local_authority/E08000013,St. Helens,local_authority,country/england,,,E08000013,local_authority/St. Helens +local_authority/E08000014,Sefton,local_authority,country/england,,,E08000014,local_authority/Sefton +local_authority/E08000015,Wirral,local_authority,country/england,,,E08000015,local_authority/Wirral +local_authority/E08000016,Barnsley,local_authority,country/england,,,E08000016,local_authority/Barnsley +local_authority/E08000017,Doncaster,local_authority,country/england,,,E08000017,local_authority/Doncaster +local_authority/E08000018,Rotherham,local_authority,country/england,,,E08000018,local_authority/Rotherham +local_authority/E08000019,Sheffield,local_authority,country/england,,,E08000019,local_authority/Sheffield +local_authority/E08000021,Newcastle upon Tyne,local_authority,country/england,,,E08000021,local_authority/Newcastle upon Tyne +local_authority/E08000022,North Tyneside,local_authority,country/england,,,E08000022,local_authority/North Tyneside +local_authority/E08000023,South Tyneside,local_authority,country/england,,,E08000023,local_authority/South Tyneside +local_authority/E08000024,Sunderland,local_authority,country/england,,,E08000024,local_authority/Sunderland +local_authority/E08000025,Birmingham,local_authority,country/england,,,E08000025,local_authority/Birmingham +local_authority/E08000026,Coventry,local_authority,country/england,,,E08000026,local_authority/Coventry +local_authority/E08000027,Dudley,local_authority,country/england,,,E08000027,local_authority/Dudley +local_authority/E08000028,Sandwell,local_authority,country/england,,,E08000028,local_authority/Sandwell +local_authority/E08000029,Solihull,local_authority,country/england,,,E08000029,local_authority/Solihull +local_authority/E08000030,Walsall,local_authority,country/england,,,E08000030,local_authority/Walsall +local_authority/E08000031,Wolverhampton,local_authority,country/england,,,E08000031,local_authority/Wolverhampton +local_authority/E08000032,Bradford,local_authority,country/england,,,E08000032,local_authority/Bradford +local_authority/E08000033,Calderdale,local_authority,country/england,,,E08000033,local_authority/Calderdale +local_authority/E08000034,Kirklees,local_authority,country/england,,,E08000034,local_authority/Kirklees +local_authority/E08000035,Leeds,local_authority,country/england,,,E08000035,local_authority/Leeds +local_authority/E08000036,Wakefield,local_authority,country/england,,,E08000036,local_authority/Wakefield +local_authority/E08000037,Gateshead,local_authority,country/england,,,E08000037,local_authority/Gateshead +local_authority/E09000001,City of London,local_authority,country/england,,,E09000001,local_authority/City of London +local_authority/E09000002,Barking and Dagenham,local_authority,country/england,,,E09000002,local_authority/Barking and Dagenham +local_authority/E09000003,Barnet,local_authority,country/england,,,E09000003,local_authority/Barnet +local_authority/E09000004,Bexley,local_authority,country/england,,,E09000004,local_authority/Bexley +local_authority/E09000005,Brent,local_authority,country/england,,,E09000005,local_authority/Brent +local_authority/E09000006,Bromley,local_authority,country/england,,,E09000006,local_authority/Bromley +local_authority/E09000007,Camden,local_authority,country/england,,,E09000007,local_authority/Camden +local_authority/E09000008,Croydon,local_authority,country/england,,,E09000008,local_authority/Croydon +local_authority/E09000009,Ealing,local_authority,country/england,,,E09000009,local_authority/Ealing +local_authority/E09000010,Enfield,local_authority,country/england,,,E09000010,local_authority/Enfield +local_authority/E09000011,Greenwich,local_authority,country/england,,,E09000011,local_authority/Greenwich +local_authority/E09000012,Hackney,local_authority,country/england,,,E09000012,local_authority/Hackney +local_authority/E09000013,Hammersmith and Fulham,local_authority,country/england,,,E09000013,local_authority/Hammersmith and Fulham +local_authority/E09000014,Haringey,local_authority,country/england,,,E09000014,local_authority/Haringey +local_authority/E09000015,Harrow,local_authority,country/england,,,E09000015,local_authority/Harrow +local_authority/E09000016,Havering,local_authority,country/england,,,E09000016,local_authority/Havering +local_authority/E09000017,Hillingdon,local_authority,country/england,,,E09000017,local_authority/Hillingdon +local_authority/E09000018,Hounslow,local_authority,country/england,,,E09000018,local_authority/Hounslow +local_authority/E09000019,Islington,local_authority,country/england,,,E09000019,local_authority/Islington +local_authority/E09000020,Kensington and Chelsea,local_authority,country/england,,,E09000020,local_authority/Kensington and Chelsea +local_authority/E09000021,Kingston upon Thames,local_authority,country/england,,,E09000021,local_authority/Kingston upon Thames +local_authority/E09000022,Lambeth,local_authority,country/england,,,E09000022,local_authority/Lambeth +local_authority/E09000023,Lewisham,local_authority,country/england,,,E09000023,local_authority/Lewisham +local_authority/E09000024,Merton,local_authority,country/england,,,E09000024,local_authority/Merton +local_authority/E09000025,Newham,local_authority,country/england,,,E09000025,local_authority/Newham +local_authority/E09000026,Redbridge,local_authority,country/england,,,E09000026,local_authority/Redbridge +local_authority/E09000027,Richmond upon Thames,local_authority,country/england,,,E09000027,local_authority/Richmond upon Thames +local_authority/E09000028,Southwark,local_authority,country/england,,,E09000028,local_authority/Southwark +local_authority/E09000029,Sutton,local_authority,country/england,,,E09000029,local_authority/Sutton +local_authority/E09000030,Tower Hamlets,local_authority,country/england,,,E09000030,local_authority/Tower Hamlets +local_authority/E09000031,Waltham Forest,local_authority,country/england,,,E09000031,local_authority/Waltham Forest +local_authority/E09000032,Wandsworth,local_authority,country/england,,,E09000032,local_authority/Wandsworth +local_authority/E09000033,Westminster,local_authority,country/england,,,E09000033,local_authority/Westminster +local_authority/N09000001,Antrim and Newtownabbey,local_authority,country/northern_ireland,,,N09000001,local_authority/Antrim and Newtownabbey +local_authority/N09000002,"Armagh City, Banbridge and Craigavon",local_authority,country/northern_ireland,,,N09000002,"local_authority/Armagh City, Banbridge and Craigavon" +local_authority/N09000003,Belfast,local_authority,country/northern_ireland,,,N09000003,local_authority/Belfast +local_authority/N09000004,Causeway Coast and Glens,local_authority,country/northern_ireland,,,N09000004,local_authority/Causeway Coast and Glens +local_authority/N09000005,Derry City and Strabane,local_authority,country/northern_ireland,,,N09000005,local_authority/Derry City and Strabane +local_authority/N09000006,Fermanagh and Omagh,local_authority,country/northern_ireland,,,N09000006,local_authority/Fermanagh and Omagh +local_authority/N09000007,Lisburn and Castlereagh,local_authority,country/northern_ireland,,,N09000007,local_authority/Lisburn and Castlereagh +local_authority/N09000008,Mid and East Antrim,local_authority,country/northern_ireland,,,N09000008,local_authority/Mid and East Antrim +local_authority/N09000009,Mid Ulster,local_authority,country/northern_ireland,,,N09000009,local_authority/Mid Ulster +local_authority/N09000010,"Newry, Mourne and Down",local_authority,country/northern_ireland,,,N09000010,"local_authority/Newry, Mourne and Down" +local_authority/N09000011,Ards and North Down,local_authority,country/northern_ireland,,,N09000011,local_authority/Ards and North Down +local_authority/S12000005,Clackmannanshire,local_authority,country/scotland,,,S12000005,local_authority/Clackmannanshire +local_authority/S12000006,Dumfries and Galloway,local_authority,country/scotland,,,S12000006,local_authority/Dumfries and Galloway +local_authority/S12000008,East Ayrshire,local_authority,country/scotland,,,S12000008,local_authority/East Ayrshire +local_authority/S12000010,East Lothian,local_authority,country/scotland,,,S12000010,local_authority/East Lothian +local_authority/S12000011,East Renfrewshire,local_authority,country/scotland,,,S12000011,local_authority/East Renfrewshire +local_authority/S12000013,Na h-Eileanan Siar,local_authority,country/scotland,,,S12000013,local_authority/Na h-Eileanan Siar +local_authority/S12000014,Falkirk,local_authority,country/scotland,,,S12000014,local_authority/Falkirk +local_authority/S12000017,Highland,local_authority,country/scotland,,,S12000017,local_authority/Highland +local_authority/S12000018,Inverclyde,local_authority,country/scotland,,,S12000018,local_authority/Inverclyde +local_authority/S12000019,Midlothian,local_authority,country/scotland,,,S12000019,local_authority/Midlothian +local_authority/S12000020,Moray,local_authority,country/scotland,,,S12000020,local_authority/Moray +local_authority/S12000021,North Ayrshire,local_authority,country/scotland,,,S12000021,local_authority/North Ayrshire +local_authority/S12000023,Orkney Islands,local_authority,country/scotland,,,S12000023,local_authority/Orkney Islands +local_authority/S12000026,Scottish Borders,local_authority,country/scotland,,,S12000026,local_authority/Scottish Borders +local_authority/S12000027,Shetland Islands,local_authority,country/scotland,,,S12000027,local_authority/Shetland Islands +local_authority/S12000028,South Ayrshire,local_authority,country/scotland,,,S12000028,local_authority/South Ayrshire +local_authority/S12000029,South Lanarkshire,local_authority,country/scotland,,,S12000029,local_authority/South Lanarkshire +local_authority/S12000030,Stirling,local_authority,country/scotland,,,S12000030,local_authority/Stirling +local_authority/S12000033,Aberdeen City,local_authority,country/scotland,,,S12000033,local_authority/Aberdeen City +local_authority/S12000034,Aberdeenshire,local_authority,country/scotland,,,S12000034,local_authority/Aberdeenshire +local_authority/S12000035,Argyll and Bute,local_authority,country/scotland,,,S12000035,local_authority/Argyll and Bute +local_authority/S12000036,City of Edinburgh,local_authority,country/scotland,,,S12000036,local_authority/City of Edinburgh +local_authority/S12000038,Renfrewshire,local_authority,country/scotland,,,S12000038,local_authority/Renfrewshire +local_authority/S12000039,West Dunbartonshire,local_authority,country/scotland,,,S12000039,local_authority/West Dunbartonshire +local_authority/S12000040,West Lothian,local_authority,country/scotland,,,S12000040,local_authority/West Lothian +local_authority/S12000041,Angus,local_authority,country/scotland,,,S12000041,local_authority/Angus +local_authority/S12000042,Dundee City,local_authority,country/scotland,,,S12000042,local_authority/Dundee City +local_authority/S12000045,East Dunbartonshire,local_authority,country/scotland,,,S12000045,local_authority/East Dunbartonshire +local_authority/S12000047,Fife,local_authority,country/scotland,,,S12000047,local_authority/Fife +local_authority/S12000048,Perth and Kinross,local_authority,country/scotland,,,S12000048,local_authority/Perth and Kinross +local_authority/S12000049,Glasgow City,local_authority,country/scotland,,,S12000049,local_authority/Glasgow City +local_authority/S12000050,North Lanarkshire,local_authority,country/scotland,,,S12000050,local_authority/North Lanarkshire +local_authority/W06000001,Isle of Anglesey,local_authority,country/wales,,,W06000001,local_authority/Isle of Anglesey +local_authority/W06000002,Gwynedd,local_authority,country/wales,,,W06000002,local_authority/Gwynedd +local_authority/W06000003,Conwy,local_authority,country/wales,,,W06000003,local_authority/Conwy +local_authority/W06000004,Denbighshire,local_authority,country/wales,,,W06000004,local_authority/Denbighshire +local_authority/W06000005,Flintshire,local_authority,country/wales,,,W06000005,local_authority/Flintshire +local_authority/W06000006,Wrexham,local_authority,country/wales,,,W06000006,local_authority/Wrexham +local_authority/W06000008,Ceredigion,local_authority,country/wales,,,W06000008,local_authority/Ceredigion +local_authority/W06000009,Pembrokeshire,local_authority,country/wales,,,W06000009,local_authority/Pembrokeshire +local_authority/W06000010,Carmarthenshire,local_authority,country/wales,,,W06000010,local_authority/Carmarthenshire +local_authority/W06000011,Swansea,local_authority,country/wales,,,W06000011,local_authority/Swansea +local_authority/W06000012,Neath Port Talbot,local_authority,country/wales,,,W06000012,local_authority/Neath Port Talbot +local_authority/W06000013,Bridgend,local_authority,country/wales,,,W06000013,local_authority/Bridgend +local_authority/W06000014,Vale of Glamorgan,local_authority,country/wales,,,W06000014,local_authority/Vale of Glamorgan +local_authority/W06000015,Cardiff,local_authority,country/wales,,,W06000015,local_authority/Cardiff +local_authority/W06000016,Rhondda Cynon Taf,local_authority,country/wales,,,W06000016,local_authority/Rhondda Cynon Taf +local_authority/W06000018,Caerphilly,local_authority,country/wales,,,W06000018,local_authority/Caerphilly +local_authority/W06000019,Blaenau Gwent,local_authority,country/wales,,,W06000019,local_authority/Blaenau Gwent +local_authority/W06000020,Torfaen,local_authority,country/wales,,,W06000020,local_authority/Torfaen +local_authority/W06000021,Monmouthshire,local_authority,country/wales,,,W06000021,local_authority/Monmouthshire +local_authority/W06000022,Newport,local_authority,country/wales,,,W06000022,local_authority/Newport +local_authority/W06000023,Powys,local_authority,country/wales,,,W06000023,local_authority/Powys +local_authority/W06000024,Merthyr Tydfil,local_authority,country/wales,,,W06000024,local_authority/Merthyr Tydfil diff --git a/policyengine_api/data/v2/catalog/region_reference_files/us.csv b/policyengine_api/data/v2/catalog/region_reference_files/us.csv new file mode 100644 index 000000000..a8a3b3166 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_reference_files/us.csv @@ -0,0 +1,822 @@ +code,label,region_type,parent_code,state_code,state_name,filter_value,v1_code +us,United States,national,,,,,us +state/ak,Alaska,state,us,AK,Alaska,2,state/ak +state/al,Alabama,state,us,AL,Alabama,1,state/al +state/ar,Arkansas,state,us,AR,Arkansas,5,state/ar +state/az,Arizona,state,us,AZ,Arizona,4,state/az +state/ca,California,state,us,CA,California,6,state/ca +state/co,Colorado,state,us,CO,Colorado,8,state/co +state/ct,Connecticut,state,us,CT,Connecticut,9,state/ct +state/dc,District of Columbia,state,us,DC,District of Columbia,11,state/dc +state/de,Delaware,state,us,DE,Delaware,10,state/de +state/fl,Florida,state,us,FL,Florida,12,state/fl +state/ga,Georgia,state,us,GA,Georgia,13,state/ga +state/hi,Hawaii,state,us,HI,Hawaii,15,state/hi +state/ia,Iowa,state,us,IA,Iowa,19,state/ia +state/id,Idaho,state,us,ID,Idaho,16,state/id +state/il,Illinois,state,us,IL,Illinois,17,state/il +state/in,Indiana,state,us,IN,Indiana,18,state/in +state/ks,Kansas,state,us,KS,Kansas,20,state/ks +state/ky,Kentucky,state,us,KY,Kentucky,21,state/ky +state/la,Louisiana,state,us,LA,Louisiana,22,state/la +state/ma,Massachusetts,state,us,MA,Massachusetts,25,state/ma +state/md,Maryland,state,us,MD,Maryland,24,state/md +state/me,Maine,state,us,ME,Maine,23,state/me +state/mi,Michigan,state,us,MI,Michigan,26,state/mi +state/mn,Minnesota,state,us,MN,Minnesota,27,state/mn +state/mo,Missouri,state,us,MO,Missouri,29,state/mo +state/ms,Mississippi,state,us,MS,Mississippi,28,state/ms +state/mt,Montana,state,us,MT,Montana,30,state/mt +state/nc,North Carolina,state,us,NC,North Carolina,37,state/nc +state/nd,North Dakota,state,us,ND,North Dakota,38,state/nd +state/ne,Nebraska,state,us,NE,Nebraska,31,state/ne +state/nh,New Hampshire,state,us,NH,New Hampshire,33,state/nh +state/nj,New Jersey,state,us,NJ,New Jersey,34,state/nj +state/nm,New Mexico,state,us,NM,New Mexico,35,state/nm +state/nv,Nevada,state,us,NV,Nevada,32,state/nv +state/ny,New York,state,us,NY,New York,36,state/ny +state/oh,Ohio,state,us,OH,Ohio,39,state/oh +state/ok,Oklahoma,state,us,OK,Oklahoma,40,state/ok +state/or,Oregon,state,us,OR,Oregon,41,state/or +state/pa,Pennsylvania,state,us,PA,Pennsylvania,42,state/pa +state/ri,Rhode Island,state,us,RI,Rhode Island,44,state/ri +state/sc,South Carolina,state,us,SC,South Carolina,45,state/sc +state/sd,South Dakota,state,us,SD,South Dakota,46,state/sd +state/tn,Tennessee,state,us,TN,Tennessee,47,state/tn +state/tx,Texas,state,us,TX,Texas,48,state/tx +state/ut,Utah,state,us,UT,Utah,49,state/ut +state/va,Virginia,state,us,VA,Virginia,51,state/va +state/vt,Vermont,state,us,VT,Vermont,50,state/vt +state/wa,Washington,state,us,WA,Washington,53,state/wa +state/wi,Wisconsin,state,us,WI,Wisconsin,55,state/wi +state/wv,West Virginia,state,us,WV,West Virginia,54,state/wv +state/wy,Wyoming,state,us,WY,Wyoming,56,state/wy +congressional_district/AK-01,Alaska's at-large congressional district,congressional_district,state/ak,AK,Alaska,200,congressional_district/AK-01 +congressional_district/AL-01,Alabama's 1st congressional district,congressional_district,state/al,AL,Alabama,101,congressional_district/AL-01 +congressional_district/AL-02,Alabama's 2nd congressional district,congressional_district,state/al,AL,Alabama,102,congressional_district/AL-02 +congressional_district/AL-03,Alabama's 3rd congressional district,congressional_district,state/al,AL,Alabama,103,congressional_district/AL-03 +congressional_district/AL-04,Alabama's 4th congressional district,congressional_district,state/al,AL,Alabama,104,congressional_district/AL-04 +congressional_district/AL-05,Alabama's 5th congressional district,congressional_district,state/al,AL,Alabama,105,congressional_district/AL-05 +congressional_district/AL-06,Alabama's 6th congressional district,congressional_district,state/al,AL,Alabama,106,congressional_district/AL-06 +congressional_district/AL-07,Alabama's 7th congressional district,congressional_district,state/al,AL,Alabama,107,congressional_district/AL-07 +congressional_district/AR-01,Arkansas's 1st congressional district,congressional_district,state/ar,AR,Arkansas,501,congressional_district/AR-01 +congressional_district/AR-02,Arkansas's 2nd congressional district,congressional_district,state/ar,AR,Arkansas,502,congressional_district/AR-02 +congressional_district/AR-03,Arkansas's 3rd congressional district,congressional_district,state/ar,AR,Arkansas,503,congressional_district/AR-03 +congressional_district/AR-04,Arkansas's 4th congressional district,congressional_district,state/ar,AR,Arkansas,504,congressional_district/AR-04 +congressional_district/AZ-01,Arizona's 1st congressional district,congressional_district,state/az,AZ,Arizona,401,congressional_district/AZ-01 +congressional_district/AZ-02,Arizona's 2nd congressional district,congressional_district,state/az,AZ,Arizona,402,congressional_district/AZ-02 +congressional_district/AZ-03,Arizona's 3rd congressional district,congressional_district,state/az,AZ,Arizona,403,congressional_district/AZ-03 +congressional_district/AZ-04,Arizona's 4th congressional district,congressional_district,state/az,AZ,Arizona,404,congressional_district/AZ-04 +congressional_district/AZ-05,Arizona's 5th congressional district,congressional_district,state/az,AZ,Arizona,405,congressional_district/AZ-05 +congressional_district/AZ-06,Arizona's 6th congressional district,congressional_district,state/az,AZ,Arizona,406,congressional_district/AZ-06 +congressional_district/AZ-07,Arizona's 7th congressional district,congressional_district,state/az,AZ,Arizona,407,congressional_district/AZ-07 +congressional_district/AZ-08,Arizona's 8th congressional district,congressional_district,state/az,AZ,Arizona,408,congressional_district/AZ-08 +congressional_district/AZ-09,Arizona's 9th congressional district,congressional_district,state/az,AZ,Arizona,409,congressional_district/AZ-09 +congressional_district/CA-01,California's 1st congressional district,congressional_district,state/ca,CA,California,601,congressional_district/CA-01 +congressional_district/CA-02,California's 2nd congressional district,congressional_district,state/ca,CA,California,602,congressional_district/CA-02 +congressional_district/CA-03,California's 3rd congressional district,congressional_district,state/ca,CA,California,603,congressional_district/CA-03 +congressional_district/CA-04,California's 4th congressional district,congressional_district,state/ca,CA,California,604,congressional_district/CA-04 +congressional_district/CA-05,California's 5th congressional district,congressional_district,state/ca,CA,California,605,congressional_district/CA-05 +congressional_district/CA-06,California's 6th congressional district,congressional_district,state/ca,CA,California,606,congressional_district/CA-06 +congressional_district/CA-07,California's 7th congressional district,congressional_district,state/ca,CA,California,607,congressional_district/CA-07 +congressional_district/CA-08,California's 8th congressional district,congressional_district,state/ca,CA,California,608,congressional_district/CA-08 +congressional_district/CA-09,California's 9th congressional district,congressional_district,state/ca,CA,California,609,congressional_district/CA-09 +congressional_district/CA-10,California's 10th congressional district,congressional_district,state/ca,CA,California,610,congressional_district/CA-10 +congressional_district/CA-11,California's 11th congressional district,congressional_district,state/ca,CA,California,611,congressional_district/CA-11 +congressional_district/CA-12,California's 12th congressional district,congressional_district,state/ca,CA,California,612,congressional_district/CA-12 +congressional_district/CA-13,California's 13th congressional district,congressional_district,state/ca,CA,California,613,congressional_district/CA-13 +congressional_district/CA-14,California's 14th congressional district,congressional_district,state/ca,CA,California,614,congressional_district/CA-14 +congressional_district/CA-15,California's 15th congressional district,congressional_district,state/ca,CA,California,615,congressional_district/CA-15 +congressional_district/CA-16,California's 16th congressional district,congressional_district,state/ca,CA,California,616,congressional_district/CA-16 +congressional_district/CA-17,California's 17th congressional district,congressional_district,state/ca,CA,California,617,congressional_district/CA-17 +congressional_district/CA-18,California's 18th congressional district,congressional_district,state/ca,CA,California,618,congressional_district/CA-18 +congressional_district/CA-19,California's 19th congressional district,congressional_district,state/ca,CA,California,619,congressional_district/CA-19 +congressional_district/CA-20,California's 20th congressional district,congressional_district,state/ca,CA,California,620,congressional_district/CA-20 +congressional_district/CA-21,California's 21st congressional district,congressional_district,state/ca,CA,California,621,congressional_district/CA-21 +congressional_district/CA-22,California's 22nd congressional district,congressional_district,state/ca,CA,California,622,congressional_district/CA-22 +congressional_district/CA-23,California's 23rd congressional district,congressional_district,state/ca,CA,California,623,congressional_district/CA-23 +congressional_district/CA-24,California's 24th congressional district,congressional_district,state/ca,CA,California,624,congressional_district/CA-24 +congressional_district/CA-25,California's 25th congressional district,congressional_district,state/ca,CA,California,625,congressional_district/CA-25 +congressional_district/CA-26,California's 26th congressional district,congressional_district,state/ca,CA,California,626,congressional_district/CA-26 +congressional_district/CA-27,California's 27th congressional district,congressional_district,state/ca,CA,California,627,congressional_district/CA-27 +congressional_district/CA-28,California's 28th congressional district,congressional_district,state/ca,CA,California,628,congressional_district/CA-28 +congressional_district/CA-29,California's 29th congressional district,congressional_district,state/ca,CA,California,629,congressional_district/CA-29 +congressional_district/CA-30,California's 30th congressional district,congressional_district,state/ca,CA,California,630,congressional_district/CA-30 +congressional_district/CA-31,California's 31st congressional district,congressional_district,state/ca,CA,California,631,congressional_district/CA-31 +congressional_district/CA-32,California's 32nd congressional district,congressional_district,state/ca,CA,California,632,congressional_district/CA-32 +congressional_district/CA-33,California's 33rd congressional district,congressional_district,state/ca,CA,California,633,congressional_district/CA-33 +congressional_district/CA-34,California's 34th congressional district,congressional_district,state/ca,CA,California,634,congressional_district/CA-34 +congressional_district/CA-35,California's 35th congressional district,congressional_district,state/ca,CA,California,635,congressional_district/CA-35 +congressional_district/CA-36,California's 36th congressional district,congressional_district,state/ca,CA,California,636,congressional_district/CA-36 +congressional_district/CA-37,California's 37th congressional district,congressional_district,state/ca,CA,California,637,congressional_district/CA-37 +congressional_district/CA-38,California's 38th congressional district,congressional_district,state/ca,CA,California,638,congressional_district/CA-38 +congressional_district/CA-39,California's 39th congressional district,congressional_district,state/ca,CA,California,639,congressional_district/CA-39 +congressional_district/CA-40,California's 40th congressional district,congressional_district,state/ca,CA,California,640,congressional_district/CA-40 +congressional_district/CA-41,California's 41st congressional district,congressional_district,state/ca,CA,California,641,congressional_district/CA-41 +congressional_district/CA-42,California's 42nd congressional district,congressional_district,state/ca,CA,California,642,congressional_district/CA-42 +congressional_district/CA-43,California's 43rd congressional district,congressional_district,state/ca,CA,California,643,congressional_district/CA-43 +congressional_district/CA-44,California's 44th congressional district,congressional_district,state/ca,CA,California,644,congressional_district/CA-44 +congressional_district/CA-45,California's 45th congressional district,congressional_district,state/ca,CA,California,645,congressional_district/CA-45 +congressional_district/CA-46,California's 46th congressional district,congressional_district,state/ca,CA,California,646,congressional_district/CA-46 +congressional_district/CA-47,California's 47th congressional district,congressional_district,state/ca,CA,California,647,congressional_district/CA-47 +congressional_district/CA-48,California's 48th congressional district,congressional_district,state/ca,CA,California,648,congressional_district/CA-48 +congressional_district/CA-49,California's 49th congressional district,congressional_district,state/ca,CA,California,649,congressional_district/CA-49 +congressional_district/CA-50,California's 50th congressional district,congressional_district,state/ca,CA,California,650,congressional_district/CA-50 +congressional_district/CA-51,California's 51st congressional district,congressional_district,state/ca,CA,California,651,congressional_district/CA-51 +congressional_district/CA-52,California's 52nd congressional district,congressional_district,state/ca,CA,California,652,congressional_district/CA-52 +congressional_district/CO-01,Colorado's 1st congressional district,congressional_district,state/co,CO,Colorado,801,congressional_district/CO-01 +congressional_district/CO-02,Colorado's 2nd congressional district,congressional_district,state/co,CO,Colorado,802,congressional_district/CO-02 +congressional_district/CO-03,Colorado's 3rd congressional district,congressional_district,state/co,CO,Colorado,803,congressional_district/CO-03 +congressional_district/CO-04,Colorado's 4th congressional district,congressional_district,state/co,CO,Colorado,804,congressional_district/CO-04 +congressional_district/CO-05,Colorado's 5th congressional district,congressional_district,state/co,CO,Colorado,805,congressional_district/CO-05 +congressional_district/CO-06,Colorado's 6th congressional district,congressional_district,state/co,CO,Colorado,806,congressional_district/CO-06 +congressional_district/CO-07,Colorado's 7th congressional district,congressional_district,state/co,CO,Colorado,807,congressional_district/CO-07 +congressional_district/CO-08,Colorado's 8th congressional district,congressional_district,state/co,CO,Colorado,808,congressional_district/CO-08 +congressional_district/CT-01,Connecticut's 1st congressional district,congressional_district,state/ct,CT,Connecticut,901,congressional_district/CT-01 +congressional_district/CT-02,Connecticut's 2nd congressional district,congressional_district,state/ct,CT,Connecticut,902,congressional_district/CT-02 +congressional_district/CT-03,Connecticut's 3rd congressional district,congressional_district,state/ct,CT,Connecticut,903,congressional_district/CT-03 +congressional_district/CT-04,Connecticut's 4th congressional district,congressional_district,state/ct,CT,Connecticut,904,congressional_district/CT-04 +congressional_district/CT-05,Connecticut's 5th congressional district,congressional_district,state/ct,CT,Connecticut,905,congressional_district/CT-05 +congressional_district/DC-01,District of Columbia's at-large congressional district,congressional_district,state/dc,DC,District of Columbia,1100,congressional_district/DC-01 +congressional_district/DE-01,Delaware's at-large congressional district,congressional_district,state/de,DE,Delaware,1000,congressional_district/DE-01 +congressional_district/FL-01,Florida's 1st congressional district,congressional_district,state/fl,FL,Florida,1201,congressional_district/FL-01 +congressional_district/FL-02,Florida's 2nd congressional district,congressional_district,state/fl,FL,Florida,1202,congressional_district/FL-02 +congressional_district/FL-03,Florida's 3rd congressional district,congressional_district,state/fl,FL,Florida,1203,congressional_district/FL-03 +congressional_district/FL-04,Florida's 4th congressional district,congressional_district,state/fl,FL,Florida,1204,congressional_district/FL-04 +congressional_district/FL-05,Florida's 5th congressional district,congressional_district,state/fl,FL,Florida,1205,congressional_district/FL-05 +congressional_district/FL-06,Florida's 6th congressional district,congressional_district,state/fl,FL,Florida,1206,congressional_district/FL-06 +congressional_district/FL-07,Florida's 7th congressional district,congressional_district,state/fl,FL,Florida,1207,congressional_district/FL-07 +congressional_district/FL-08,Florida's 8th congressional district,congressional_district,state/fl,FL,Florida,1208,congressional_district/FL-08 +congressional_district/FL-09,Florida's 9th congressional district,congressional_district,state/fl,FL,Florida,1209,congressional_district/FL-09 +congressional_district/FL-10,Florida's 10th congressional district,congressional_district,state/fl,FL,Florida,1210,congressional_district/FL-10 +congressional_district/FL-11,Florida's 11th congressional district,congressional_district,state/fl,FL,Florida,1211,congressional_district/FL-11 +congressional_district/FL-12,Florida's 12th congressional district,congressional_district,state/fl,FL,Florida,1212,congressional_district/FL-12 +congressional_district/FL-13,Florida's 13th congressional district,congressional_district,state/fl,FL,Florida,1213,congressional_district/FL-13 +congressional_district/FL-14,Florida's 14th congressional district,congressional_district,state/fl,FL,Florida,1214,congressional_district/FL-14 +congressional_district/FL-15,Florida's 15th congressional district,congressional_district,state/fl,FL,Florida,1215,congressional_district/FL-15 +congressional_district/FL-16,Florida's 16th congressional district,congressional_district,state/fl,FL,Florida,1216,congressional_district/FL-16 +congressional_district/FL-17,Florida's 17th congressional district,congressional_district,state/fl,FL,Florida,1217,congressional_district/FL-17 +congressional_district/FL-18,Florida's 18th congressional district,congressional_district,state/fl,FL,Florida,1218,congressional_district/FL-18 +congressional_district/FL-19,Florida's 19th congressional district,congressional_district,state/fl,FL,Florida,1219,congressional_district/FL-19 +congressional_district/FL-20,Florida's 20th congressional district,congressional_district,state/fl,FL,Florida,1220,congressional_district/FL-20 +congressional_district/FL-21,Florida's 21st congressional district,congressional_district,state/fl,FL,Florida,1221,congressional_district/FL-21 +congressional_district/FL-22,Florida's 22nd congressional district,congressional_district,state/fl,FL,Florida,1222,congressional_district/FL-22 +congressional_district/FL-23,Florida's 23rd congressional district,congressional_district,state/fl,FL,Florida,1223,congressional_district/FL-23 +congressional_district/FL-24,Florida's 24th congressional district,congressional_district,state/fl,FL,Florida,1224,congressional_district/FL-24 +congressional_district/FL-25,Florida's 25th congressional district,congressional_district,state/fl,FL,Florida,1225,congressional_district/FL-25 +congressional_district/FL-26,Florida's 26th congressional district,congressional_district,state/fl,FL,Florida,1226,congressional_district/FL-26 +congressional_district/FL-27,Florida's 27th congressional district,congressional_district,state/fl,FL,Florida,1227,congressional_district/FL-27 +congressional_district/FL-28,Florida's 28th congressional district,congressional_district,state/fl,FL,Florida,1228,congressional_district/FL-28 +congressional_district/GA-01,Georgia's 1st congressional district,congressional_district,state/ga,GA,Georgia,1301,congressional_district/GA-01 +congressional_district/GA-02,Georgia's 2nd congressional district,congressional_district,state/ga,GA,Georgia,1302,congressional_district/GA-02 +congressional_district/GA-03,Georgia's 3rd congressional district,congressional_district,state/ga,GA,Georgia,1303,congressional_district/GA-03 +congressional_district/GA-04,Georgia's 4th congressional district,congressional_district,state/ga,GA,Georgia,1304,congressional_district/GA-04 +congressional_district/GA-05,Georgia's 5th congressional district,congressional_district,state/ga,GA,Georgia,1305,congressional_district/GA-05 +congressional_district/GA-06,Georgia's 6th congressional district,congressional_district,state/ga,GA,Georgia,1306,congressional_district/GA-06 +congressional_district/GA-07,Georgia's 7th congressional district,congressional_district,state/ga,GA,Georgia,1307,congressional_district/GA-07 +congressional_district/GA-08,Georgia's 8th congressional district,congressional_district,state/ga,GA,Georgia,1308,congressional_district/GA-08 +congressional_district/GA-09,Georgia's 9th congressional district,congressional_district,state/ga,GA,Georgia,1309,congressional_district/GA-09 +congressional_district/GA-10,Georgia's 10th congressional district,congressional_district,state/ga,GA,Georgia,1310,congressional_district/GA-10 +congressional_district/GA-11,Georgia's 11th congressional district,congressional_district,state/ga,GA,Georgia,1311,congressional_district/GA-11 +congressional_district/GA-12,Georgia's 12th congressional district,congressional_district,state/ga,GA,Georgia,1312,congressional_district/GA-12 +congressional_district/GA-13,Georgia's 13th congressional district,congressional_district,state/ga,GA,Georgia,1313,congressional_district/GA-13 +congressional_district/GA-14,Georgia's 14th congressional district,congressional_district,state/ga,GA,Georgia,1314,congressional_district/GA-14 +congressional_district/HI-01,Hawaii's 1st congressional district,congressional_district,state/hi,HI,Hawaii,1501,congressional_district/HI-01 +congressional_district/HI-02,Hawaii's 2nd congressional district,congressional_district,state/hi,HI,Hawaii,1502,congressional_district/HI-02 +congressional_district/IA-01,Iowa's 1st congressional district,congressional_district,state/ia,IA,Iowa,1901,congressional_district/IA-01 +congressional_district/IA-02,Iowa's 2nd congressional district,congressional_district,state/ia,IA,Iowa,1902,congressional_district/IA-02 +congressional_district/IA-03,Iowa's 3rd congressional district,congressional_district,state/ia,IA,Iowa,1903,congressional_district/IA-03 +congressional_district/IA-04,Iowa's 4th congressional district,congressional_district,state/ia,IA,Iowa,1904,congressional_district/IA-04 +congressional_district/ID-01,Idaho's 1st congressional district,congressional_district,state/id,ID,Idaho,1601,congressional_district/ID-01 +congressional_district/ID-02,Idaho's 2nd congressional district,congressional_district,state/id,ID,Idaho,1602,congressional_district/ID-02 +congressional_district/IL-01,Illinois's 1st congressional district,congressional_district,state/il,IL,Illinois,1701,congressional_district/IL-01 +congressional_district/IL-02,Illinois's 2nd congressional district,congressional_district,state/il,IL,Illinois,1702,congressional_district/IL-02 +congressional_district/IL-03,Illinois's 3rd congressional district,congressional_district,state/il,IL,Illinois,1703,congressional_district/IL-03 +congressional_district/IL-04,Illinois's 4th congressional district,congressional_district,state/il,IL,Illinois,1704,congressional_district/IL-04 +congressional_district/IL-05,Illinois's 5th congressional district,congressional_district,state/il,IL,Illinois,1705,congressional_district/IL-05 +congressional_district/IL-06,Illinois's 6th congressional district,congressional_district,state/il,IL,Illinois,1706,congressional_district/IL-06 +congressional_district/IL-07,Illinois's 7th congressional district,congressional_district,state/il,IL,Illinois,1707,congressional_district/IL-07 +congressional_district/IL-08,Illinois's 8th congressional district,congressional_district,state/il,IL,Illinois,1708,congressional_district/IL-08 +congressional_district/IL-09,Illinois's 9th congressional district,congressional_district,state/il,IL,Illinois,1709,congressional_district/IL-09 +congressional_district/IL-10,Illinois's 10th congressional district,congressional_district,state/il,IL,Illinois,1710,congressional_district/IL-10 +congressional_district/IL-11,Illinois's 11th congressional district,congressional_district,state/il,IL,Illinois,1711,congressional_district/IL-11 +congressional_district/IL-12,Illinois's 12th congressional district,congressional_district,state/il,IL,Illinois,1712,congressional_district/IL-12 +congressional_district/IL-13,Illinois's 13th congressional district,congressional_district,state/il,IL,Illinois,1713,congressional_district/IL-13 +congressional_district/IL-14,Illinois's 14th congressional district,congressional_district,state/il,IL,Illinois,1714,congressional_district/IL-14 +congressional_district/IL-15,Illinois's 15th congressional district,congressional_district,state/il,IL,Illinois,1715,congressional_district/IL-15 +congressional_district/IL-16,Illinois's 16th congressional district,congressional_district,state/il,IL,Illinois,1716,congressional_district/IL-16 +congressional_district/IL-17,Illinois's 17th congressional district,congressional_district,state/il,IL,Illinois,1717,congressional_district/IL-17 +congressional_district/IN-01,Indiana's 1st congressional district,congressional_district,state/in,IN,Indiana,1801,congressional_district/IN-01 +congressional_district/IN-02,Indiana's 2nd congressional district,congressional_district,state/in,IN,Indiana,1802,congressional_district/IN-02 +congressional_district/IN-03,Indiana's 3rd congressional district,congressional_district,state/in,IN,Indiana,1803,congressional_district/IN-03 +congressional_district/IN-04,Indiana's 4th congressional district,congressional_district,state/in,IN,Indiana,1804,congressional_district/IN-04 +congressional_district/IN-05,Indiana's 5th congressional district,congressional_district,state/in,IN,Indiana,1805,congressional_district/IN-05 +congressional_district/IN-06,Indiana's 6th congressional district,congressional_district,state/in,IN,Indiana,1806,congressional_district/IN-06 +congressional_district/IN-07,Indiana's 7th congressional district,congressional_district,state/in,IN,Indiana,1807,congressional_district/IN-07 +congressional_district/IN-08,Indiana's 8th congressional district,congressional_district,state/in,IN,Indiana,1808,congressional_district/IN-08 +congressional_district/IN-09,Indiana's 9th congressional district,congressional_district,state/in,IN,Indiana,1809,congressional_district/IN-09 +congressional_district/KS-01,Kansas's 1st congressional district,congressional_district,state/ks,KS,Kansas,2001,congressional_district/KS-01 +congressional_district/KS-02,Kansas's 2nd congressional district,congressional_district,state/ks,KS,Kansas,2002,congressional_district/KS-02 +congressional_district/KS-03,Kansas's 3rd congressional district,congressional_district,state/ks,KS,Kansas,2003,congressional_district/KS-03 +congressional_district/KS-04,Kansas's 4th congressional district,congressional_district,state/ks,KS,Kansas,2004,congressional_district/KS-04 +congressional_district/KY-01,Kentucky's 1st congressional district,congressional_district,state/ky,KY,Kentucky,2101,congressional_district/KY-01 +congressional_district/KY-02,Kentucky's 2nd congressional district,congressional_district,state/ky,KY,Kentucky,2102,congressional_district/KY-02 +congressional_district/KY-03,Kentucky's 3rd congressional district,congressional_district,state/ky,KY,Kentucky,2103,congressional_district/KY-03 +congressional_district/KY-04,Kentucky's 4th congressional district,congressional_district,state/ky,KY,Kentucky,2104,congressional_district/KY-04 +congressional_district/KY-05,Kentucky's 5th congressional district,congressional_district,state/ky,KY,Kentucky,2105,congressional_district/KY-05 +congressional_district/KY-06,Kentucky's 6th congressional district,congressional_district,state/ky,KY,Kentucky,2106,congressional_district/KY-06 +congressional_district/LA-01,Louisiana's 1st congressional district,congressional_district,state/la,LA,Louisiana,2201,congressional_district/LA-01 +congressional_district/LA-02,Louisiana's 2nd congressional district,congressional_district,state/la,LA,Louisiana,2202,congressional_district/LA-02 +congressional_district/LA-03,Louisiana's 3rd congressional district,congressional_district,state/la,LA,Louisiana,2203,congressional_district/LA-03 +congressional_district/LA-04,Louisiana's 4th congressional district,congressional_district,state/la,LA,Louisiana,2204,congressional_district/LA-04 +congressional_district/LA-05,Louisiana's 5th congressional district,congressional_district,state/la,LA,Louisiana,2205,congressional_district/LA-05 +congressional_district/LA-06,Louisiana's 6th congressional district,congressional_district,state/la,LA,Louisiana,2206,congressional_district/LA-06 +congressional_district/MA-01,Massachusetts's 1st congressional district,congressional_district,state/ma,MA,Massachusetts,2501,congressional_district/MA-01 +congressional_district/MA-02,Massachusetts's 2nd congressional district,congressional_district,state/ma,MA,Massachusetts,2502,congressional_district/MA-02 +congressional_district/MA-03,Massachusetts's 3rd congressional district,congressional_district,state/ma,MA,Massachusetts,2503,congressional_district/MA-03 +congressional_district/MA-04,Massachusetts's 4th congressional district,congressional_district,state/ma,MA,Massachusetts,2504,congressional_district/MA-04 +congressional_district/MA-05,Massachusetts's 5th congressional district,congressional_district,state/ma,MA,Massachusetts,2505,congressional_district/MA-05 +congressional_district/MA-06,Massachusetts's 6th congressional district,congressional_district,state/ma,MA,Massachusetts,2506,congressional_district/MA-06 +congressional_district/MA-07,Massachusetts's 7th congressional district,congressional_district,state/ma,MA,Massachusetts,2507,congressional_district/MA-07 +congressional_district/MA-08,Massachusetts's 8th congressional district,congressional_district,state/ma,MA,Massachusetts,2508,congressional_district/MA-08 +congressional_district/MA-09,Massachusetts's 9th congressional district,congressional_district,state/ma,MA,Massachusetts,2509,congressional_district/MA-09 +congressional_district/MD-01,Maryland's 1st congressional district,congressional_district,state/md,MD,Maryland,2401,congressional_district/MD-01 +congressional_district/MD-02,Maryland's 2nd congressional district,congressional_district,state/md,MD,Maryland,2402,congressional_district/MD-02 +congressional_district/MD-03,Maryland's 3rd congressional district,congressional_district,state/md,MD,Maryland,2403,congressional_district/MD-03 +congressional_district/MD-04,Maryland's 4th congressional district,congressional_district,state/md,MD,Maryland,2404,congressional_district/MD-04 +congressional_district/MD-05,Maryland's 5th congressional district,congressional_district,state/md,MD,Maryland,2405,congressional_district/MD-05 +congressional_district/MD-06,Maryland's 6th congressional district,congressional_district,state/md,MD,Maryland,2406,congressional_district/MD-06 +congressional_district/MD-07,Maryland's 7th congressional district,congressional_district,state/md,MD,Maryland,2407,congressional_district/MD-07 +congressional_district/MD-08,Maryland's 8th congressional district,congressional_district,state/md,MD,Maryland,2408,congressional_district/MD-08 +congressional_district/ME-01,Maine's 1st congressional district,congressional_district,state/me,ME,Maine,2301,congressional_district/ME-01 +congressional_district/ME-02,Maine's 2nd congressional district,congressional_district,state/me,ME,Maine,2302,congressional_district/ME-02 +congressional_district/MI-01,Michigan's 1st congressional district,congressional_district,state/mi,MI,Michigan,2601,congressional_district/MI-01 +congressional_district/MI-02,Michigan's 2nd congressional district,congressional_district,state/mi,MI,Michigan,2602,congressional_district/MI-02 +congressional_district/MI-03,Michigan's 3rd congressional district,congressional_district,state/mi,MI,Michigan,2603,congressional_district/MI-03 +congressional_district/MI-04,Michigan's 4th congressional district,congressional_district,state/mi,MI,Michigan,2604,congressional_district/MI-04 +congressional_district/MI-05,Michigan's 5th congressional district,congressional_district,state/mi,MI,Michigan,2605,congressional_district/MI-05 +congressional_district/MI-06,Michigan's 6th congressional district,congressional_district,state/mi,MI,Michigan,2606,congressional_district/MI-06 +congressional_district/MI-07,Michigan's 7th congressional district,congressional_district,state/mi,MI,Michigan,2607,congressional_district/MI-07 +congressional_district/MI-08,Michigan's 8th congressional district,congressional_district,state/mi,MI,Michigan,2608,congressional_district/MI-08 +congressional_district/MI-09,Michigan's 9th congressional district,congressional_district,state/mi,MI,Michigan,2609,congressional_district/MI-09 +congressional_district/MI-10,Michigan's 10th congressional district,congressional_district,state/mi,MI,Michigan,2610,congressional_district/MI-10 +congressional_district/MI-11,Michigan's 11th congressional district,congressional_district,state/mi,MI,Michigan,2611,congressional_district/MI-11 +congressional_district/MI-12,Michigan's 12th congressional district,congressional_district,state/mi,MI,Michigan,2612,congressional_district/MI-12 +congressional_district/MI-13,Michigan's 13th congressional district,congressional_district,state/mi,MI,Michigan,2613,congressional_district/MI-13 +congressional_district/MN-01,Minnesota's 1st congressional district,congressional_district,state/mn,MN,Minnesota,2701,congressional_district/MN-01 +congressional_district/MN-02,Minnesota's 2nd congressional district,congressional_district,state/mn,MN,Minnesota,2702,congressional_district/MN-02 +congressional_district/MN-03,Minnesota's 3rd congressional district,congressional_district,state/mn,MN,Minnesota,2703,congressional_district/MN-03 +congressional_district/MN-04,Minnesota's 4th congressional district,congressional_district,state/mn,MN,Minnesota,2704,congressional_district/MN-04 +congressional_district/MN-05,Minnesota's 5th congressional district,congressional_district,state/mn,MN,Minnesota,2705,congressional_district/MN-05 +congressional_district/MN-06,Minnesota's 6th congressional district,congressional_district,state/mn,MN,Minnesota,2706,congressional_district/MN-06 +congressional_district/MN-07,Minnesota's 7th congressional district,congressional_district,state/mn,MN,Minnesota,2707,congressional_district/MN-07 +congressional_district/MN-08,Minnesota's 8th congressional district,congressional_district,state/mn,MN,Minnesota,2708,congressional_district/MN-08 +congressional_district/MO-01,Missouri's 1st congressional district,congressional_district,state/mo,MO,Missouri,2901,congressional_district/MO-01 +congressional_district/MO-02,Missouri's 2nd congressional district,congressional_district,state/mo,MO,Missouri,2902,congressional_district/MO-02 +congressional_district/MO-03,Missouri's 3rd congressional district,congressional_district,state/mo,MO,Missouri,2903,congressional_district/MO-03 +congressional_district/MO-04,Missouri's 4th congressional district,congressional_district,state/mo,MO,Missouri,2904,congressional_district/MO-04 +congressional_district/MO-05,Missouri's 5th congressional district,congressional_district,state/mo,MO,Missouri,2905,congressional_district/MO-05 +congressional_district/MO-06,Missouri's 6th congressional district,congressional_district,state/mo,MO,Missouri,2906,congressional_district/MO-06 +congressional_district/MO-07,Missouri's 7th congressional district,congressional_district,state/mo,MO,Missouri,2907,congressional_district/MO-07 +congressional_district/MO-08,Missouri's 8th congressional district,congressional_district,state/mo,MO,Missouri,2908,congressional_district/MO-08 +congressional_district/MS-01,Mississippi's 1st congressional district,congressional_district,state/ms,MS,Mississippi,2801,congressional_district/MS-01 +congressional_district/MS-02,Mississippi's 2nd congressional district,congressional_district,state/ms,MS,Mississippi,2802,congressional_district/MS-02 +congressional_district/MS-03,Mississippi's 3rd congressional district,congressional_district,state/ms,MS,Mississippi,2803,congressional_district/MS-03 +congressional_district/MS-04,Mississippi's 4th congressional district,congressional_district,state/ms,MS,Mississippi,2804,congressional_district/MS-04 +congressional_district/MT-01,Montana's 1st congressional district,congressional_district,state/mt,MT,Montana,3001,congressional_district/MT-01 +congressional_district/MT-02,Montana's 2nd congressional district,congressional_district,state/mt,MT,Montana,3002,congressional_district/MT-02 +congressional_district/NC-01,North Carolina's 1st congressional district,congressional_district,state/nc,NC,North Carolina,3701,congressional_district/NC-01 +congressional_district/NC-02,North Carolina's 2nd congressional district,congressional_district,state/nc,NC,North Carolina,3702,congressional_district/NC-02 +congressional_district/NC-03,North Carolina's 3rd congressional district,congressional_district,state/nc,NC,North Carolina,3703,congressional_district/NC-03 +congressional_district/NC-04,North Carolina's 4th congressional district,congressional_district,state/nc,NC,North Carolina,3704,congressional_district/NC-04 +congressional_district/NC-05,North Carolina's 5th congressional district,congressional_district,state/nc,NC,North Carolina,3705,congressional_district/NC-05 +congressional_district/NC-06,North Carolina's 6th congressional district,congressional_district,state/nc,NC,North Carolina,3706,congressional_district/NC-06 +congressional_district/NC-07,North Carolina's 7th congressional district,congressional_district,state/nc,NC,North Carolina,3707,congressional_district/NC-07 +congressional_district/NC-08,North Carolina's 8th congressional district,congressional_district,state/nc,NC,North Carolina,3708,congressional_district/NC-08 +congressional_district/NC-09,North Carolina's 9th congressional district,congressional_district,state/nc,NC,North Carolina,3709,congressional_district/NC-09 +congressional_district/NC-10,North Carolina's 10th congressional district,congressional_district,state/nc,NC,North Carolina,3710,congressional_district/NC-10 +congressional_district/NC-11,North Carolina's 11th congressional district,congressional_district,state/nc,NC,North Carolina,3711,congressional_district/NC-11 +congressional_district/NC-12,North Carolina's 12th congressional district,congressional_district,state/nc,NC,North Carolina,3712,congressional_district/NC-12 +congressional_district/NC-13,North Carolina's 13th congressional district,congressional_district,state/nc,NC,North Carolina,3713,congressional_district/NC-13 +congressional_district/NC-14,North Carolina's 14th congressional district,congressional_district,state/nc,NC,North Carolina,3714,congressional_district/NC-14 +congressional_district/ND-01,North Dakota's at-large congressional district,congressional_district,state/nd,ND,North Dakota,3800,congressional_district/ND-01 +congressional_district/NE-01,Nebraska's 1st congressional district,congressional_district,state/ne,NE,Nebraska,3101,congressional_district/NE-01 +congressional_district/NE-02,Nebraska's 2nd congressional district,congressional_district,state/ne,NE,Nebraska,3102,congressional_district/NE-02 +congressional_district/NE-03,Nebraska's 3rd congressional district,congressional_district,state/ne,NE,Nebraska,3103,congressional_district/NE-03 +congressional_district/NH-01,New Hampshire's 1st congressional district,congressional_district,state/nh,NH,New Hampshire,3301,congressional_district/NH-01 +congressional_district/NH-02,New Hampshire's 2nd congressional district,congressional_district,state/nh,NH,New Hampshire,3302,congressional_district/NH-02 +congressional_district/NJ-01,New Jersey's 1st congressional district,congressional_district,state/nj,NJ,New Jersey,3401,congressional_district/NJ-01 +congressional_district/NJ-02,New Jersey's 2nd congressional district,congressional_district,state/nj,NJ,New Jersey,3402,congressional_district/NJ-02 +congressional_district/NJ-03,New Jersey's 3rd congressional district,congressional_district,state/nj,NJ,New Jersey,3403,congressional_district/NJ-03 +congressional_district/NJ-04,New Jersey's 4th congressional district,congressional_district,state/nj,NJ,New Jersey,3404,congressional_district/NJ-04 +congressional_district/NJ-05,New Jersey's 5th congressional district,congressional_district,state/nj,NJ,New Jersey,3405,congressional_district/NJ-05 +congressional_district/NJ-06,New Jersey's 6th congressional district,congressional_district,state/nj,NJ,New Jersey,3406,congressional_district/NJ-06 +congressional_district/NJ-07,New Jersey's 7th congressional district,congressional_district,state/nj,NJ,New Jersey,3407,congressional_district/NJ-07 +congressional_district/NJ-08,New Jersey's 8th congressional district,congressional_district,state/nj,NJ,New Jersey,3408,congressional_district/NJ-08 +congressional_district/NJ-09,New Jersey's 9th congressional district,congressional_district,state/nj,NJ,New Jersey,3409,congressional_district/NJ-09 +congressional_district/NJ-10,New Jersey's 10th congressional district,congressional_district,state/nj,NJ,New Jersey,3410,congressional_district/NJ-10 +congressional_district/NJ-11,New Jersey's 11th congressional district,congressional_district,state/nj,NJ,New Jersey,3411,congressional_district/NJ-11 +congressional_district/NJ-12,New Jersey's 12th congressional district,congressional_district,state/nj,NJ,New Jersey,3412,congressional_district/NJ-12 +congressional_district/NM-01,New Mexico's 1st congressional district,congressional_district,state/nm,NM,New Mexico,3501,congressional_district/NM-01 +congressional_district/NM-02,New Mexico's 2nd congressional district,congressional_district,state/nm,NM,New Mexico,3502,congressional_district/NM-02 +congressional_district/NM-03,New Mexico's 3rd congressional district,congressional_district,state/nm,NM,New Mexico,3503,congressional_district/NM-03 +congressional_district/NV-01,Nevada's 1st congressional district,congressional_district,state/nv,NV,Nevada,3201,congressional_district/NV-01 +congressional_district/NV-02,Nevada's 2nd congressional district,congressional_district,state/nv,NV,Nevada,3202,congressional_district/NV-02 +congressional_district/NV-03,Nevada's 3rd congressional district,congressional_district,state/nv,NV,Nevada,3203,congressional_district/NV-03 +congressional_district/NV-04,Nevada's 4th congressional district,congressional_district,state/nv,NV,Nevada,3204,congressional_district/NV-04 +congressional_district/NY-01,New York's 1st congressional district,congressional_district,state/ny,NY,New York,3601,congressional_district/NY-01 +congressional_district/NY-02,New York's 2nd congressional district,congressional_district,state/ny,NY,New York,3602,congressional_district/NY-02 +congressional_district/NY-03,New York's 3rd congressional district,congressional_district,state/ny,NY,New York,3603,congressional_district/NY-03 +congressional_district/NY-04,New York's 4th congressional district,congressional_district,state/ny,NY,New York,3604,congressional_district/NY-04 +congressional_district/NY-05,New York's 5th congressional district,congressional_district,state/ny,NY,New York,3605,congressional_district/NY-05 +congressional_district/NY-06,New York's 6th congressional district,congressional_district,state/ny,NY,New York,3606,congressional_district/NY-06 +congressional_district/NY-07,New York's 7th congressional district,congressional_district,state/ny,NY,New York,3607,congressional_district/NY-07 +congressional_district/NY-08,New York's 8th congressional district,congressional_district,state/ny,NY,New York,3608,congressional_district/NY-08 +congressional_district/NY-09,New York's 9th congressional district,congressional_district,state/ny,NY,New York,3609,congressional_district/NY-09 +congressional_district/NY-10,New York's 10th congressional district,congressional_district,state/ny,NY,New York,3610,congressional_district/NY-10 +congressional_district/NY-11,New York's 11th congressional district,congressional_district,state/ny,NY,New York,3611,congressional_district/NY-11 +congressional_district/NY-12,New York's 12th congressional district,congressional_district,state/ny,NY,New York,3612,congressional_district/NY-12 +congressional_district/NY-13,New York's 13th congressional district,congressional_district,state/ny,NY,New York,3613,congressional_district/NY-13 +congressional_district/NY-14,New York's 14th congressional district,congressional_district,state/ny,NY,New York,3614,congressional_district/NY-14 +congressional_district/NY-15,New York's 15th congressional district,congressional_district,state/ny,NY,New York,3615,congressional_district/NY-15 +congressional_district/NY-16,New York's 16th congressional district,congressional_district,state/ny,NY,New York,3616,congressional_district/NY-16 +congressional_district/NY-17,New York's 17th congressional district,congressional_district,state/ny,NY,New York,3617,congressional_district/NY-17 +congressional_district/NY-18,New York's 18th congressional district,congressional_district,state/ny,NY,New York,3618,congressional_district/NY-18 +congressional_district/NY-19,New York's 19th congressional district,congressional_district,state/ny,NY,New York,3619,congressional_district/NY-19 +congressional_district/NY-20,New York's 20th congressional district,congressional_district,state/ny,NY,New York,3620,congressional_district/NY-20 +congressional_district/NY-21,New York's 21st congressional district,congressional_district,state/ny,NY,New York,3621,congressional_district/NY-21 +congressional_district/NY-22,New York's 22nd congressional district,congressional_district,state/ny,NY,New York,3622,congressional_district/NY-22 +congressional_district/NY-23,New York's 23rd congressional district,congressional_district,state/ny,NY,New York,3623,congressional_district/NY-23 +congressional_district/NY-24,New York's 24th congressional district,congressional_district,state/ny,NY,New York,3624,congressional_district/NY-24 +congressional_district/NY-25,New York's 25th congressional district,congressional_district,state/ny,NY,New York,3625,congressional_district/NY-25 +congressional_district/NY-26,New York's 26th congressional district,congressional_district,state/ny,NY,New York,3626,congressional_district/NY-26 +congressional_district/OH-01,Ohio's 1st congressional district,congressional_district,state/oh,OH,Ohio,3901,congressional_district/OH-01 +congressional_district/OH-02,Ohio's 2nd congressional district,congressional_district,state/oh,OH,Ohio,3902,congressional_district/OH-02 +congressional_district/OH-03,Ohio's 3rd congressional district,congressional_district,state/oh,OH,Ohio,3903,congressional_district/OH-03 +congressional_district/OH-04,Ohio's 4th congressional district,congressional_district,state/oh,OH,Ohio,3904,congressional_district/OH-04 +congressional_district/OH-05,Ohio's 5th congressional district,congressional_district,state/oh,OH,Ohio,3905,congressional_district/OH-05 +congressional_district/OH-06,Ohio's 6th congressional district,congressional_district,state/oh,OH,Ohio,3906,congressional_district/OH-06 +congressional_district/OH-07,Ohio's 7th congressional district,congressional_district,state/oh,OH,Ohio,3907,congressional_district/OH-07 +congressional_district/OH-08,Ohio's 8th congressional district,congressional_district,state/oh,OH,Ohio,3908,congressional_district/OH-08 +congressional_district/OH-09,Ohio's 9th congressional district,congressional_district,state/oh,OH,Ohio,3909,congressional_district/OH-09 +congressional_district/OH-10,Ohio's 10th congressional district,congressional_district,state/oh,OH,Ohio,3910,congressional_district/OH-10 +congressional_district/OH-11,Ohio's 11th congressional district,congressional_district,state/oh,OH,Ohio,3911,congressional_district/OH-11 +congressional_district/OH-12,Ohio's 12th congressional district,congressional_district,state/oh,OH,Ohio,3912,congressional_district/OH-12 +congressional_district/OH-13,Ohio's 13th congressional district,congressional_district,state/oh,OH,Ohio,3913,congressional_district/OH-13 +congressional_district/OH-14,Ohio's 14th congressional district,congressional_district,state/oh,OH,Ohio,3914,congressional_district/OH-14 +congressional_district/OH-15,Ohio's 15th congressional district,congressional_district,state/oh,OH,Ohio,3915,congressional_district/OH-15 +congressional_district/OK-01,Oklahoma's 1st congressional district,congressional_district,state/ok,OK,Oklahoma,4001,congressional_district/OK-01 +congressional_district/OK-02,Oklahoma's 2nd congressional district,congressional_district,state/ok,OK,Oklahoma,4002,congressional_district/OK-02 +congressional_district/OK-03,Oklahoma's 3rd congressional district,congressional_district,state/ok,OK,Oklahoma,4003,congressional_district/OK-03 +congressional_district/OK-04,Oklahoma's 4th congressional district,congressional_district,state/ok,OK,Oklahoma,4004,congressional_district/OK-04 +congressional_district/OK-05,Oklahoma's 5th congressional district,congressional_district,state/ok,OK,Oklahoma,4005,congressional_district/OK-05 +congressional_district/OR-01,Oregon's 1st congressional district,congressional_district,state/or,OR,Oregon,4101,congressional_district/OR-01 +congressional_district/OR-02,Oregon's 2nd congressional district,congressional_district,state/or,OR,Oregon,4102,congressional_district/OR-02 +congressional_district/OR-03,Oregon's 3rd congressional district,congressional_district,state/or,OR,Oregon,4103,congressional_district/OR-03 +congressional_district/OR-04,Oregon's 4th congressional district,congressional_district,state/or,OR,Oregon,4104,congressional_district/OR-04 +congressional_district/OR-05,Oregon's 5th congressional district,congressional_district,state/or,OR,Oregon,4105,congressional_district/OR-05 +congressional_district/OR-06,Oregon's 6th congressional district,congressional_district,state/or,OR,Oregon,4106,congressional_district/OR-06 +congressional_district/PA-01,Pennsylvania's 1st congressional district,congressional_district,state/pa,PA,Pennsylvania,4201,congressional_district/PA-01 +congressional_district/PA-02,Pennsylvania's 2nd congressional district,congressional_district,state/pa,PA,Pennsylvania,4202,congressional_district/PA-02 +congressional_district/PA-03,Pennsylvania's 3rd congressional district,congressional_district,state/pa,PA,Pennsylvania,4203,congressional_district/PA-03 +congressional_district/PA-04,Pennsylvania's 4th congressional district,congressional_district,state/pa,PA,Pennsylvania,4204,congressional_district/PA-04 +congressional_district/PA-05,Pennsylvania's 5th congressional district,congressional_district,state/pa,PA,Pennsylvania,4205,congressional_district/PA-05 +congressional_district/PA-06,Pennsylvania's 6th congressional district,congressional_district,state/pa,PA,Pennsylvania,4206,congressional_district/PA-06 +congressional_district/PA-07,Pennsylvania's 7th congressional district,congressional_district,state/pa,PA,Pennsylvania,4207,congressional_district/PA-07 +congressional_district/PA-08,Pennsylvania's 8th congressional district,congressional_district,state/pa,PA,Pennsylvania,4208,congressional_district/PA-08 +congressional_district/PA-09,Pennsylvania's 9th congressional district,congressional_district,state/pa,PA,Pennsylvania,4209,congressional_district/PA-09 +congressional_district/PA-10,Pennsylvania's 10th congressional district,congressional_district,state/pa,PA,Pennsylvania,4210,congressional_district/PA-10 +congressional_district/PA-11,Pennsylvania's 11th congressional district,congressional_district,state/pa,PA,Pennsylvania,4211,congressional_district/PA-11 +congressional_district/PA-12,Pennsylvania's 12th congressional district,congressional_district,state/pa,PA,Pennsylvania,4212,congressional_district/PA-12 +congressional_district/PA-13,Pennsylvania's 13th congressional district,congressional_district,state/pa,PA,Pennsylvania,4213,congressional_district/PA-13 +congressional_district/PA-14,Pennsylvania's 14th congressional district,congressional_district,state/pa,PA,Pennsylvania,4214,congressional_district/PA-14 +congressional_district/PA-15,Pennsylvania's 15th congressional district,congressional_district,state/pa,PA,Pennsylvania,4215,congressional_district/PA-15 +congressional_district/PA-16,Pennsylvania's 16th congressional district,congressional_district,state/pa,PA,Pennsylvania,4216,congressional_district/PA-16 +congressional_district/PA-17,Pennsylvania's 17th congressional district,congressional_district,state/pa,PA,Pennsylvania,4217,congressional_district/PA-17 +congressional_district/RI-01,Rhode Island's 1st congressional district,congressional_district,state/ri,RI,Rhode Island,4401,congressional_district/RI-01 +congressional_district/RI-02,Rhode Island's 2nd congressional district,congressional_district,state/ri,RI,Rhode Island,4402,congressional_district/RI-02 +congressional_district/SC-01,South Carolina's 1st congressional district,congressional_district,state/sc,SC,South Carolina,4501,congressional_district/SC-01 +congressional_district/SC-02,South Carolina's 2nd congressional district,congressional_district,state/sc,SC,South Carolina,4502,congressional_district/SC-02 +congressional_district/SC-03,South Carolina's 3rd congressional district,congressional_district,state/sc,SC,South Carolina,4503,congressional_district/SC-03 +congressional_district/SC-04,South Carolina's 4th congressional district,congressional_district,state/sc,SC,South Carolina,4504,congressional_district/SC-04 +congressional_district/SC-05,South Carolina's 5th congressional district,congressional_district,state/sc,SC,South Carolina,4505,congressional_district/SC-05 +congressional_district/SC-06,South Carolina's 6th congressional district,congressional_district,state/sc,SC,South Carolina,4506,congressional_district/SC-06 +congressional_district/SC-07,South Carolina's 7th congressional district,congressional_district,state/sc,SC,South Carolina,4507,congressional_district/SC-07 +congressional_district/SD-01,South Dakota's at-large congressional district,congressional_district,state/sd,SD,South Dakota,4600,congressional_district/SD-01 +congressional_district/TN-01,Tennessee's 1st congressional district,congressional_district,state/tn,TN,Tennessee,4701,congressional_district/TN-01 +congressional_district/TN-02,Tennessee's 2nd congressional district,congressional_district,state/tn,TN,Tennessee,4702,congressional_district/TN-02 +congressional_district/TN-03,Tennessee's 3rd congressional district,congressional_district,state/tn,TN,Tennessee,4703,congressional_district/TN-03 +congressional_district/TN-04,Tennessee's 4th congressional district,congressional_district,state/tn,TN,Tennessee,4704,congressional_district/TN-04 +congressional_district/TN-05,Tennessee's 5th congressional district,congressional_district,state/tn,TN,Tennessee,4705,congressional_district/TN-05 +congressional_district/TN-06,Tennessee's 6th congressional district,congressional_district,state/tn,TN,Tennessee,4706,congressional_district/TN-06 +congressional_district/TN-07,Tennessee's 7th congressional district,congressional_district,state/tn,TN,Tennessee,4707,congressional_district/TN-07 +congressional_district/TN-08,Tennessee's 8th congressional district,congressional_district,state/tn,TN,Tennessee,4708,congressional_district/TN-08 +congressional_district/TN-09,Tennessee's 9th congressional district,congressional_district,state/tn,TN,Tennessee,4709,congressional_district/TN-09 +congressional_district/TX-01,Texas's 1st congressional district,congressional_district,state/tx,TX,Texas,4801,congressional_district/TX-01 +congressional_district/TX-02,Texas's 2nd congressional district,congressional_district,state/tx,TX,Texas,4802,congressional_district/TX-02 +congressional_district/TX-03,Texas's 3rd congressional district,congressional_district,state/tx,TX,Texas,4803,congressional_district/TX-03 +congressional_district/TX-04,Texas's 4th congressional district,congressional_district,state/tx,TX,Texas,4804,congressional_district/TX-04 +congressional_district/TX-05,Texas's 5th congressional district,congressional_district,state/tx,TX,Texas,4805,congressional_district/TX-05 +congressional_district/TX-06,Texas's 6th congressional district,congressional_district,state/tx,TX,Texas,4806,congressional_district/TX-06 +congressional_district/TX-07,Texas's 7th congressional district,congressional_district,state/tx,TX,Texas,4807,congressional_district/TX-07 +congressional_district/TX-08,Texas's 8th congressional district,congressional_district,state/tx,TX,Texas,4808,congressional_district/TX-08 +congressional_district/TX-09,Texas's 9th congressional district,congressional_district,state/tx,TX,Texas,4809,congressional_district/TX-09 +congressional_district/TX-10,Texas's 10th congressional district,congressional_district,state/tx,TX,Texas,4810,congressional_district/TX-10 +congressional_district/TX-11,Texas's 11th congressional district,congressional_district,state/tx,TX,Texas,4811,congressional_district/TX-11 +congressional_district/TX-12,Texas's 12th congressional district,congressional_district,state/tx,TX,Texas,4812,congressional_district/TX-12 +congressional_district/TX-13,Texas's 13th congressional district,congressional_district,state/tx,TX,Texas,4813,congressional_district/TX-13 +congressional_district/TX-14,Texas's 14th congressional district,congressional_district,state/tx,TX,Texas,4814,congressional_district/TX-14 +congressional_district/TX-15,Texas's 15th congressional district,congressional_district,state/tx,TX,Texas,4815,congressional_district/TX-15 +congressional_district/TX-16,Texas's 16th congressional district,congressional_district,state/tx,TX,Texas,4816,congressional_district/TX-16 +congressional_district/TX-17,Texas's 17th congressional district,congressional_district,state/tx,TX,Texas,4817,congressional_district/TX-17 +congressional_district/TX-18,Texas's 18th congressional district,congressional_district,state/tx,TX,Texas,4818,congressional_district/TX-18 +congressional_district/TX-19,Texas's 19th congressional district,congressional_district,state/tx,TX,Texas,4819,congressional_district/TX-19 +congressional_district/TX-20,Texas's 20th congressional district,congressional_district,state/tx,TX,Texas,4820,congressional_district/TX-20 +congressional_district/TX-21,Texas's 21st congressional district,congressional_district,state/tx,TX,Texas,4821,congressional_district/TX-21 +congressional_district/TX-22,Texas's 22nd congressional district,congressional_district,state/tx,TX,Texas,4822,congressional_district/TX-22 +congressional_district/TX-23,Texas's 23rd congressional district,congressional_district,state/tx,TX,Texas,4823,congressional_district/TX-23 +congressional_district/TX-24,Texas's 24th congressional district,congressional_district,state/tx,TX,Texas,4824,congressional_district/TX-24 +congressional_district/TX-25,Texas's 25th congressional district,congressional_district,state/tx,TX,Texas,4825,congressional_district/TX-25 +congressional_district/TX-26,Texas's 26th congressional district,congressional_district,state/tx,TX,Texas,4826,congressional_district/TX-26 +congressional_district/TX-27,Texas's 27th congressional district,congressional_district,state/tx,TX,Texas,4827,congressional_district/TX-27 +congressional_district/TX-28,Texas's 28th congressional district,congressional_district,state/tx,TX,Texas,4828,congressional_district/TX-28 +congressional_district/TX-29,Texas's 29th congressional district,congressional_district,state/tx,TX,Texas,4829,congressional_district/TX-29 +congressional_district/TX-30,Texas's 30th congressional district,congressional_district,state/tx,TX,Texas,4830,congressional_district/TX-30 +congressional_district/TX-31,Texas's 31st congressional district,congressional_district,state/tx,TX,Texas,4831,congressional_district/TX-31 +congressional_district/TX-32,Texas's 32nd congressional district,congressional_district,state/tx,TX,Texas,4832,congressional_district/TX-32 +congressional_district/TX-33,Texas's 33rd congressional district,congressional_district,state/tx,TX,Texas,4833,congressional_district/TX-33 +congressional_district/TX-34,Texas's 34th congressional district,congressional_district,state/tx,TX,Texas,4834,congressional_district/TX-34 +congressional_district/TX-35,Texas's 35th congressional district,congressional_district,state/tx,TX,Texas,4835,congressional_district/TX-35 +congressional_district/TX-36,Texas's 36th congressional district,congressional_district,state/tx,TX,Texas,4836,congressional_district/TX-36 +congressional_district/TX-37,Texas's 37th congressional district,congressional_district,state/tx,TX,Texas,4837,congressional_district/TX-37 +congressional_district/TX-38,Texas's 38th congressional district,congressional_district,state/tx,TX,Texas,4838,congressional_district/TX-38 +congressional_district/UT-01,Utah's 1st congressional district,congressional_district,state/ut,UT,Utah,4901,congressional_district/UT-01 +congressional_district/UT-02,Utah's 2nd congressional district,congressional_district,state/ut,UT,Utah,4902,congressional_district/UT-02 +congressional_district/UT-03,Utah's 3rd congressional district,congressional_district,state/ut,UT,Utah,4903,congressional_district/UT-03 +congressional_district/UT-04,Utah's 4th congressional district,congressional_district,state/ut,UT,Utah,4904,congressional_district/UT-04 +congressional_district/VA-01,Virginia's 1st congressional district,congressional_district,state/va,VA,Virginia,5101,congressional_district/VA-01 +congressional_district/VA-02,Virginia's 2nd congressional district,congressional_district,state/va,VA,Virginia,5102,congressional_district/VA-02 +congressional_district/VA-03,Virginia's 3rd congressional district,congressional_district,state/va,VA,Virginia,5103,congressional_district/VA-03 +congressional_district/VA-04,Virginia's 4th congressional district,congressional_district,state/va,VA,Virginia,5104,congressional_district/VA-04 +congressional_district/VA-05,Virginia's 5th congressional district,congressional_district,state/va,VA,Virginia,5105,congressional_district/VA-05 +congressional_district/VA-06,Virginia's 6th congressional district,congressional_district,state/va,VA,Virginia,5106,congressional_district/VA-06 +congressional_district/VA-07,Virginia's 7th congressional district,congressional_district,state/va,VA,Virginia,5107,congressional_district/VA-07 +congressional_district/VA-08,Virginia's 8th congressional district,congressional_district,state/va,VA,Virginia,5108,congressional_district/VA-08 +congressional_district/VA-09,Virginia's 9th congressional district,congressional_district,state/va,VA,Virginia,5109,congressional_district/VA-09 +congressional_district/VA-10,Virginia's 10th congressional district,congressional_district,state/va,VA,Virginia,5110,congressional_district/VA-10 +congressional_district/VA-11,Virginia's 11th congressional district,congressional_district,state/va,VA,Virginia,5111,congressional_district/VA-11 +congressional_district/VT-01,Vermont's at-large congressional district,congressional_district,state/vt,VT,Vermont,5000,congressional_district/VT-01 +congressional_district/WA-01,Washington's 1st congressional district,congressional_district,state/wa,WA,Washington,5301,congressional_district/WA-01 +congressional_district/WA-02,Washington's 2nd congressional district,congressional_district,state/wa,WA,Washington,5302,congressional_district/WA-02 +congressional_district/WA-03,Washington's 3rd congressional district,congressional_district,state/wa,WA,Washington,5303,congressional_district/WA-03 +congressional_district/WA-04,Washington's 4th congressional district,congressional_district,state/wa,WA,Washington,5304,congressional_district/WA-04 +congressional_district/WA-05,Washington's 5th congressional district,congressional_district,state/wa,WA,Washington,5305,congressional_district/WA-05 +congressional_district/WA-06,Washington's 6th congressional district,congressional_district,state/wa,WA,Washington,5306,congressional_district/WA-06 +congressional_district/WA-07,Washington's 7th congressional district,congressional_district,state/wa,WA,Washington,5307,congressional_district/WA-07 +congressional_district/WA-08,Washington's 8th congressional district,congressional_district,state/wa,WA,Washington,5308,congressional_district/WA-08 +congressional_district/WA-09,Washington's 9th congressional district,congressional_district,state/wa,WA,Washington,5309,congressional_district/WA-09 +congressional_district/WA-10,Washington's 10th congressional district,congressional_district,state/wa,WA,Washington,5310,congressional_district/WA-10 +congressional_district/WI-01,Wisconsin's 1st congressional district,congressional_district,state/wi,WI,Wisconsin,5501,congressional_district/WI-01 +congressional_district/WI-02,Wisconsin's 2nd congressional district,congressional_district,state/wi,WI,Wisconsin,5502,congressional_district/WI-02 +congressional_district/WI-03,Wisconsin's 3rd congressional district,congressional_district,state/wi,WI,Wisconsin,5503,congressional_district/WI-03 +congressional_district/WI-04,Wisconsin's 4th congressional district,congressional_district,state/wi,WI,Wisconsin,5504,congressional_district/WI-04 +congressional_district/WI-05,Wisconsin's 5th congressional district,congressional_district,state/wi,WI,Wisconsin,5505,congressional_district/WI-05 +congressional_district/WI-06,Wisconsin's 6th congressional district,congressional_district,state/wi,WI,Wisconsin,5506,congressional_district/WI-06 +congressional_district/WI-07,Wisconsin's 7th congressional district,congressional_district,state/wi,WI,Wisconsin,5507,congressional_district/WI-07 +congressional_district/WI-08,Wisconsin's 8th congressional district,congressional_district,state/wi,WI,Wisconsin,5508,congressional_district/WI-08 +congressional_district/WV-01,West Virginia's 1st congressional district,congressional_district,state/wv,WV,West Virginia,5401,congressional_district/WV-01 +congressional_district/WV-02,West Virginia's 2nd congressional district,congressional_district,state/wv,WV,West Virginia,5402,congressional_district/WV-02 +congressional_district/WY-01,Wyoming's at-large congressional district,congressional_district,state/wy,WY,Wyoming,5600,congressional_district/WY-01 +place/AK-03000,Anchorage,place,state/ak,AK,Alaska,,place/AK-03000 +place/AL-07000,Birmingham,place,state/al,AL,Alabama,,place/AL-07000 +place/AL-37000,Huntsville,place,state/al,AL,Alabama,,place/AL-37000 +place/AL-50000,Mobile,place,state/al,AL,Alabama,,place/AL-50000 +place/AL-51000,Montgomery,place,state/al,AL,Alabama,,place/AL-51000 +place/AL-77256,Tuscaloosa,place,state/al,AL,Alabama,,place/AL-77256 +place/AR-23290,Fayetteville,place,state/ar,AR,Arkansas,,place/AR-23290 +place/AR-41000,Little Rock,place,state/ar,AR,Arkansas,,place/AR-41000 +place/AZ-07940,Buckeye,place,state/az,AZ,Arizona,,place/AZ-07940 +place/AZ-12000,Chandler,place,state/az,AZ,Arizona,,place/AZ-12000 +place/AZ-27400,Gilbert,place,state/az,AZ,Arizona,,place/AZ-27400 +place/AZ-27820,Glendale,place,state/az,AZ,Arizona,,place/AZ-27820 +place/AZ-28380,Goodyear,place,state/az,AZ,Arizona,,place/AZ-28380 +place/AZ-46000,Mesa,place,state/az,AZ,Arizona,,place/AZ-46000 +place/AZ-54050,Peoria,place,state/az,AZ,Arizona,,place/AZ-54050 +place/AZ-55000,Phoenix,place,state/az,AZ,Arizona,,place/AZ-55000 +place/AZ-65000,Scottsdale,place,state/az,AZ,Arizona,,place/AZ-65000 +place/AZ-71510,Surprise,place,state/az,AZ,Arizona,,place/AZ-71510 +place/AZ-73000,Tempe,place,state/az,AZ,Arizona,,place/AZ-73000 +place/AZ-77000,Tucson,place,state/az,AZ,Arizona,,place/AZ-77000 +place/AZ-85540,Yuma,place,state/az,AZ,Arizona,,place/AZ-85540 +place/CA-02000,Anaheim,place,state/ca,CA,California,,place/CA-02000 +place/CA-02252,Antioch,place,state/ca,CA,California,,place/CA-02252 +place/CA-03526,Bakersfield,place,state/ca,CA,California,,place/CA-03526 +place/CA-06000,Berkeley,place,state/ca,CA,California,,place/CA-06000 +place/CA-08954,Burbank,place,state/ca,CA,California,,place/CA-08954 +place/CA-11194,Carlsbad,place,state/ca,CA,California,,place/CA-11194 +place/CA-13014,Chico,place,state/ca,CA,California,,place/CA-13014 +place/CA-13392,Chula Vista,place,state/ca,CA,California,,place/CA-13392 +place/CA-14218,Clovis,place,state/ca,CA,California,,place/CA-14218 +place/CA-16000,Concord,place,state/ca,CA,California,,place/CA-16000 +place/CA-16350,Corona,place,state/ca,CA,California,,place/CA-16350 +place/CA-16532,Costa Mesa,place,state/ca,CA,California,,place/CA-16532 +place/CA-19766,Downey,place,state/ca,CA,California,,place/CA-19766 +place/CA-21712,El Cajon,place,state/ca,CA,California,,place/CA-21712 +place/CA-22020,Elk Grove,place,state/ca,CA,California,,place/CA-22020 +place/CA-22230,El Monte,place,state/ca,CA,California,,place/CA-22230 +place/CA-22804,Escondido,place,state/ca,CA,California,,place/CA-22804 +place/CA-23182,Fairfield,place,state/ca,CA,California,,place/CA-23182 +place/CA-24680,Fontana,place,state/ca,CA,California,,place/CA-24680 +place/CA-26000,Fremont,place,state/ca,CA,California,,place/CA-26000 +place/CA-27000,Fresno,place,state/ca,CA,California,,place/CA-27000 +place/CA-28000,Fullerton,place,state/ca,CA,California,,place/CA-28000 +place/CA-29000,Garden Grove,place,state/ca,CA,California,,place/CA-29000 +place/CA-30000,Glendale,place,state/ca,CA,California,,place/CA-30000 +place/CA-33000,Hayward,place,state/ca,CA,California,,place/CA-33000 +place/CA-33434,Hesperia,place,state/ca,CA,California,,place/CA-33434 +place/CA-36000,Huntington Beach,place,state/ca,CA,California,,place/CA-36000 +place/CA-36546,Inglewood,place,state/ca,CA,California,,place/CA-36546 +place/CA-36770,Irvine,place,state/ca,CA,California,,place/CA-36770 +place/CA-37692,Jurupa Valley,place,state/ca,CA,California,,place/CA-37692 +place/CA-40130,Lancaster,place,state/ca,CA,California,,place/CA-40130 +place/CA-43000,Long Beach,place,state/ca,CA,California,,place/CA-43000 +place/CA-44000,Los Angeles,place,state/ca,CA,California,,place/CA-44000 +place/CA-46842,Menifee,place,state/ca,CA,California,,place/CA-46842 +place/CA-48354,Modesto,place,state/ca,CA,California,,place/CA-48354 +place/CA-49270,Moreno Valley,place,state/ca,CA,California,,place/CA-49270 +place/CA-50076,Murrieta,place,state/ca,CA,California,,place/CA-50076 +place/CA-53000,Oakland,place,state/ca,CA,California,,place/CA-53000 +place/CA-53322,Oceanside,place,state/ca,CA,California,,place/CA-53322 +place/CA-53896,Ontario,place,state/ca,CA,California,,place/CA-53896 +place/CA-53980,Orange,place,state/ca,CA,California,,place/CA-53980 +place/CA-54652,Oxnard,place,state/ca,CA,California,,place/CA-54652 +place/CA-55156,Palmdale,place,state/ca,CA,California,,place/CA-55156 +place/CA-56000,Pasadena,place,state/ca,CA,California,,place/CA-56000 +place/CA-58072,Pomona,place,state/ca,CA,California,,place/CA-58072 +place/CA-59451,Rancho Cucamonga,place,state/ca,CA,California,,place/CA-59451 +place/CA-60466,Rialto,place,state/ca,CA,California,,place/CA-60466 +place/CA-60620,Richmond,place,state/ca,CA,California,,place/CA-60620 +place/CA-62000,Riverside,place,state/ca,CA,California,,place/CA-62000 +place/CA-62938,Roseville,place,state/ca,CA,California,,place/CA-62938 +place/CA-64000,Sacramento,place,state/ca,CA,California,,place/CA-64000 +place/CA-64224,Salinas,place,state/ca,CA,California,,place/CA-64224 +place/CA-65000,San Bernardino,place,state/ca,CA,California,,place/CA-65000 +place/CA-65042,Ventura,place,state/ca,CA,California,,place/CA-65042 +place/CA-66000,San Diego,place,state/ca,CA,California,,place/CA-66000 +place/CA-67000,San Francisco,place,state/ca,CA,California,,place/CA-67000 +place/CA-68000,San Jose,place,state/ca,CA,California,,place/CA-68000 +place/CA-68252,San Mateo,place,state/ca,CA,California,,place/CA-68252 +place/CA-69000,Santa Ana,place,state/ca,CA,California,,place/CA-69000 +place/CA-69084,Santa Clara,place,state/ca,CA,California,,place/CA-69084 +place/CA-69088,Santa Clarita,place,state/ca,CA,California,,place/CA-69088 +place/CA-69196,Santa Maria,place,state/ca,CA,California,,place/CA-69196 +place/CA-70098,Santa Rosa,place,state/ca,CA,California,,place/CA-70098 +place/CA-72016,Simi Valley,place,state/ca,CA,California,,place/CA-72016 +place/CA-75000,Stockton,place,state/ca,CA,California,,place/CA-75000 +place/CA-77000,Sunnyvale,place,state/ca,CA,California,,place/CA-77000 +place/CA-78120,Temecula,place,state/ca,CA,California,,place/CA-78120 +place/CA-78582,Thousand Oaks,place,state/ca,CA,California,,place/CA-78582 +place/CA-80000,Torrance,place,state/ca,CA,California,,place/CA-80000 +place/CA-81554,Vacaville,place,state/ca,CA,California,,place/CA-81554 +place/CA-81666,Vallejo,place,state/ca,CA,California,,place/CA-81666 +place/CA-82590,Victorville,place,state/ca,CA,California,,place/CA-82590 +place/CA-82954,Visalia,place,state/ca,CA,California,,place/CA-82954 +place/CA-84200,West Covina,place,state/ca,CA,California,,place/CA-84200 +place/CO-03455,Arvada,place,state/co,CO,Colorado,,place/CO-03455 +place/CO-04000,Aurora,place,state/co,CO,Colorado,,place/CO-04000 +place/CO-07850,Boulder,place,state/co,CO,Colorado,,place/CO-07850 +place/CO-12815,Centennial,place,state/co,CO,Colorado,,place/CO-12815 +place/CO-16000,Colorado Springs,place,state/co,CO,Colorado,,place/CO-16000 +place/CO-20000,Denver,place,state/co,CO,Colorado,,place/CO-20000 +place/CO-27425,Fort Collins,place,state/co,CO,Colorado,,place/CO-27425 +place/CO-32155,Greeley,place,state/co,CO,Colorado,,place/CO-32155 +place/CO-43000,Lakewood,place,state/co,CO,Colorado,,place/CO-43000 +place/CO-62000,Pueblo,place,state/co,CO,Colorado,,place/CO-62000 +place/CO-77290,Thornton,place,state/co,CO,Colorado,,place/CO-77290 +place/CO-83835,Westminster,place,state/co,CO,Colorado,,place/CO-83835 +place/CT-08000,Bridgeport,place,state/ct,CT,Connecticut,,place/CT-08000 +place/CT-37000,Hartford,place,state/ct,CT,Connecticut,,place/CT-37000 +place/CT-52000,New Haven,place,state/ct,CT,Connecticut,,place/CT-52000 +place/CT-73000,Stamford,place,state/ct,CT,Connecticut,,place/CT-73000 +place/CT-80000,Waterbury,place,state/ct,CT,Connecticut,,place/CT-80000 +place/DC-50000,Washington,place,state/dc,DC,District of Columbia,,place/DC-50000 +place/FL-10275,Cape Coral,place,state/fl,FL,Florida,,place/FL-10275 +place/FL-12875,Clearwater,place,state/fl,FL,Florida,,place/FL-12875 +place/FL-14400,Coral Springs,place,state/fl,FL,Florida,,place/FL-14400 +place/FL-16475,Davie,place,state/fl,FL,Florida,,place/FL-16475 +place/FL-24000,Fort Lauderdale,place,state/fl,FL,Florida,,place/FL-24000 +place/FL-25175,Gainesville,place,state/fl,FL,Florida,,place/FL-25175 +place/FL-30000,Hialeah,place,state/fl,FL,Florida,,place/FL-30000 +place/FL-32000,Hollywood,place,state/fl,FL,Florida,,place/FL-32000 +place/FL-35000,Jacksonville,place,state/fl,FL,Florida,,place/FL-35000 +place/FL-38250,Lakeland,place,state/fl,FL,Florida,,place/FL-38250 +place/FL-45000,Miami,place,state/fl,FL,Florida,,place/FL-45000 +place/FL-45060,Miami Gardens,place,state/fl,FL,Florida,,place/FL-45060 +place/FL-45975,Miramar,place,state/fl,FL,Florida,,place/FL-45975 +place/FL-53000,Orlando,place,state/fl,FL,Florida,,place/FL-53000 +place/FL-54000,Palm Bay,place,state/fl,FL,Florida,,place/FL-54000 +place/FL-54200,Palm Coast,place,state/fl,FL,Florida,,place/FL-54200 +place/FL-55775,Pembroke Pines,place,state/fl,FL,Florida,,place/FL-55775 +place/FL-58050,Pompano Beach,place,state/fl,FL,Florida,,place/FL-58050 +place/FL-58715,Port St. Lucie,place,state/fl,FL,Florida,,place/FL-58715 +place/FL-63000,St. Petersburg,place,state/fl,FL,Florida,,place/FL-63000 +place/FL-70600,Tallahassee,place,state/fl,FL,Florida,,place/FL-70600 +place/FL-71000,Tampa,place,state/fl,FL,Florida,,place/FL-71000 +place/FL-76600,West Palm Beach,place,state/fl,FL,Florida,,place/FL-76600 +place/GA-03440,Athens-Clarke County,place,state/ga,GA,Georgia,,place/GA-03440 +place/GA-04000,Atlanta,place,state/ga,GA,Georgia,,place/GA-04000 +place/GA-04204,Augusta-Richmond County,place,state/ga,GA,Georgia,,place/GA-04204 +place/GA-19000,Columbus,place,state/ga,GA,Georgia,,place/GA-19000 +place/GA-49008,Macon-Bibb County,place,state/ga,GA,Georgia,,place/GA-49008 +place/GA-68516,Sandy Springs,place,state/ga,GA,Georgia,,place/GA-68516 +place/GA-69000,Savannah,place,state/ga,GA,Georgia,,place/GA-69000 +place/GA-72122,South Fulton,place,state/ga,GA,Georgia,,place/GA-72122 +place/HI-71550,Urban Honolulu,place,state/hi,HI,Hawaii,,place/HI-71550 +place/IA-12000,Cedar Rapids,place,state/ia,IA,Iowa,,place/IA-12000 +place/IA-19000,Davenport,place,state/ia,IA,Iowa,,place/IA-19000 +place/IA-21000,Des Moines,place,state/ia,IA,Iowa,,place/IA-21000 +place/ID-08830,Boise City,place,state/id,ID,Idaho,,place/ID-08830 +place/ID-52120,Meridian,place,state/id,ID,Idaho,,place/ID-52120 +place/ID-56260,Nampa,place,state/id,ID,Idaho,,place/ID-56260 +place/IL-03012,Aurora,place,state/il,IL,Illinois,,place/IL-03012 +place/IL-14000,Chicago,place,state/il,IL,Illinois,,place/IL-14000 +place/IL-23074,Elgin,place,state/il,IL,Illinois,,place/IL-23074 +place/IL-38570,Joliet,place,state/il,IL,Illinois,,place/IL-38570 +place/IL-51622,Naperville,place,state/il,IL,Illinois,,place/IL-51622 +place/IL-59000,Peoria,place,state/il,IL,Illinois,,place/IL-59000 +place/IL-65000,Rockford,place,state/il,IL,Illinois,,place/IL-65000 +place/IL-72000,Springfield,place,state/il,IL,Illinois,,place/IL-72000 +place/IN-10342,Carmel,place,state/in,IN,Indiana,,place/IN-10342 +place/IN-22000,Evansville,place,state/in,IN,Indiana,,place/IN-22000 +place/IN-23278,Fishers,place,state/in,IN,Indiana,,place/IN-23278 +place/IN-25000,Fort Wayne,place,state/in,IN,Indiana,,place/IN-25000 +place/IN-36003,Indianapolis,place,state/in,IN,Indiana,,place/IN-36003 +place/IN-71000,South Bend,place,state/in,IN,Indiana,,place/IN-71000 +place/KS-36000,Kansas City,place,state/ks,KS,Kansas,,place/KS-36000 +place/KS-52575,Olathe,place,state/ks,KS,Kansas,,place/KS-52575 +place/KS-53775,Overland Park,place,state/ks,KS,Kansas,,place/KS-53775 +place/KS-71000,Topeka,place,state/ks,KS,Kansas,,place/KS-71000 +place/KS-79000,Wichita,place,state/ks,KS,Kansas,,place/KS-79000 +place/KY-46027,Lexington-Fayette,place,state/ky,KY,Kentucky,,place/KY-46027 +place/KY-48006,Louisville/Jefferson County,place,state/ky,KY,Kentucky,,place/KY-48006 +place/LA-05000,Baton Rouge,place,state/la,LA,Louisiana,,place/LA-05000 +place/LA-40735,Lafayette,place,state/la,LA,Louisiana,,place/LA-40735 +place/LA-55000,New Orleans,place,state/la,LA,Louisiana,,place/LA-55000 +place/LA-70000,Shreveport,place,state/la,LA,Louisiana,,place/LA-70000 +place/MA-07000,Boston,place,state/ma,MA,Massachusetts,,place/MA-07000 +place/MA-09000,Brockton,place,state/ma,MA,Massachusetts,,place/MA-09000 +place/MA-11000,Cambridge,place,state/ma,MA,Massachusetts,,place/MA-11000 +place/MA-37000,Lowell,place,state/ma,MA,Massachusetts,,place/MA-37000 +place/MA-37490,Lynn,place,state/ma,MA,Massachusetts,,place/MA-37490 +place/MA-45000,New Bedford,place,state/ma,MA,Massachusetts,,place/MA-45000 +place/MA-55745,Quincy,place,state/ma,MA,Massachusetts,,place/MA-55745 +place/MA-67000,Springfield,place,state/ma,MA,Massachusetts,,place/MA-67000 +place/MA-82000,Worcester,place,state/ma,MA,Massachusetts,,place/MA-82000 +place/MD-04000,Baltimore,place,state/md,MD,Maryland,,place/MD-04000 +place/MI-03000,Ann Arbor,place,state/mi,MI,Michigan,,place/MI-03000 +place/MI-21000,Dearborn,place,state/mi,MI,Michigan,,place/MI-21000 +place/MI-22000,Detroit,place,state/mi,MI,Michigan,,place/MI-22000 +place/MI-34000,Grand Rapids,place,state/mi,MI,Michigan,,place/MI-34000 +place/MI-46000,Lansing,place,state/mi,MI,Michigan,,place/MI-46000 +place/MI-76460,Sterling Heights,place,state/mi,MI,Michigan,,place/MI-76460 +place/MI-84000,Warren,place,state/mi,MI,Michigan,,place/MI-84000 +place/MN-43000,Minneapolis,place,state/mn,MN,Minnesota,,place/MN-43000 +place/MN-54880,Rochester,place,state/mn,MN,Minnesota,,place/MN-54880 +place/MN-58000,St. Paul,place,state/mn,MN,Minnesota,,place/MN-58000 +place/MO-15670,Columbia,place,state/mo,MO,Missouri,,place/MO-15670 +place/MO-35000,Independence,place,state/mo,MO,Missouri,,place/MO-35000 +place/MO-38000,Kansas City,place,state/mo,MO,Missouri,,place/MO-38000 +place/MO-41348,Lee's Summit,place,state/mo,MO,Missouri,,place/MO-41348 +place/MO-65000,St. Louis,place,state/mo,MO,Missouri,,place/MO-65000 +place/MO-70000,Springfield,place,state/mo,MO,Missouri,,place/MO-70000 +place/MS-36000,Jackson,place,state/ms,MS,Mississippi,,place/MS-36000 +place/MT-06550,Billings,place,state/mt,MT,Montana,,place/MT-06550 +place/NC-10740,Cary,place,state/nc,NC,North Carolina,,place/NC-10740 +place/NC-12000,Charlotte,place,state/nc,NC,North Carolina,,place/NC-12000 +place/NC-14100,Concord,place,state/nc,NC,North Carolina,,place/NC-14100 +place/NC-19000,Durham,place,state/nc,NC,North Carolina,,place/NC-19000 +place/NC-22920,Fayetteville,place,state/nc,NC,North Carolina,,place/NC-22920 +place/NC-28000,Greensboro,place,state/nc,NC,North Carolina,,place/NC-28000 +place/NC-31400,High Point,place,state/nc,NC,North Carolina,,place/NC-31400 +place/NC-55000,Raleigh,place,state/nc,NC,North Carolina,,place/NC-55000 +place/NC-74440,Wilmington,place,state/nc,NC,North Carolina,,place/NC-74440 +place/NC-75000,Winston-Salem,place,state/nc,NC,North Carolina,,place/NC-75000 +place/ND-25700,Fargo,place,state/nd,ND,North Dakota,,place/ND-25700 +place/NE-28000,Lincoln,place,state/ne,NE,Nebraska,,place/NE-28000 +place/NE-37000,Omaha,place,state/ne,NE,Nebraska,,place/NE-37000 +place/NH-45140,Manchester,place,state/nh,NH,New Hampshire,,place/NH-45140 +place/NJ-21000,Elizabeth,place,state/nj,NJ,New Jersey,,place/NJ-21000 +place/NJ-36000,Jersey City,place,state/nj,NJ,New Jersey,,place/NJ-36000 +place/NJ-51000,Newark,place,state/nj,NJ,New Jersey,,place/NJ-51000 +place/NJ-57000,Paterson,place,state/nj,NJ,New Jersey,,place/NJ-57000 +place/NM-02000,Albuquerque,place,state/nm,NM,New Mexico,,place/NM-02000 +place/NM-39380,Las Cruces,place,state/nm,NM,New Mexico,,place/NM-39380 +place/NM-63460,Rio Rancho,place,state/nm,NM,New Mexico,,place/NM-63460 +place/NV-31900,Henderson,place,state/nv,NV,Nevada,,place/NV-31900 +place/NV-40000,Las Vegas,place,state/nv,NV,Nevada,,place/NV-40000 +place/NV-51800,North Las Vegas,place,state/nv,NV,Nevada,,place/NV-51800 +place/NV-60600,Reno,place,state/nv,NV,Nevada,,place/NV-60600 +place/NV-68400,Sparks,place,state/nv,NV,Nevada,,place/NV-68400 +place/NY-01000,Albany,place,state/ny,NY,New York,,place/NY-01000 +place/NY-11000,Buffalo,place,state/ny,NY,New York,,place/NY-11000 +place/NY-51000,New York City,place,state/ny,NY,New York,,place/NY-51000 +place/NY-63000,Rochester,place,state/ny,NY,New York,,place/NY-63000 +place/NY-73000,Syracuse,place,state/ny,NY,New York,,place/NY-73000 +place/NY-84000,Yonkers,place,state/ny,NY,New York,,place/NY-84000 +place/OH-01000,Akron,place,state/oh,OH,Ohio,,place/OH-01000 +place/OH-15000,Cincinnati,place,state/oh,OH,Ohio,,place/OH-15000 +place/OH-16000,Cleveland,place,state/oh,OH,Ohio,,place/OH-16000 +place/OH-18000,Columbus,place,state/oh,OH,Ohio,,place/OH-18000 +place/OH-21000,Dayton,place,state/oh,OH,Ohio,,place/OH-21000 +place/OH-77000,Toledo,place,state/oh,OH,Ohio,,place/OH-77000 +place/OK-09050,Broken Arrow,place,state/ok,OK,Oklahoma,,place/OK-09050 +place/OK-52500,Norman,place,state/ok,OK,Oklahoma,,place/OK-52500 +place/OK-55000,Oklahoma City,place,state/ok,OK,Oklahoma,,place/OK-55000 +place/OK-75000,Tulsa,place,state/ok,OK,Oklahoma,,place/OK-75000 +place/OR-05800,Bend,place,state/or,OR,Oregon,,place/OR-05800 +place/OR-23850,Eugene,place,state/or,OR,Oregon,,place/OR-23850 +place/OR-31250,Gresham,place,state/or,OR,Oregon,,place/OR-31250 +place/OR-34100,Hillsboro,place,state/or,OR,Oregon,,place/OR-34100 +place/OR-59000,Portland,place,state/or,OR,Oregon,,place/OR-59000 +place/OR-64900,Salem,place,state/or,OR,Oregon,,place/OR-64900 +place/PA-02000,Allentown,place,state/pa,PA,Pennsylvania,,place/PA-02000 +place/PA-60000,Philadelphia,place,state/pa,PA,Pennsylvania,,place/PA-60000 +place/PA-61000,Pittsburgh,place,state/pa,PA,Pennsylvania,,place/PA-61000 +place/RI-59000,Providence,place,state/ri,RI,Rhode Island,,place/RI-59000 +place/SC-13330,Charleston,place,state/sc,SC,South Carolina,,place/SC-13330 +place/SC-16000,Columbia,place,state/sc,SC,South Carolina,,place/SC-16000 +place/SC-50875,North Charleston,place,state/sc,SC,South Carolina,,place/SC-50875 +place/SD-59020,Sioux Falls,place,state/sd,SD,South Dakota,,place/SD-59020 +place/TN-14000,Chattanooga,place,state/tn,TN,Tennessee,,place/TN-14000 +place/TN-15160,Clarksville,place,state/tn,TN,Tennessee,,place/TN-15160 +place/TN-40000,Knoxville,place,state/tn,TN,Tennessee,,place/TN-40000 +place/TN-48000,Memphis,place,state/tn,TN,Tennessee,,place/TN-48000 +place/TN-51560,Murfreesboro,place,state/tn,TN,Tennessee,,place/TN-51560 +place/TN-52006,Nashville-Davidson,place,state/tn,TN,Tennessee,,place/TN-52006 +place/TX-01000,Abilene,place,state/tx,TX,Texas,,place/TX-01000 +place/TX-01924,Allen,place,state/tx,TX,Texas,,place/TX-01924 +place/TX-03000,Amarillo,place,state/tx,TX,Texas,,place/TX-03000 +place/TX-04000,Arlington,place,state/tx,TX,Texas,,place/TX-04000 +place/TX-05000,Austin,place,state/tx,TX,Texas,,place/TX-05000 +place/TX-07000,Beaumont,place,state/tx,TX,Texas,,place/TX-07000 +place/TX-10768,Brownsville,place,state/tx,TX,Texas,,place/TX-10768 +place/TX-13024,Carrollton,place,state/tx,TX,Texas,,place/TX-13024 +place/TX-15976,College Station,place,state/tx,TX,Texas,,place/TX-15976 +place/TX-16432,Conroe,place,state/tx,TX,Texas,,place/TX-16432 +place/TX-17000,Corpus Christi,place,state/tx,TX,Texas,,place/TX-17000 +place/TX-19000,Dallas,place,state/tx,TX,Texas,,place/TX-19000 +place/TX-19972,Denton,place,state/tx,TX,Texas,,place/TX-19972 +place/TX-22660,Edinburg,place,state/tx,TX,Texas,,place/TX-22660 +place/TX-24000,El Paso,place,state/tx,TX,Texas,,place/TX-24000 +place/TX-27000,Fort Worth,place,state/tx,TX,Texas,,place/TX-27000 +place/TX-27684,Frisco,place,state/tx,TX,Texas,,place/TX-27684 +place/TX-29000,Garland,place,state/tx,TX,Texas,,place/TX-29000 +place/TX-30464,Grand Prairie,place,state/tx,TX,Texas,,place/TX-30464 +place/TX-35000,Houston,place,state/tx,TX,Texas,,place/TX-35000 +place/TX-37000,Irving,place,state/tx,TX,Texas,,place/TX-37000 +place/TX-39148,Killeen,place,state/tx,TX,Texas,,place/TX-39148 +place/TX-41464,Laredo,place,state/tx,TX,Texas,,place/TX-41464 +place/TX-41980,League City,place,state/tx,TX,Texas,,place/TX-41980 +place/TX-42508,Lewisville,place,state/tx,TX,Texas,,place/TX-42508 +place/TX-45000,Lubbock,place,state/tx,TX,Texas,,place/TX-45000 +place/TX-45384,McAllen,place,state/tx,TX,Texas,,place/TX-45384 +place/TX-45744,McKinney,place,state/tx,TX,Texas,,place/TX-45744 +place/TX-47892,Mesquite,place,state/tx,TX,Texas,,place/TX-47892 +place/TX-48072,Midland,place,state/tx,TX,Texas,,place/TX-48072 +place/TX-50820,New Braunfels,place,state/tx,TX,Texas,,place/TX-50820 +place/TX-53388,Odessa,place,state/tx,TX,Texas,,place/TX-53388 +place/TX-56000,Pasadena,place,state/tx,TX,Texas,,place/TX-56000 +place/TX-56348,Pearland,place,state/tx,TX,Texas,,place/TX-56348 +place/TX-58016,Plano,place,state/tx,TX,Texas,,place/TX-58016 +place/TX-61796,Richardson,place,state/tx,TX,Texas,,place/TX-61796 +place/TX-63500,Round Rock,place,state/tx,TX,Texas,,place/TX-63500 +place/TX-65000,San Antonio,place,state/tx,TX,Texas,,place/TX-65000 +place/TX-70808,Sugar Land,place,state/tx,TX,Texas,,place/TX-70808 +place/TX-74144,Tyler,place,state/tx,TX,Texas,,place/TX-74144 +place/TX-76000,Waco,place,state/tx,TX,Texas,,place/TX-76000 +place/TX-79000,Wichita Falls,place,state/tx,TX,Texas,,place/TX-79000 +place/UT-62470,Provo,place,state/ut,UT,Utah,,place/UT-62470 +place/UT-65330,St. George,place,state/ut,UT,Utah,,place/UT-65330 +place/UT-67000,Salt Lake City,place,state/ut,UT,Utah,,place/UT-67000 +place/UT-82950,West Jordan,place,state/ut,UT,Utah,,place/UT-82950 +place/UT-83470,West Valley City,place,state/ut,UT,Utah,,place/UT-83470 +place/VA-01000,Alexandria,place,state/va,VA,Virginia,,place/VA-01000 +place/VA-16000,Chesapeake,place,state/va,VA,Virginia,,place/VA-16000 +place/VA-35000,Hampton,place,state/va,VA,Virginia,,place/VA-35000 +place/VA-56000,Newport News,place,state/va,VA,Virginia,,place/VA-56000 +place/VA-57000,Norfolk,place,state/va,VA,Virginia,,place/VA-57000 +place/VA-67000,Richmond,place,state/va,VA,Virginia,,place/VA-67000 +place/VA-76432,Suffolk,place,state/va,VA,Virginia,,place/VA-76432 +place/VA-82000,Virginia Beach,place,state/va,VA,Virginia,,place/VA-82000 +place/WA-05210,Bellevue,place,state/wa,WA,Washington,,place/WA-05210 +place/WA-22640,Everett,place,state/wa,WA,Washington,,place/WA-22640 +place/WA-35415,Kent,place,state/wa,WA,Washington,,place/WA-35415 +place/WA-57745,Renton,place,state/wa,WA,Washington,,place/WA-57745 +place/WA-63000,Seattle,place,state/wa,WA,Washington,,place/WA-63000 +place/WA-67000,Spokane,place,state/wa,WA,Washington,,place/WA-67000 +place/WA-67167,Spokane Valley,place,state/wa,WA,Washington,,place/WA-67167 +place/WA-70000,Tacoma,place,state/wa,WA,Washington,,place/WA-70000 +place/WA-74060,Vancouver,place,state/wa,WA,Washington,,place/WA-74060 +place/WI-31000,Green Bay,place,state/wi,WI,Wisconsin,,place/WI-31000 +place/WI-48000,Madison,place,state/wi,WI,Wisconsin,,place/WI-48000 +place/WI-53000,Milwaukee,place,state/wi,WI,Wisconsin,,place/WI-53000 diff --git a/policyengine_api/data/v2/catalog/region_references.py b/policyengine_api/data/v2/catalog/region_references.py new file mode 100644 index 000000000..ce37a5908 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_references.py @@ -0,0 +1,251 @@ +"""The Public API's checked-in region reference files, one per country. + +Each file lists every region of one country: its code, label, type, parent, +US state, the value that selects its households in its dataset, and the v1 +identifier it replaces. Catalog publication reads regions from these files, +never from PolicyEngine.py's region registry. +""" + +from __future__ import annotations + +import csv +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + +REGION_REFERENCE_DIRECTORY = Path(__file__).parent / "region_reference_files" +REGION_REFERENCE_FIELDS = ( + "code", + "label", + "region_type", + "parent_code", + "state_code", + "state_name", + "filter_value", + "v1_code", +) +SUPPORTED_REGION_TYPES = frozenset( + { + "national", + "country", + "state", + "congressional_district", + "constituency", + "local_authority", + "city", + "place", + } +) +# The catalog's region columns hold at most these many characters. +MAX_REGION_CODE_LENGTH = 255 +MAX_REGION_LABEL_LENGTH = 255 +MAX_REGION_STATE_CODE_LENGTH = 16 +MAX_REGION_STATE_NAME_LENGTH = 128 +MAX_REGION_FILTER_VALUE_LENGTH = 255 + + +@dataclass(frozen=True, slots=True) +class RegionReference: + """One region as its country's reference file lists it. + + ``filter_value`` is the text that selects the region's households: a FIPS + number or district GEOID for US states and districts, the stored region + names of a UK nation, or the ONS code of a UK constituency or local + authority (its row in the weight matrix). The national region and regions + that cannot run yet have none. ``v1_code`` is the v1 region identifier this + region replaces, if v1 had one. + """ + + code: str + label: str + region_type: str + parent_code: str | None + state_code: str | None + state_name: str | None + filter_value: str | None + v1_code: str | None + + +def _text( + row: Mapping[str, str | None], + field: str, + *, + code: str, + maximum: int, + required: bool, +) -> str | None: + value = row.get(field) or "" + if value != value.strip(): + msg = f"region {code!r} {field} has surrounding whitespace" + raise CatalogExtractionError(msg) + if not value: + if required: + msg = f"region {code!r} {field} must be a non-empty string" + raise CatalogExtractionError(msg) + return None + if len(value) > maximum: + msg = f"region {code!r} {field} exceeds {maximum} characters" + raise CatalogExtractionError(msg) + return value + + +def _reference(country_id: str, row: Mapping[str, str | None]) -> RegionReference: + code = row.get("code") or "" + if not code or code != code.strip() or len(code) > MAX_REGION_CODE_LENGTH: + msg = f"{country_id} region reference file has an invalid code {code!r}" + raise CatalogExtractionError(msg) + region_type = row.get("region_type") or "" + if region_type not in SUPPORTED_REGION_TYPES: + msg = f"region {code!r} has unsupported type {region_type!r}" + raise CatalogExtractionError(msg) + if region_type != "national" and not code.startswith(f"{region_type}/"): + msg = f"region {code!r} does not start with its type {region_type + '/'!r}" + raise CatalogExtractionError(msg) + return RegionReference( + code=code, + label=_text( + row, "label", code=code, maximum=MAX_REGION_LABEL_LENGTH, required=True + ) + or "", + region_type=region_type, + parent_code=_text( + row, + "parent_code", + code=code, + maximum=MAX_REGION_CODE_LENGTH, + required=False, + ), + state_code=_text( + row, + "state_code", + code=code, + maximum=MAX_REGION_STATE_CODE_LENGTH, + required=False, + ), + state_name=_text( + row, + "state_name", + code=code, + maximum=MAX_REGION_STATE_NAME_LENGTH, + required=False, + ), + filter_value=_text( + row, + "filter_value", + code=code, + maximum=MAX_REGION_FILTER_VALUE_LENGTH, + required=False, + ), + v1_code=_text( + row, "v1_code", code=code, maximum=MAX_REGION_CODE_LENGTH, required=False + ), + ) + + +def _repeated(values: Iterable[str | None]) -> list[str]: + return sorted( + value + for value, count in Counter(value for value in values if value).items() + if count > 1 + ) + + +def _check_parents_present( + country_id: str, + references: tuple[RegionReference, ...], +) -> None: + """Require one parentless national region and a parent for every other region.""" + if not any(reference.code == country_id for reference in references): + msg = ( + f"{country_id} region reference file has no national region {country_id!r}" + ) + raise CatalogExtractionError(msg) + for reference in references: + if reference.region_type == "national": + if reference.code != country_id: + msg = f"national region {reference.code!r} must be named {country_id!r}" + raise CatalogExtractionError(msg) + if reference.parent_code is not None: + msg = f"national region {reference.code!r} must have no parent" + raise CatalogExtractionError(msg) + elif reference.parent_code is None: + msg = f"region {reference.code!r} has no parent" + raise CatalogExtractionError(msg) + + +def _check_ancestry( + country_id: str, + references: tuple[RegionReference, ...], +) -> None: + """Require every region's parents to exist and lead to the national region.""" + parents = {reference.code: reference.parent_code for reference in references} + for reference in references: + visited = {reference.code} + ancestor = reference.parent_code + while ancestor is not None and ancestor != country_id: + if ancestor not in parents: + msg = ( + f"region {reference.code!r} names an unknown parent " + f"{reference.parent_code!r}" + ) + raise CatalogExtractionError(msg) + if ancestor in visited: + msg = f"region {reference.code!r} has a parent cycle" + raise CatalogExtractionError(msg) + visited.add(ancestor) + ancestor = parents[ancestor] + + +def parse_region_references( + country_id: str, + rows: Iterable[Mapping[str, str | None]], +) -> tuple[RegionReference, ...]: + """Validate one country's reference rows and return them in file order. + + Codes and v1 identifiers must be unique, every region's code must start + with its type, and every region must descend from the national region. + """ + references = tuple(_reference(country_id, row) for row in rows) + repeated_codes = _repeated(reference.code for reference in references) + if repeated_codes: + msg = ( + f"{country_id} region reference file repeats region codes: {repeated_codes}" + ) + raise CatalogExtractionError(msg) + repeated_v1_codes = _repeated(reference.v1_code for reference in references) + if repeated_v1_codes: + msg = ( + f"{country_id} region reference file repeats v1 identifiers: " + f"{repeated_v1_codes}" + ) + raise CatalogExtractionError(msg) + _check_parents_present(country_id, references) + _check_ancestry(country_id, references) + return references + + +def load_region_references( + country_id: str, + *, + directory: Path = REGION_REFERENCE_DIRECTORY, +) -> tuple[RegionReference, ...]: + """Read and validate the checked-in reference file of one country.""" + path = directory / f"{country_id}.csv" + if not path.is_file(): + msg = f"{country_id} has no region reference file" + raise CatalogExtractionError(msg) + with path.open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + if tuple(reader.fieldnames or ()) != REGION_REFERENCE_FIELDS: + msg = ( + f"{country_id} region reference file columns must be " + f"{', '.join(REGION_REFERENCE_FIELDS)}" + ) + raise CatalogExtractionError(msg) + return parse_region_references(country_id, reader) diff --git a/tests/unit/v2/test_region_references.py b/tests/unit/v2/test_region_references.py new file mode 100644 index 000000000..dd97be4b4 --- /dev/null +++ b/tests/unit/v2/test_region_references.py @@ -0,0 +1,318 @@ +"""The checked-in region reference files and the rules their loader enforces.""" + +from __future__ import annotations + +import csv +import json +from collections import Counter +from pathlib import Path + +import pytest + +from policyengine_api.data.congressional_districts import ( + AT_LARGE_STATES, + CONGRESSIONAL_DISTRICTS, + STATE_CODE_TO_NAME, + build_congressional_district_metadata, +) +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.region_references import ( + REGION_REFERENCE_FIELDS, + RegionReference, + load_region_references, + parse_region_references, +) + +V1_DATA_DIRECTORY = Path(__file__).parents[3] / "policyengine_api" / "data" +NATION_BY_ONS_PREFIX = { + "E": "country/england", + "S": "country/scotland", + "W": "country/wales", + "N": "country/northern_ireland", +} +ENGLISH_ITL1_REGIONS = [ + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", +] + + +def _by_code(references: tuple[RegionReference, ...]) -> dict[str, RegionReference]: + return {reference.code: reference for reference in references} + + +def _v1_lookup_rows(filename: str) -> list[dict[str, str]]: + with (V1_DATA_DIRECTORY / filename).open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def _row(code: str, region_type: str, **fields: str) -> dict[str, str]: + """Return one reference-file row; unspecified columns are empty.""" + row = dict.fromkeys(REGION_REFERENCE_FIELDS, "") + row.update(code=code, label=code.title(), region_type=region_type, v1_code=code) + row.update(fields) + return row + + +def _small_us_rows() -> list[dict[str, str]]: + return [ + _row("us", "national"), + _row("state/ca", "state", parent_code="us", filter_value="6"), + _row( + "congressional_district/CA-01", + "congressional_district", + parent_code="state/ca", + ), + ] + + +def test_each_country_lists_every_region_once_with_the_expected_counts() -> None: + us = load_region_references("us") + uk = load_region_references("uk") + + assert Counter(reference.region_type for reference in us) == { + "national": 1, + "state": 51, + "congressional_district": 436, + "place": 333, + } + assert Counter(reference.region_type for reference in uk) == { + "national": 1, + "country": 4, + "constituency": 650, + "local_authority": 361, + } + assert len(uk) == 1_016 + for references in (us, uk): + codes = [reference.code for reference in references] + v1_codes = [reference.v1_code for reference in references] + assert len(set(codes)) == len(codes) + assert len(set(v1_codes)) == len(v1_codes) + + +def test_national_regions_and_the_v1_identifiers_they_replace() -> None: + us = _by_code(load_region_references("us")) + uk = _by_code(load_region_references("uk")) + + assert us["us"].label == "United States" + assert uk["uk"].label == "United Kingdom" + assert us["us"].parent_code is None + assert uk["uk"].parent_code is None + assert uk["country/northern_ireland"].v1_code == "country/ni" + assert uk["country/northern_ireland"].label == "Northern Ireland" + assert "country/ni" not in uk + assert uk["constituency/E14001063"].label == "Aldershot" + assert uk["constituency/E14001063"].v1_code == "constituency/Aldershot" + assert uk["local_authority/E07000110"].label == "Maidstone" + assert uk["local_authority/E07000110"].v1_code == "local_authority/Maidstone" + assert uk["local_authority/N09000011"].label == "Ards and North Down" + assert uk["local_authority/N09000011"].parent_code == "country/northern_ireland" + assert uk["local_authority/N09000011"].v1_code == ( + "local_authority/Ards and North Down" + ) + assert us["state/ca"].v1_code == "state/ca" + assert us["congressional_district/CA-01"].v1_code == "congressional_district/CA-01" + + +def test_uk_constituencies_and_local_authorities_use_ons_codes_from_v1_lookups() -> ( + None +): + uk = load_region_references("uk") + for region_type, filename in ( + ("constituency", "constituencies_2024.csv"), + ("local_authority", "local_authorities_2021.csv"), + ): + expected = { + f"{region_type}/{row['code']}": ( + row["name"], + f"{region_type}/{row['name']}", + row["code"], + NATION_BY_ONS_PREFIX[row["code"][0]], + ) + for row in _v1_lookup_rows(filename) + } + actual = { + reference.code: ( + reference.label, + reference.v1_code, + reference.filter_value, + reference.parent_code, + ) + for reference in uk + if reference.region_type == region_type + } + assert actual == expected + + +def test_uk_nations_select_their_stored_regions() -> None: + uk = _by_code(load_region_references("uk")) + + assert json.loads(uk["country/england"].filter_value or "") == ( + ENGLISH_ITL1_REGIONS + ) + assert uk["country/scotland"].filter_value == "SCOTLAND" + assert uk["country/wales"].filter_value == "WALES" + assert uk["country/northern_ireland"].filter_value == "NORTHERN_IRELAND" + assert all(uk[code].parent_code == "uk" for code in NATION_BY_ONS_PREFIX.values()) + + +def test_us_states_and_districts_match_v1_and_select_by_fips_and_geoid() -> None: + us = load_region_references("us") + states = { + reference.state_code: reference + for reference in us + if reference.region_type == "state" + } + districts = { + reference.code: reference + for reference in us + if reference.region_type == "congressional_district" + } + + assert {code: state.label for code, state in states.items()} == (STATE_CODE_TO_NAME) + assert all(state.code == f"state/{code.lower()}" for code, state in states.items()) + assert all(state.parent_code == "us" for state in states.values()) + assert states["CA"].filter_value == "6" + assert states["DC"].filter_value == "11" + + v1_districts = { + entry["name"]: entry for entry in build_congressional_district_metadata() + } + assert set(districts) == set(v1_districts) + for district in CONGRESSIONAL_DISTRICTS: + code = f"congressional_district/{district.state_code}-{district.number:02d}" + reference = districts[code] + number = 0 if district.state_code in AT_LARGE_STATES else district.number + state_fips = int(states[district.state_code].filter_value or "") + assert reference.label == v1_districts[code]["label"] + assert reference.parent_code == f"state/{district.state_code.lower()}" + assert reference.state_code == district.state_code + assert reference.state_name == STATE_CODE_TO_NAME[district.state_code] + assert reference.filter_value == str(state_fips * 100 + number) + assert districts["congressional_district/CA-01"].filter_value == "601" + assert districts["congressional_district/AK-01"].filter_value == "200" + assert districts["congressional_district/DC-01"].filter_value == "1100" + + +def test_us_places_are_listed_under_their_state_without_a_filter() -> None: + places = [ + reference + for reference in load_region_references("us") + if reference.region_type == "place" + ] + + assert places + assert all(place.filter_value is None for place in places) + assert all( + place.parent_code == f"state/{(place.state_code or '').lower()}" + for place in places + ) + assert all(place.v1_code == place.code for place in places) + + +def test_parses_a_valid_minimal_file() -> None: + references = parse_region_references("us", _small_us_rows()) + + assert [reference.code for reference in references] == [ + "us", + "state/ca", + "congressional_district/CA-01", + ] + assert references[1] == RegionReference( + code="state/ca", + label="State/Ca", + region_type="state", + parent_code="us", + state_code=None, + state_name=None, + filter_value="6", + v1_code="state/ca", + ) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ( + lambda rows: rows.append(_row("state/ca", "state", parent_code="us")), + r"^us region reference file repeats region codes: \['state/ca'\]$", + ), + ( + lambda rows: rows[2].update(v1_code="state/ca"), + r"^us region reference file repeats v1 identifiers: \['state/ca'\]$", + ), + ( + lambda rows: rows[2].update(parent_code="state/ny"), + r"^region 'congressional_district/CA-01' names an unknown parent " + r"'state/ny'$", + ), + ( + lambda rows: rows.pop(0), + r"^us region reference file has no national region 'us'$", + ), + ( + lambda rows: rows[1].update(region_type="place"), + r"^region 'state/ca' does not start with its type 'place/'$", + ), + ( + lambda rows: rows.append(_row("county/06037", "county", parent_code="us")), + r"^region 'county/06037' has unsupported type 'county'$", + ), + ( + lambda rows: rows[1].update(parent_code="congressional_district/CA-01"), + r"^region 'state/ca' has a parent cycle$", + ), + ( + lambda rows: rows[0].update(parent_code="state/ca"), + r"^national region 'us' must have no parent$", + ), + ( + lambda rows: rows[1].update(parent_code=""), + r"^region 'state/ca' has no parent$", + ), + ( + lambda rows: rows[1].update(filter_value="x" * 256), + r"^region 'state/ca' filter_value exceeds 255 characters$", + ), + ( + lambda rows: rows[1].update(label=" California"), + r"^region 'state/ca' label has surrounding whitespace$", + ), + ( + lambda rows: rows[1].update(label=""), + r"^region 'state/ca' label must be a non-empty string$", + ), + ], +) +def test_rejects_an_inconsistent_reference_file(change, message: str) -> None: + rows = _small_us_rows() + change(rows) + + with pytest.raises(CatalogExtractionError, match=message): + parse_region_references("us", rows) + + +def test_rejects_a_file_whose_columns_differ(tmp_path: Path) -> None: + (tmp_path / "us.csv").write_text( + "code,label,region_type\nus,United States,national\n", encoding="utf-8" + ) + + with pytest.raises( + CatalogExtractionError, + match=r"^us region reference file columns must be ", + ): + load_region_references("us", directory=tmp_path) + + +def test_rejects_a_country_without_a_reference_file(tmp_path: Path) -> None: + with pytest.raises( + CatalogExtractionError, + match=r"^ca has no region reference file$", + ): + load_region_references("ca", directory=tmp_path) From f50890fa2d01fbc3a92356acd616fc6732852eb5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:07:14 +0400 Subject: [PATCH 019/118] Publish catalog regions from the reference files and the certified data release Catalog extraction no longer reads PolicyEngine.py's region registry. It takes every region from the checked-in reference file of its country and gives it a dataset and filter from a scoping rule per country and region type: - US national, and UK national: the release's national dataset, no filter. - US states and congressional districts: the release's state and district dataset, a row filter on state_fips or congressional_district_geoid. - UK nations: the national dataset, a row filter on the stored region. - UK constituencies and local authorities: the national dataset, a weight_replacement filter whose field names the release's weight matrix (parliamentary_constituency_weights, local_authority_weights) and whose value is the region's ONS code. The matrix must come from the same release as the dataset, since weights attach to households by position. Extraction fails if the national region would not use the reviewed dataset, a dataset role names no certified dataset or a per-region template, a filtered region has no filter value or a malformed list, or a runnable region sits under one that is left out. Region types without a rule (US places) are left out and counted in the publication evidence. The installed 6.7.5 catalog now publishes 1,503 regions: 488 US and 1,015 UK. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-123-region-reference-files.changed.md | 1 + .../data/v2/catalog/extraction.py | 307 ++++------------- .../data/v2/catalog/publication.py | 5 + .../data/v2/catalog/publication_types.py | 9 + policyengine_api/data/v2/catalog/records.py | 16 +- .../data/v2/catalog/region_scoping.py | 313 ++++++++++++++++++ policyengine_api/data/v2/models/metadata.py | 7 +- .../services/v2/metadata/types.py | 8 +- tests/fixtures/v2_catalog.py | 226 ++++++++----- .../integration/test_v2_catalog_installed.py | 36 +- .../test_v2_catalog_publication.py | 10 +- tests/unit/v2/test_catalog_extraction.py | 227 +++++++------ .../v2/test_catalog_extraction_failures.py | 198 ++++++++--- tests/unit/v2/test_catalog_publication.py | 4 + 14 files changed, 868 insertions(+), 499 deletions(-) create mode 100644 changelog.d/v2plan-1f-123-region-reference-files.changed.md create mode 100644 policyengine_api/data/v2/catalog/region_scoping.py diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md new file mode 100644 index 000000000..589a50194 --- /dev/null +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -0,0 +1 @@ +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 99ac8086b..76895175e 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -2,16 +2,13 @@ from __future__ import annotations -import json import math import re -from collections import Counter from collections.abc import Callable, Mapping, Sequence from datetime import UTC, date, datetime, timedelta from enum import Enum from importlib import metadata as importlib_metadata -from typing import TYPE_CHECKING, Any, NamedTuple, Protocol -from urllib.parse import urlsplit +from typing import TYPE_CHECKING, Any from uuid import NAMESPACE_URL, UUID, uuid5 from policyengine_api.data.v2.catalog.errors import CatalogExtractionError @@ -22,6 +19,7 @@ ModelRecord, ModelVersionRecord, NormalizedCatalog, + OmittedRegionSummary, ParameterNodeRecord, ParameterRecord, ParameterValueRecord, @@ -29,8 +27,12 @@ VariableRecord, ) from policyengine_api.data.v2.catalog.region_references import ( - MAX_REGION_FILTER_VALUE_LENGTH, - SUPPORTED_REGION_TYPES, + RegionReference, + load_region_references, +) +from policyengine_api.data.v2.catalog.region_scoping import ( + ScopedRegion, + scope_regions, ) if TYPE_CHECKING: @@ -88,40 +90,6 @@ def _optional_text( return value -def _serialize_region_filter_value(value: object, *, region_code: str) -> str: - """Keep scalar text or encode flat membership values in the existing column. - - Since PolicyEngine 6.2.7, the England filter names all nine English regions. The - catalog's existing text column can retain that complete list as a JSON - array without changing its schema or the representation of scalar filters. - """ - scalar_types = (str, int, float, bool) - if isinstance(value, scalar_types): - serialized = str(value) - elif ( - isinstance(value, (list, tuple)) - and value - and all(isinstance(item, scalar_types) for item in value) - ): - try: - serialized = json.dumps( - value, ensure_ascii=False, separators=(",", ":"), allow_nan=False - ) - except ValueError as error: - msg = f"region {region_code!r} has an unsupported filter value" - raise CatalogExtractionError(msg) from error - else: - msg = f"region {region_code!r} has an unsupported filter value" - raise CatalogExtractionError(msg) - if len(serialized) > MAX_REGION_FILTER_VALUE_LENGTH: - msg = ( - f"region {region_code!r} filter value exceeds " - f"{MAX_REGION_FILTER_VALUE_LENGTH} characters" - ) - raise CatalogExtractionError(msg) - return serialized - - def _type_name(value: object) -> str | None: if value is None: return None @@ -221,17 +189,6 @@ def _optional_aware_datetime(value: object, *, field_name: str) -> datetime | No return _aware_datetime(value, field_name=field_name) -def _dataset_name_from_path(path: str) -> str: - without_revision = path.split("@", maxsplit=1)[0] - parsed_path = urlsplit(without_revision).path or without_revision - filename = parsed_path.rstrip("/").rsplit("/", maxsplit=1)[-1] - for suffix in (".hdf5", ".parquet", ".csv", ".h5"): - if filename.endswith(suffix): - filename = filename[: -len(suffix)] - break - return _required_text(filename, field_name="dataset name", maximum=255) - - def _dataset_year(name: str) -> int: matches = DATASET_YEAR_PATTERN.findall(name) if not matches: @@ -477,18 +434,6 @@ def _public_named_records( return tuple(normalized) -class _RegionDatasetSelection(NamedTuple): - """The logical input datasets a country's regions select. - - Names every selected dataset, each region code's dataset, and how many - regions of each type fall back to the reviewed default dataset. - """ - - dataset_names: set[str] - region_dataset_names: dict[str, str] - fallback_counts: Counter[str] - - def _verified_model_name(country_id: str, source_model: object) -> str: """Return the public model's name once it matches the country's model.""" expected_model_name = f"policyengine-{country_id}" @@ -580,184 +525,39 @@ def _normalize_parameter_nodes( return tuple(normalized_nodes) -def _verified_default_dataset( - country_id: str, - release_manifest: object, -) -> tuple[str, object]: - """Return the reviewed default dataset and the manifest's URI for it. - - The manifest's default dataset must be the reviewed one. - """ +def _verified_default_dataset(country_id: str, release_manifest: object) -> str: + """Return the reviewed default dataset once the manifest names it as its default.""" reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] manifest_default = getattr(release_manifest, "default_dataset", None) if manifest_default != reviewed_default: msg = f"{country_id} default dataset does not match the reviewed selection" raise CatalogExtractionError(msg) - return reviewed_default, getattr(release_manifest, "default_dataset_uri", None) - - -def _verified_source_regions(country_id: str, source: object) -> tuple[Any, ...]: - """Return the country's public regions once their codes are unique and complete. - - The registry must belong to the country, and its codes must be unique and - include the national region. - """ - registry = getattr(source, "region_registry", None) - if registry is None or getattr(registry, "country_id", None) != country_id: - msg = f"{country_id} public region registry is absent" - raise CatalogExtractionError(msg) - - source_regions = tuple(registry) - region_codes = [getattr(region, "code", None) for region in source_regions] - duplicate_region_codes = sorted( - code for code, count in Counter(region_codes).items() if count > 1 - ) - if duplicate_region_codes: - msg = f"duplicate region natural keys: {duplicate_region_codes[:3]}" - raise CatalogExtractionError(msg) - if country_id not in region_codes: - msg = f"{country_id} national region is absent" - raise CatalogExtractionError(msg) - return source_regions - - -def _select_region_datasets( - country_id: str, - source_regions: tuple[Any, ...], - *, - reviewed_default: str, - default_dataset_uri: object, -) -> _RegionDatasetSelection: - """Choose each region's logical input dataset. - - A subnational region with a dataset path uses that dataset: US states and - districts, and the UK constituencies and local authorities that .py routes - to its local-area release. Every other region, such as a UK country that - filters the national file, uses the reviewed default, and the national - region may only name it. A US subnational region without a path is - counted as a fallback. - """ - selected_dataset_names: set[str] = {reviewed_default} - region_dataset_names: dict[str, str] = {} - fallback_counts: Counter[str] = Counter() - for region in source_regions: - code = _required_text( - getattr(region, "code", None), - field_name="region code", - maximum=255, - ) - region_type = _required_text( - getattr(region, "region_type", None), - field_name=f"region {code} type", - maximum=64, - ) - if region_type not in SUPPORTED_REGION_TYPES: - msg = f"region {code!r} has unsupported type {region_type!r}" - raise CatalogExtractionError(msg) - dataset_path = getattr(region, "dataset_path", None) - if region_type != "national" and dataset_path: - dataset_name = _dataset_name_from_path(dataset_path) - selected_dataset_names.add(dataset_name) - else: - dataset_name = reviewed_default - if country_id == "us" and region_type != "national" and not dataset_path: - fallback_counts[region_type] += 1 - if region_type == "national" and dataset_path not in { - None, - default_dataset_uri, - }: - msg = f"{country_id} national region does not use the reviewed dataset" - raise CatalogExtractionError(msg) - region_dataset_names[code] = dataset_name - return _RegionDatasetSelection( - dataset_names=selected_dataset_names, - region_dataset_names=region_dataset_names, - fallback_counts=fallback_counts, - ) - - -def _region_filter(strategy: object, code: str) -> tuple[str, str, str]: - """Return a filtered region's strategy type, filter field and filter value.""" - strategy_type = _required_text( - getattr(strategy, "strategy_type", None), - field_name=f"region {code} filter strategy", - maximum=64, - ) - filter_field = _required_text( - getattr(strategy, "variable_name", None), - field_name=f"region {code} filter field", - maximum=128, - ) - filter_value = _serialize_region_filter_value( - getattr(strategy, "variable_value", None), region_code=code - ) - additional_filters = getattr(strategy, "additional_filters", {}) - if additional_filters: - msg = f"region {code!r} has unsupported additional filters" - raise CatalogExtractionError(msg) - return strategy_type, filter_field, filter_value - - -class _SourceRegion(Protocol): - """The attributes of a PolicyEngine.py region that _normalize_region reads directly. - - It reads every other region attribute through getattr, with a default. - """ - - @property - def code(self) -> object: ... - - @property - def region_type(self) -> object: ... + return reviewed_default def _normalize_region( - source_region: _SourceRegion, + scoped_region: ScopedRegion, *, model_version_id: UUID, - region_dataset_ids: Mapping[str, UUID], + dataset_ids: Mapping[str, UUID], ) -> RegionRecord: - """Normalize one public region, with its filter and default dataset.""" - code = str(source_region.code) - strategy = getattr(source_region, "scoping_strategy", None) - requires_filter = bool(getattr(source_region, "requires_filter", False)) - if requires_filter: - strategy_type, filter_field, filter_value = _region_filter(strategy, code) - else: - strategy_type = None - filter_field = None - filter_value = None - + """Build one region's record from its reference, dataset and filter.""" + reference = scoped_region.reference + requires_filter = scoped_region.filter_strategy is not None return RegionRecord( - id=_identifier("region", model_version_id, code), + id=_identifier("region", model_version_id, reference.code), model_version_id=model_version_id, - default_dataset_id=region_dataset_ids[code], - code=code, - label=_required_text( - getattr(source_region, "label", None), - field_name=f"region {code} label", - maximum=255, - ), - region_type=str(source_region.region_type), + default_dataset_id=dataset_ids[scoped_region.dataset_name], + code=reference.code, + label=reference.label, + region_type=reference.region_type, requires_filter=requires_filter, - filter_field=filter_field, - filter_value=filter_value, - filter_strategy=strategy_type, - parent_code=_optional_text( - getattr(source_region, "parent_code", None), - field_name=f"region {code} parent_code", - maximum=255, - ), - state_code=_optional_text( - getattr(source_region, "state_code", None), - field_name=f"region {code} state_code", - maximum=16, - ), - state_name=_optional_text( - getattr(source_region, "state_name", None), - field_name=f"region {code} state_name", - maximum=128, - ), + filter_field=scoped_region.filter_field, + filter_value=reference.filter_value if requires_filter else None, + filter_strategy=scoped_region.filter_strategy, + parent_code=reference.parent_code, + state_code=reference.state_code, + state_name=reference.state_name, ) @@ -767,6 +567,7 @@ def _normalize_country( source: object, policyengine_version: str, expected_country_package_version: str, + region_references: tuple[RegionReference, ...], ) -> CountryCatalog: source_model = getattr(source, "model", None) model_name = _verified_model_name(country_id, source_model) @@ -799,15 +600,12 @@ def _normalize_country( ) ) - reviewed_default, default_dataset_uri = _verified_default_dataset( - country_id, release_manifest - ) - source_regions = _verified_source_regions(country_id, source) - dataset_selection = _select_region_datasets( + reviewed_default = _verified_default_dataset(country_id, release_manifest) + scoping = scope_regions( country_id, - source_regions, + region_references, + release_manifest, reviewed_default=reviewed_default, - default_dataset_uri=default_dataset_uri, ) datasets = tuple( @@ -818,20 +616,16 @@ def _normalize_country( description=f"PolicyEngine.py logical input dataset {name}", year=_dataset_year(name), ) - for name in sorted(dataset_selection.dataset_names) + for name in sorted(scoping.dataset_names) ) dataset_ids = {dataset.name: dataset.id for dataset in datasets} - region_dataset_ids = { - code: dataset_ids[name] - for code, name in dataset_selection.region_dataset_names.items() - } regions = [ _normalize_region( - source_region, + scoped_region, model_version_id=model_version_id, - region_dataset_ids=region_dataset_ids, + dataset_ids=dataset_ids, ) - for source_region in source_regions + for scoped_region in scoping.regions ] return CountryCatalog( @@ -863,7 +657,11 @@ def _normalize_country( regions=tuple(sorted(regions, key=lambda record: record.code)), fallback_summaries=tuple( FallbackSummary(region_type=region_type, count=count) - for region_type, count in sorted(dataset_selection.fallback_counts.items()) + for region_type, count in scoping.fallback_counts.items() + ), + omitted_region_summaries=tuple( + OmittedRegionSummary(region_type=region_type, count=count) + for region_type, count in scoping.omitted_counts.items() ), ) @@ -873,9 +671,14 @@ def extract_catalog( bundle: Mapping[str, Any], policyengine_version: str, models: Mapping[str, object], + region_references: Mapping[str, Sequence[RegionReference]], installed_version: Callable[[str], str] = importlib_metadata.version, ) -> NormalizedCatalog: - """Validate and normalize supplied PolicyEngine.py public objects.""" + """Validate and normalize supplied PolicyEngine.py public objects. + + Regions come from the supplied reference files, scoped by each country's + certified data release. + """ dependencies = _verify_bundle( bundle=bundle, policyengine_version=policyengine_version, @@ -885,6 +688,12 @@ def extract_catalog( if missing: msg = f"PolicyEngine.py public models are absent for: {', '.join(missing)}" raise CatalogExtractionError(msg) + missing = [ + country for country in SUPPORTED_COUNTRIES if country not in region_references + ] + if missing: + msg = f"region references are absent for: {', '.join(missing)}" + raise CatalogExtractionError(msg) package_map = bundle["packages"] countries = tuple( @@ -895,6 +704,7 @@ def extract_catalog( expected_country_package_version=package_map[f"policyengine-{country_id}"][ "version" ], + region_references=tuple(region_references[country_id]), ) for country_id in SUPPORTED_COUNTRIES ) @@ -906,7 +716,10 @@ def extract_catalog( def extract_installed_catalog() -> NormalizedCatalog: - """Load and normalize the installed PolicyEngine.py certified catalog.""" + """Load and normalize the installed PolicyEngine.py certified catalog. + + Regions come from the Public API's checked-in reference files. + """ import policyengine from policyengine.bundle import ( get_current_bundle, @@ -920,4 +733,8 @@ def extract_installed_catalog() -> NormalizedCatalog: "us": policyengine.us.model, "uk": policyengine.uk.model, }, + region_references={ + country_id: load_region_references(country_id) + for country_id in SUPPORTED_COUNTRIES + }, ) diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 5e629fb05..88da73310 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -137,6 +137,11 @@ def publish_catalog( entity_counts=catalog.entity_counts(), fallback_summaries=fallback_summaries, elapsed_seconds=time.monotonic() - started, + omitted_region_summaries=tuple( + (country.country_id, summary.region_type, summary.count) + for country in catalog.countries + for summary in country.omitted_region_summaries + ), ) diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index 884bcef07..51865f7c7 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -18,6 +18,7 @@ class PublicationEvidence: entity_counts: dict[str, int] fallback_summaries: tuple[tuple[str, str, int], ...] elapsed_seconds: float + omitted_region_summaries: tuple[tuple[str, str, int], ...] = () def as_dict(self) -> dict[str, object]: """Return the evidence as the JSON object printed after a publication.""" @@ -34,5 +35,13 @@ def as_dict(self) -> dict[str, object]: } for country_id, region_type, count in self.fallback_summaries ], + "omitted_region_summaries": [ + { + "country_id": country_id, + "region_type": region_type, + "count": count, + } + for country_id, region_type, count in self.omitted_region_summaries + ], "elapsed_seconds": round(self.elapsed_seconds, 3), } diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index afccc4add..5be00f616 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -128,9 +128,20 @@ class RegionRecord: @dataclass(frozen=True, slots=True) class FallbackSummary: - """How many US sub-national regions of one type fell back to the default dataset. + """How many sub-national regions of one type fell back to the default dataset. - These regions had no dataset path of their own. + The certified data release names no dataset for their type. + """ + + region_type: str + count: int + + +@dataclass(frozen=True, slots=True) +class OmittedRegionSummary: + """How many reference regions of one type publication left out. + + No scoping rule can select their households yet, such as US places. """ region_type: str @@ -166,6 +177,7 @@ class CountryCatalog: datasets: tuple[DatasetRecord, ...] regions: tuple[RegionRecord, ...] fallback_summaries: tuple[FallbackSummary, ...] + omitted_region_summaries: tuple[OmittedRegionSummary, ...] = () def parameter_value_batches( self, diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py new file mode 100644 index 000000000..d3ea6629d --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -0,0 +1,313 @@ +"""Give each reference region its dataset and filter from the certified data release. + +A scoping rule per country and region type names the release's dataset role +(a key of the release manifest's ``region_datasets``) and how the region +selects its households: a row filter on one column of that dataset, or a +weight-replacement filter whose field names the release's weight-matrix +artifact and whose value is the region's row code in it. Region types without +a rule cannot run yet, so publication leaves them out and counts them. +""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.region_references import RegionReference + +ROW_FILTER = "row_filter" +WEIGHT_REPLACEMENT = "weight_replacement" +NATIONAL_DATASET_ROLE = "national" + + +@dataclass(frozen=True, slots=True) +class RegionScopingRule: + """How regions of one type find their dataset and select their households. + + ``filter_field`` is the dataset column of a row filter, or the release's + weight-matrix artifact name of a weight-replacement filter. + """ + + dataset_role: str + filter_strategy: str | None = None + filter_field: str | None = None + + +REGION_SCOPING_RULES: Mapping[tuple[str, str], RegionScopingRule] = { + ("us", "national"): RegionScopingRule(NATIONAL_DATASET_ROLE), + ("us", "state"): RegionScopingRule("state", ROW_FILTER, "state_fips"), + ("us", "congressional_district"): RegionScopingRule( + "congressional_district", ROW_FILTER, "congressional_district_geoid" + ), + ("uk", "national"): RegionScopingRule(NATIONAL_DATASET_ROLE), + ("uk", "country"): RegionScopingRule(NATIONAL_DATASET_ROLE, ROW_FILTER, "region"), + ("uk", "constituency"): RegionScopingRule( + NATIONAL_DATASET_ROLE, + WEIGHT_REPLACEMENT, + "parliamentary_constituency_weights", + ), + ("uk", "local_authority"): RegionScopingRule( + NATIONAL_DATASET_ROLE, WEIGHT_REPLACEMENT, "local_authority_weights" + ), +} + + +@dataclass(frozen=True, slots=True) +class CertifiedArtifact: + """One artifact of a country's certified data release, pinned to a revision.""" + + name: str + repo_id: str + repo_type: str + path: str + revision: str + sha256: str | None + + +@dataclass(frozen=True, slots=True) +class ScopedRegion: + """A reference region with its dataset and the filter selecting its households.""" + + reference: RegionReference + dataset_name: str + filter_strategy: str | None + filter_field: str | None + + +@dataclass(frozen=True, slots=True) +class RegionScoping: + """Every runnable region of one country, and what the others fell back to. + + ``fallback_counts`` counts, by region type, regions whose dataset role the + release does not name, so they use the default dataset; ``omitted_counts`` + counts the regions left out because their type cannot run yet. + """ + + regions: tuple[ScopedRegion, ...] + dataset_names: frozenset[str] + certified_artifacts: Mapping[str, CertifiedArtifact] + fallback_counts: Mapping[str, int] + omitted_counts: Mapping[str, int] + + +def _manifest_text(value: object, *, description: str) -> str: + if not isinstance(value, str) or not value: + msg = f"{description} must be a non-empty string" + raise CatalogExtractionError(msg) + return value + + +def certified_artifacts( + country_id: str, + release_manifest: object, +) -> dict[str, CertifiedArtifact]: + """Return the release manifest's artifacts, with the data package's defaults. + + An artifact without its own repository, repository type or revision is in + the data package's repository at the release revision. + """ + data_package = getattr(release_manifest, "data_package", None) + default_repo_id = getattr(data_package, "repo_id", None) + default_repo_type = getattr(data_package, "repo_type", None) or "model" + default_revision = getattr( + data_package, "release_manifest_revision", None + ) or getattr(data_package, "version", None) + datasets = getattr(release_manifest, "datasets", None) + if not isinstance(datasets, Mapping): + msg = f"{country_id} release manifest has no dataset map" + raise CatalogExtractionError(msg) + artifacts: dict[str, CertifiedArtifact] = {} + for name, reference in datasets.items(): + description = f"{country_id} release artifact {name}" + artifacts[name] = CertifiedArtifact( + name=_manifest_text(name, description=f"{country_id} release artifact"), + repo_id=_manifest_text( + getattr(reference, "repo_id", None) or default_repo_id, + description=f"{description} repository", + ), + repo_type=_manifest_text( + getattr(reference, "repo_type", None) or default_repo_type, + description=f"{description} repository type", + ), + path=_manifest_text( + getattr(reference, "path", None), description=f"{description} path" + ), + revision=_manifest_text( + getattr(reference, "revision", None) or default_revision, + description=f"{description} revision", + ), + sha256=getattr(reference, "sha256", None), + ) + return artifacts + + +def _role_datasets( + country_id: str, + release_manifest: object, + artifacts: Mapping[str, CertifiedArtifact], +) -> dict[str, str]: + """Map each dataset role the release names to its certified dataset's name.""" + region_datasets = getattr(release_manifest, "region_datasets", None) + if not isinstance(region_datasets, Mapping): + msg = f"{country_id} release manifest has no region dataset map" + raise CatalogExtractionError(msg) + names_by_path = {artifact.path: name for name, artifact in artifacts.items()} + role_datasets: dict[str, str] = {} + for role, template in region_datasets.items(): + path = _manifest_text( + getattr(template, "path_template", None), + description=f"{country_id} {role} region dataset", + ) + if "{" in path: + msg = ( + f"{country_id} {role} regions name a per-region dataset template " + f"{path!r}, which the catalog does not support" + ) + raise CatalogExtractionError(msg) + if path not in names_by_path: + msg = ( + f"{country_id} release names no certified dataset at path {path!r} " + f"for {role} regions" + ) + raise CatalogExtractionError(msg) + role_datasets[role] = names_by_path[path] + return role_datasets + + +def _checked_filter_value(reference: RegionReference, rule: RegionScopingRule) -> None: + """Require a filter value exactly when the rule filters, and valid list values.""" + value = reference.filter_value + if rule.filter_strategy is None: + if value is not None: + msg = ( + f"region {reference.code!r} has a filter value, but a national region " + "selects every household" + ) + raise CatalogExtractionError(msg) + return + if value is None: + msg = f"region {reference.code!r} has no filter value" + raise CatalogExtractionError(msg) + if value.startswith(("[", "{")): + try: + members = json.loads(value, parse_constant=_reject_constant) + except ValueError: + members = None + if not ( + isinstance(members, list) + and members + and all(isinstance(member, (str, int, float, bool)) for member in members) + ): + msg = f"region {reference.code!r} has an unsupported filter value" + raise CatalogExtractionError(msg) + + +def _reject_constant(constant: str) -> object: + msg = f"{constant} is not a filter value" + raise ValueError(msg) + + +def _checked_weight_matrix( + country_id: str, + reference: RegionReference, + matrix_name: str, + dataset_name: str, + artifacts: Mapping[str, CertifiedArtifact], +) -> None: + """Require the weight matrix from the same release as the dataset it reweights. + + Weights attach to households by position, so only a matrix built with the + region's dataset lines up with its households. + """ + matrix = artifacts.get(matrix_name) + if matrix is None: + msg = ( + f"region {reference.code!r} reweights households with {matrix_name}, " + f"which the {country_id} release does not certify" + ) + raise CatalogExtractionError(msg) + dataset = artifacts[dataset_name] + if (matrix.repo_id, matrix.revision) != (dataset.repo_id, dataset.revision): + msg = ( + f"region {reference.code!r} reweights households with {matrix_name}, " + f"which is not certified in the release of {dataset_name}" + ) + raise CatalogExtractionError(msg) + + +def scope_regions( + country_id: str, + references: tuple[RegionReference, ...], + release_manifest: object, + *, + reviewed_default: str, +) -> RegionScoping: + """Give every runnable reference region its certified dataset and filter. + + The national region must use the reviewed default dataset. A region whose + type has no scoping rule is left out, and a runnable region may not sit + under a region that is left out. + """ + artifacts = certified_artifacts(country_id, release_manifest) + if reviewed_default not in artifacts: + msg = f"{country_id} release does not certify {reviewed_default}" + raise CatalogExtractionError(msg) + role_datasets = _role_datasets(country_id, release_manifest, artifacts) + if role_datasets.get(NATIONAL_DATASET_ROLE, reviewed_default) != reviewed_default: + msg = f"{country_id} national region does not use the reviewed dataset" + raise CatalogExtractionError(msg) + + omitted = { + reference.code: reference.region_type + for reference in references + if (country_id, reference.region_type) not in REGION_SCOPING_RULES + } + fallback_counts: Counter[str] = Counter() + regions: list[ScopedRegion] = [] + for reference in references: + if reference.code in omitted: + continue + if reference.parent_code in omitted: + msg = ( + f"region {reference.code!r} cannot be published because its parent " + f"{reference.parent_code!r} is left out" + ) + raise CatalogExtractionError(msg) + rule = REGION_SCOPING_RULES[country_id, reference.region_type] + _checked_filter_value(reference, rule) + dataset_name = role_datasets.get(rule.dataset_role) + if dataset_name is None: + dataset_name = reviewed_default + if rule.dataset_role != NATIONAL_DATASET_ROLE: + fallback_counts[reference.region_type] += 1 + if rule.filter_strategy == WEIGHT_REPLACEMENT: + _checked_weight_matrix( + country_id, + reference, + rule.filter_field or "", + dataset_name, + artifacts, + ) + regions.append( + ScopedRegion( + reference=reference, + dataset_name=dataset_name, + filter_strategy=rule.filter_strategy, + filter_field=rule.filter_field, + ) + ) + return RegionScoping( + regions=tuple(regions), + dataset_names=frozenset( + {reviewed_default, *(region.dataset_name for region in regions)} + ), + certified_artifacts=artifacts, + fallback_counts=dict(sorted(fallback_counts.items())), + omitted_counts=dict(sorted(Counter(omitted.values()).items())), + ) diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 8d91864df..324ad521b 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -111,8 +111,11 @@ class Region(TimestampedModel, table=True): """A region in one model version, unique by code within that version. Its default dataset must belong to the same version. Filtered regions - (``requires_filter``) also name the variable and value that select their - households. + (``requires_filter``) also name how they select their households: a + ``row_filter`` keeps the households whose ``filter_field`` column matches + ``filter_value``; a ``weight_replacement`` filter reweights every household + with row ``filter_value`` of the weight matrix ``filter_field`` names in the + certified data release. """ __tablename__ = "regions" diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 7f41e1575..97ec2ebeb 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -136,8 +136,12 @@ class MetadataDataset(StrictResponseModel): class MetadataRegion(StrictResponseModel): """A region the model can simulate, such as a country, state or constituency. - When requires_filter is true, the region is simulated by filtering its - default dataset on filter_field and filter_value. + When requires_filter is true, filter_strategy says how the region selects + households of its default dataset. With "row_filter", it keeps the + households whose filter_field column equals filter_value (or is in it, + when filter_value is a JSON list). With "weight_replacement", it keeps + every household and gives each the weight in row filter_value of the + weight matrix that filter_field names in the certified data release. """ id: UUID diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 77f5cbd20..e3de5fc62 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -2,12 +2,12 @@ from __future__ import annotations -from dataclasses import dataclass from datetime import UTC, datetime from types import SimpleNamespace from typing import TYPE_CHECKING from policyengine_api.data.v2.catalog.extraction import extract_catalog +from policyengine_api.data.v2.catalog.region_references import RegionReference if TYPE_CHECKING: from policyengine_api.data.v2.catalog.records import NormalizedCatalog @@ -20,49 +20,139 @@ } -class RegionRegistry(list): - """Small iterable with the public registry's country identifier.""" +def region_references() -> dict[str, tuple[RegionReference, ...]]: + """Return small US and UK reference files. - def __init__(self, country_id: str, regions: list[SimpleNamespace]): - super().__init__(regions) - self.country_id = country_id + The US place has no filter rule, so publication leaves it out. + """ + return { + "us": ( + RegionReference( + code="us", + label="United States", + region_type="national", + parent_code=None, + state_code=None, + state_name=None, + filter_value=None, + v1_code="us", + ), + RegionReference( + code="state/ca", + label="California", + region_type="state", + parent_code="us", + state_code="CA", + state_name="California", + filter_value="6", + v1_code="state/ca", + ), + RegionReference( + code="congressional_district/CA-01", + label="California's 1st congressional district", + region_type="congressional_district", + parent_code="state/ca", + state_code="CA", + state_name="California", + filter_value="601", + v1_code="congressional_district/CA-01", + ), + RegionReference( + code="place/CA-44000", + label="Los Angeles", + region_type="place", + parent_code="state/ca", + state_code="CA", + state_name="California", + filter_value=None, + v1_code="place/CA-44000", + ), + ), + "uk": ( + RegionReference( + code="uk", + label="United Kingdom", + region_type="national", + parent_code=None, + state_code=None, + state_name=None, + filter_value=None, + v1_code="uk", + ), + RegionReference( + code="country/england", + label="England", + region_type="country", + parent_code="uk", + state_code=None, + state_name=None, + filter_value='["NORTH_EAST","LONDON"]', + v1_code="country/england", + ), + RegionReference( + code="constituency/E14001063", + label="Aldershot", + region_type="constituency", + parent_code="country/england", + state_code=None, + state_name=None, + filter_value="E14001063", + v1_code="constituency/Aldershot", + ), + ), + } -def _strategy(field: str, value: str | int) -> SimpleNamespace: +def _certified_artifact(path: str, *, revision: str = "fixture") -> SimpleNamespace: return SimpleNamespace( - strategy_type="row_filter", - variable_name=field, - variable_value=value, - additional_filters={}, + path=path, + revision=revision, + sha256="0" * 64, + repo_id=None, + repo_type=None, ) -@dataclass(frozen=True, kw_only=True) -class _Region: - """One public region; ``source`` gives the mutable object the registry holds.""" - - code: str - label: str - region_type: str - parent_code: str | None = None - dataset_path: str | None = None - strategy: SimpleNamespace | None = None - state_code: str | None = None - state_name: str | None = None - - def source(self) -> SimpleNamespace: - """Return the region as a registry entry; it needs a filter if scoped.""" - return SimpleNamespace( - code=self.code, - label=self.label, - region_type=self.region_type, - parent_code=self.parent_code, - dataset_path=self.dataset_path, - scoping_strategy=self.strategy, - requires_filter=self.strategy is not None, - state_code=self.state_code, - state_name=self.state_name, +def _release_contents(country_id: str) -> tuple[dict, dict, SimpleNamespace]: + """Return a country's certified datasets, region datasets and data package.""" + if country_id == "us": + datasets = { + "populace_us_2024": _certified_artifact("populace_us_2024.h5"), + "populace_us_2024_acs_local": _certified_artifact( + "populace_us_2024_acs_local.h5" + ), + } + region_datasets = { + "national": SimpleNamespace(path_template="populace_us_2024.h5"), + "state": SimpleNamespace(path_template="populace_us_2024_acs_local.h5"), + "congressional_district": SimpleNamespace( + path_template="populace_us_2024_acs_local.h5" + ), + } + data_package = SimpleNamespace( + repo_id="policyengine/populace-us", + repo_type="dataset", + version="0.1.0", + release_manifest_revision="fixture", ) + return datasets, region_datasets, data_package + datasets = { + "microcosm_uk_2024_25": _certified_artifact("microcosm_uk_2024_25.h5"), + "parliamentary_constituency_weights": _certified_artifact( + "parliamentary_constituency_weights.h5" + ), + "local_authority_weights": _certified_artifact("local_authority_weights.h5"), + } + region_datasets = { + "national": SimpleNamespace(path_template="microcosm_uk_2024_25.h5"), + } + data_package = SimpleNamespace( + repo_id="example-org/example-uk-data", + repo_type="model", + version="1.0.0", + release_manifest_revision="fixture", + ) + return datasets, region_datasets, data_package def bundle(*, policyengine_version: str = POLICYENGINE_VERSION) -> dict: @@ -88,7 +178,6 @@ def _model_source( country_id: str, country_package_version: str, policyengine_version: str, - regions: list[SimpleNamespace], ) -> SimpleNamespace: model_name = f"policyengine-{country_id}" default_dataset = { @@ -141,13 +230,16 @@ def _model_source( name=model_name, version=country_package_version, ) + datasets, region_datasets, data_package = _release_contents(country_id) version_holder.release_manifest = SimpleNamespace( country_id=country_id, policyengine_version=policyengine_version, default_dataset=default_dataset, default_dataset_uri=default_uri, + datasets=datasets, + region_datasets=region_datasets, + data_package=data_package, ) - version_holder.region_registry = RegionRegistry(country_id, regions) version_holder.variables = [variable] version_holder.parameter_nodes = [parameter_node] version_holder.parameters = [parameter] @@ -162,60 +254,13 @@ def source_models( policyengine_version: str = POLICYENGINE_VERSION, ) -> dict[str, SimpleNamespace]: """Return US and UK public-model fixtures.""" - us_default_uri = "hf://policyengine/us/populace_us_2024.h5@fixture" - uk_default_uri = "hf://policyengine/uk/microcosm_uk_2024_25.h5@fixture" return { - "us": _model_source( - country_id="us", - country_package_version=DEPENDENCY_VERSIONS["policyengine-us"], - policyengine_version=policyengine_version, - regions=[ - _Region( - code="us", - label="United States", - region_type="national", - dataset_path=us_default_uri, - ).source(), - _Region( - code="state/ca", - label="California", - region_type="state", - parent_code="us", - dataset_path="hf://policyengine/us/populace_us_ca_2024.h5@fixture", - strategy=_strategy("state_fips", 6), - state_code="CA", - state_name="California", - ).source(), - _Region( - code="place/CA-44000", - label="Los Angeles", - region_type="place", - parent_code="state/ca", - state_code="CA", - state_name="California", - ).source(), - ], - ), - "uk": _model_source( - country_id="uk", - country_package_version=DEPENDENCY_VERSIONS["policyengine-uk"], + country_id: _model_source( + country_id=country_id, + country_package_version=DEPENDENCY_VERSIONS[f"policyengine-{country_id}"], policyengine_version=policyengine_version, - regions=[ - _Region( - code="uk", - label="United Kingdom", - region_type="national", - dataset_path=uk_default_uri, - ).source(), - _Region( - code="country/england", - label="England", - region_type="country", - parent_code="uk", - strategy=_strategy("country", "ENGLAND"), - ).source(), - ], - ), + ) + for country_id in ("us", "uk") } @@ -234,4 +279,5 @@ def normalized_catalog( policyengine_version=policyengine_version, models=source_models(policyengine_version=policyengine_version), installed_version=installed_version, + region_references=region_references(), ) diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index 0f9106e06..778b72d07 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -4,6 +4,7 @@ import json import os +from collections import Counter from importlib import metadata as importlib_metadata import pytest @@ -49,8 +50,8 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "parameter_nodes": 29_307, "parameters": 104_172, "parameter_values": 1_194_360, - "datasets": 4, - "regions": 1_837, + "datasets": 3, + "regions": 1_503, } for country_id in ("us", "uk"): @@ -121,7 +122,8 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: # Since 6.7.5, US state and congressional-district regions use the # ACS-local dataset again, on the PolicyEngine/microcosm#1263 rebuild # (PolicyEngine/policyengine.py#649, ending #604's national-file filter). - # The national region uses the national dataset, and places fall back to it. + # The national region uses the national dataset. Regions come from the + # checked-in reference files; US places cannot run yet and are left out. us_catalog = catalog.country("us") us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets} region_type_datasets: dict[str, set[str]] = {} @@ -133,11 +135,29 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "national": {"populace_us_2024"}, "state": {"populace_us_2024_acs_local"}, "congressional_district": {"populace_us_2024_acs_local"}, - "place": {"populace_us_2024"}, } + assert us_catalog.fallback_summaries == () assert [ (summary.region_type, summary.count) - for summary in us_catalog.fallback_summaries - ] == [ - ("place", 333), - ] + for summary in us_catalog.omitted_region_summaries + ] == [("place", 333)] + + uk_catalog = catalog.country("uk") + uk_types = Counter(region.region_type for region in uk_catalog.regions) + assert uk_types == { + "national": 1, + "country": 4, + "constituency": 650, + "local_authority": 360, + } + uk_regions = {region.code: region for region in uk_catalog.regions} + aldershot = uk_regions["constituency/E14001063"] + assert ( + aldershot.filter_strategy, + aldershot.filter_field, + aldershot.filter_value, + ) == ("weight_replacement", "parliamentary_constituency_weights", "E14001063") + assert uk_regions["uk"].label == "United Kingdom" + assert {region.code: region.label for region in us_catalog.regions}["us"] == ( + "United States" + ) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 25bc9178f..c894c8b45 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -121,6 +121,7 @@ def test_publish_and_republish_preserve_complete_catalog( second = publish_catalog(publication_engine, catalog) assert first.as_dict()["outcome"] == "ok" + assert first.omitted_region_summaries == (("us", "place", 1),) assert second.policyengine_version == first.policyengine_version assert ( _counts(publication_engine) @@ -134,7 +135,7 @@ def test_publish_and_republish_preserve_complete_catalog( "parameter_values": 4, "datasets": 3, "dataset_versions": 0, - "regions": 5, + "regions": 6, "simulations": 0, "reports": 0, "report_runs": 0, @@ -163,9 +164,10 @@ def test_publish_and_republish_preserve_complete_catalog( storage_path is None and not output for _, storage_path, output in datasets ) assert dict(region_defaults) == { + "congressional_district/CA-01": "populace_us_2024_acs_local", + "constituency/E14001063": "microcosm_uk_2024_25", "country/england": "microcosm_uk_2024_25", - "place/CA-44000": "populace_us_2024", - "state/ca": "populace_us_ca_2024", + "state/ca": "populace_us_2024_acs_local", "uk": "microcosm_uk_2024_25", "us": "populace_us_2024", } @@ -335,7 +337,7 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "parameter_values": 8, "datasets": 6, "dataset_versions": 0, - "regions": 10, + "regions": 12, "simulations": 0, "reports": 0, "report_runs": 0, diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index f88319cd8..8bd5cf519 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -3,7 +3,7 @@ from __future__ import annotations import ast -import json +from dataclasses import replace from datetime import UTC, datetime from importlib.metadata import PackageNotFoundError from pathlib import Path @@ -22,6 +22,7 @@ bundle, installed_version, normalized_catalog, + region_references, source_models, ) @@ -39,7 +40,7 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: assert {dataset.name for dataset in us.datasets} == { "populace_us_2024", - "populace_us_ca_2024", + "populace_us_2024_acs_local", } assert {dataset.name for dataset in uk.datasets} == {"microcosm_uk_2024_25"} assert all( @@ -52,74 +53,104 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: region.code: us_datasets[region.default_dataset_id] for region in us.regions } assert us_defaults == { - "place/CA-44000": "populace_us_2024", - "state/ca": "populace_us_ca_2024", + "congressional_district/CA-01": "populace_us_2024_acs_local", + "state/ca": "populace_us_2024_acs_local", "us": "populace_us_2024", } + assert us.fallback_summaries == () assert [ - (summary.region_type, summary.count) for summary in us.fallback_summaries + (summary.region_type, summary.count) for summary in us.omitted_region_summaries ] == [("place", 1)] + assert uk.omitted_region_summaries == () assert all(region.default_dataset_id == uk.datasets[0].id for region in uk.regions) -UK_DENSE_URI = "hf://policyengine/uk/microcosm_uk_2024_25_dense.h5@fixture" - +def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> None: + catalog = normalized_catalog() + regions = { + region.code: ( + region.label, + region.region_type, + region.parent_code, + region.requires_filter, + region.filter_strategy, + region.filter_field, + region.filter_value, + region.state_code, + region.state_name, + ) + for country_id in ("us", "uk") + for region in catalog.country(country_id).regions + } -def _uk_area_region(code: str, region_type: str, filter_field: str) -> SimpleNamespace: - """Return a UK area region routed to the local-area dataset, as .py 6.10 does.""" - return SimpleNamespace( - code=code, - label=code.split("/", maxsplit=1)[1], - region_type=region_type, - parent_code="uk", - dataset_path=UK_DENSE_URI, - scoping_strategy=SimpleNamespace( - strategy_type="row_filter", - variable_name=filter_field, - variable_value=code.split("/", maxsplit=1)[1], - additional_filters={}, + assert regions == { + "us": ("United States", "national", None, False, None, None, None, None, None), + "state/ca": ( + "California", + "state", + "us", + True, + "row_filter", + "state_fips", + "6", + "CA", + "California", ), - requires_filter=True, - state_code=None, - state_name=None, - ) + "congressional_district/CA-01": ( + "California's 1st congressional district", + "congressional_district", + "state/ca", + True, + "row_filter", + "congressional_district_geoid", + "601", + "CA", + "California", + ), + "uk": ("United Kingdom", "national", None, False, None, None, None, None, None), + "country/england": ( + "England", + "country", + "uk", + True, + "row_filter", + "region", + '["NORTH_EAST","LONDON"]', + None, + None, + ), + "constituency/E14001063": ( + "Aldershot", + "constituency", + "country/england", + True, + "weight_replacement", + "parliamentary_constituency_weights", + "E14001063", + None, + None, + ), + } -def test_uk_area_regions_use_the_dataset_their_path_names() -> None: +def test_sub_national_regions_without_a_release_dataset_use_the_default() -> None: models = source_models() - models["uk"].region_registry.extend( - [ - _uk_area_region( - "constituency/E14001063", "constituency", "constituency_code_oa" - ), - _uk_area_region( - "local_authority/E07000092", "local_authority", "la_code_oa" - ), - ] - ) + del models["us"].release_manifest.region_datasets["state"] + del models["us"].release_manifest.region_datasets["congressional_district"] - uk = extract_catalog( + us = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, installed_version=installed_version, - ).country("uk") + ).country("us") - assert {dataset.name for dataset in uk.datasets} == { - "microcosm_uk_2024_25", - "microcosm_uk_2024_25_dense", - } - uk_datasets = {dataset.id: dataset.name for dataset in uk.datasets} - assert { - region.code: uk_datasets[region.default_dataset_id] for region in uk.regions - } == { - "uk": "microcosm_uk_2024_25", - "country/england": "microcosm_uk_2024_25", - "constituency/E14001063": "microcosm_uk_2024_25_dense", - "local_authority/E07000092": "microcosm_uk_2024_25_dense", - } - # A UK country filters the national dataset by design; it is no fallback. - assert uk.fallback_summaries == () + assert {dataset.name for dataset in us.datasets} == {"populace_us_2024"} + assert all(region.default_dataset_id == us.datasets[0].id for region in us.regions) + assert [ + (summary.region_type, summary.count) for summary in us.fallback_summaries + ] == [("congressional_district", 1), ("state", 1)] def test_durable_ids_are_deterministic_and_stable_at_the_intended_scope() -> None: @@ -179,62 +210,56 @@ def item(self): assert normalize_json_value(float("-inf")) == "-Infinity" -@pytest.mark.parametrize("value", [("NORTH_EAST", "LONDON"), ["NORTH_EAST", "LONDON"]]) -def test_serializes_region_membership_filters_in_the_existing_text_field( - value: object, -) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value - - catalog = extract_catalog( - bundle=bundle(), - policyengine_version=POLICYENGINE_VERSION, - models=models, - installed_version=installed_version, - ) - england = next( - region - for region in catalog.country("uk").regions - if region.code == "country/england" +@pytest.mark.parametrize("value", ['["NORTH_EAST","LONDON"]', "ENGLAND", "6"]) +def test_keeps_reference_filter_values_as_text(value: str) -> None: + references = region_references() + england = references["uk"][1] + references["uk"] = ( + references["uk"][0], + replace(england, filter_value=value), + references["uk"][2], ) - assert isinstance(england.filter_value, str) - assert json.loads(england.filter_value) == ["NORTH_EAST", "LONDON"] - - -@pytest.mark.parametrize("value", ["ENGLAND", 6, True, 1.5]) -def test_preserves_existing_scalar_region_filter_serialization(value: object) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value - catalog = extract_catalog( + region_references=references, bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, - models=models, + models=source_models(), installed_version=installed_version, ) - england = next( + england_record = next( region for region in catalog.country("uk").regions if region.code == "country/england" ) - assert england.filter_value == str(value) + assert england_record.filter_value == value @pytest.mark.parametrize( "value", - [[], [["LONDON"]], [None], [object()], [float("nan")], ["x" * 256], "x" * 256], + ["[]", '[["LONDON"]]', "[null]", '[{"a": 1}]', "[NaN]", "[", '{"a": 1}'], ) -def test_rejects_invalid_or_oversized_region_membership_values(value: object) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value +def test_rejects_filter_value_lists_that_are_not_flat_json_scalars( + value: str, +) -> None: + references = region_references() + england = references["uk"][1] + references["uk"] = ( + references["uk"][0], + replace(england, filter_value=value), + references["uk"][2], + ) - with pytest.raises(CatalogExtractionError, match="filter value"): + with pytest.raises( + CatalogExtractionError, + match=r"^region 'country/england' has an unsupported filter value$", + ): extract_catalog( + region_references=references, bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, - models=models, + models=source_models(), installed_version=installed_version, ) @@ -250,6 +275,7 @@ def test_rejects_missing_or_mismatched_manifest_dependencies() -> None: del missing["packages"]["policyengine-core"] with pytest.raises(CatalogExtractionError, match="omits policyengine-core"): extract_catalog( + region_references=region_references(), bundle=missing, policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -258,6 +284,7 @@ def test_rejects_missing_or_mismatched_manifest_dependencies() -> None: with pytest.raises(CatalogExtractionError, match="does not match"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -271,6 +298,7 @@ def absent(_name: str) -> str: with pytest.raises(CatalogExtractionError, match="is absent"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -282,6 +310,7 @@ def absent(_name: str) -> str: def test_rejects_placeholder_policyengine_versions(version: str) -> None: with pytest.raises(CatalogExtractionError, match="placeholder"): extract_catalog( + region_references=region_references(), bundle=bundle(policyengine_version=version), policyengine_version=version, models=source_models(policyengine_version=version), @@ -294,6 +323,7 @@ def test_rejects_incomplete_public_model_catalogs() -> None: del models["uk"] with pytest.raises(CatalogExtractionError, match="absent for: uk"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -304,6 +334,7 @@ def test_rejects_incomplete_public_model_catalogs() -> None: models["us"].release_manifest.default_dataset = "unreviewed_us_2025" with pytest.raises(CatalogExtractionError, match="reviewed selection"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -321,6 +352,7 @@ def test_rejects_inconsistent_country_model_identity(identity_source: str) -> No with pytest.raises(CatalogExtractionError, match="identity does not match"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -335,6 +367,7 @@ def test_ignores_only_the_unnamed_structural_parameter_root() -> None: ) catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -352,6 +385,7 @@ def test_uses_and_validates_public_name_indexed_mappings() -> None: models["uk"].variables_by_name["wrong_name"] = variable with pytest.raises(CatalogExtractionError, match="does not match record name"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -359,33 +393,25 @@ def test_uses_and_validates_public_name_indexed_mappings() -> None: ) -def test_rejects_duplicate_keys_invalid_intervals_and_unknown_region_types() -> None: +def test_rejects_unindexed_mappings_and_invalid_parameter_intervals() -> None: duplicate = source_models() duplicate["us"].parameters_by_name = [] with pytest.raises(CatalogExtractionError, match="name-indexed mapping"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=duplicate, installed_version=installed_version, ) - unknown_region = source_models() - unknown_region["us"].region_registry[1].region_type = "province" - with pytest.raises(CatalogExtractionError, match="unsupported type"): - extract_catalog( - bundle=bundle(), - policyengine_version=POLICYENGINE_VERSION, - models=unknown_region, - installed_version=installed_version, - ) - invalid_interval = source_models() invalid_interval["us"].parameters_by_name["gov.example.rate"].parameter_values[ 0 ].end_date = datetime(2026, 1, 1, tzinfo=UTC) with pytest.raises(CatalogExtractionError, match="canonical inclusive intervals"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=invalid_interval, @@ -399,6 +425,7 @@ def test_rejects_parameter_values_that_are_not_oldest_to_newest() -> None: parameter.parameter_values.reverse() with pytest.raises(CatalogExtractionError, match="oldest to newest"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -419,6 +446,7 @@ def test_collapses_equal_parameter_values_at_the_same_effective_date() -> None: ) catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -446,6 +474,7 @@ def test_rejects_conflicting_parameter_values_at_the_same_effective_date() -> No with pytest.raises(CatalogExtractionError, match="conflicting values"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -467,6 +496,7 @@ def test_rejects_repeated_parameter_values_with_inconsistent_intervals() -> None with pytest.raises(CatalogExtractionError, match="inconsistent intervals"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -491,6 +521,7 @@ def test_preserves_consecutive_day_parameter_value_intervals() -> None: ] catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index fa46c3f80..20dce3fe6 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -3,9 +3,10 @@ from __future__ import annotations import math +from dataclasses import replace from datetime import date from enum import Enum -from typing import TYPE_CHECKING +from types import SimpleNamespace import pytest @@ -14,19 +15,19 @@ extract_catalog, normalize_json_value, ) +from policyengine_api.data.v2.catalog.region_references import RegionReference from tests.fixtures.v2_catalog import ( POLICYENGINE_VERSION, bundle, installed_version, + region_references, source_models, ) -if TYPE_CHECKING: - from types import SimpleNamespace - def _extract(models: dict[str, SimpleNamespace]) -> None: extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -46,97 +47,198 @@ def _break_manifest_version(models: dict[str, SimpleNamespace]) -> None: models["us"].release_manifest.policyengine_version = "9.9.9" -def _remove_region_registry(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry = None +@pytest.mark.parametrize( + ("break_models", "message"), + [ + ( + _break_package_version, + "^us public model does not match the PolicyEngine.py manifest$", + ), + ( + _break_manifest_country, + "^us public model has an inconsistent release manifest$", + ), + ( + _break_manifest_version, + "^us public model was not certified by the installed PolicyEngine.py " + "version$", + ), + ], +) +def test_inconsistent_public_models_fail_with_a_specific_message( + break_models, + message: str, +) -> None: + models = source_models() + break_models(models) + with pytest.raises(CatalogExtractionError, match=message): + _extract(models) -def _mislabel_region_registry(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry.country_id = "uk" +References = dict[str, tuple[RegionReference, ...]] -def _duplicate_region_codes(models: dict[str, SimpleNamespace]) -> None: - registry = models["us"].region_registry - registry.append(registry[1]) +def _replace_reference( + references: References, country_id: str, code: str, **changes: str | None +) -> None: + references[country_id] = tuple( + replace(reference, **changes) if reference.code == code else reference + for reference in references[country_id] + ) -def _remove_national_region(models: dict[str, SimpleNamespace]) -> None: - registry = models["us"].region_registry - del registry[0] + +def _point_national_region_at_another_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["uk"].release_manifest.region_datasets[ + "national" + ].path_template = "frs_2024_25.h5" + models["uk"].release_manifest.datasets["frs_2024_25"] = SimpleNamespace( + path="frs_2024_25.h5", revision="fixture", repo_id=None, repo_type=None + ) -def _give_national_region_another_dataset(models: dict[str, SimpleNamespace]) -> None: - models["uk"].region_registry[0].dataset_path = "hf://policyengine/uk/other.h5" +def _name_an_uncertified_region_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets["state"].path_template = "missing.h5" -def _give_filter_an_unsupported_value(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry[1].scoping_strategy.variable_value = {"state": "CA"} +def _name_a_per_region_dataset_template( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets[ + "state" + ].path_template = "populace_{state_code}.h5" -def _give_filter_additional_filters(models: dict[str, SimpleNamespace]) -> None: - models["uk"].region_registry[1].scoping_strategy.additional_filters = { - "region": "LONDON" - } +def _remove_the_weight_matrix( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["uk"].release_manifest.datasets["parliamentary_constituency_weights"] + + +def _take_the_weight_matrix_from_another_release( + models: dict[str, SimpleNamespace], _references: References +) -> None: + matrix = models["uk"].release_manifest.datasets[ + "parliamentary_constituency_weights" + ] + matrix.revision = "another-release" + + +def _drop_a_filter_value( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference(references, "us", "state/ca", filter_value=None) + + +def _give_the_national_region_a_filter_value( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference(references, "us", "us", filter_value="0") + + +def _place_a_district_under_a_place( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference( + references, + "us", + "congressional_district/CA-01", + parent_code="place/CA-44000", + ) + + +def _remove_a_country_from_the_references( + _models: dict[str, SimpleNamespace], references: References +) -> None: + del references["uk"] @pytest.mark.parametrize( - ("break_models", "message"), + ("break_sources", "message"), [ ( - _break_package_version, - "^us public model does not match the PolicyEngine.py manifest$", + _point_national_region_at_another_dataset, + "^uk national region does not use the reviewed dataset$", ), ( - _break_manifest_country, - "^us public model has an inconsistent release manifest$", + _name_an_uncertified_region_dataset, + "^us release names no certified dataset at path 'missing.h5' for state " + "regions$", ), ( - _break_manifest_version, - "^us public model was not certified by the installed PolicyEngine.py " - "version$", + _name_a_per_region_dataset_template, + r"^us state regions name a per-region dataset template " + r"'populace_\{state_code\}.h5', which the catalog does not support$", ), - (_remove_region_registry, "^us public region registry is absent$"), - (_mislabel_region_registry, "^us public region registry is absent$"), ( - _duplicate_region_codes, - r"^duplicate region natural keys: \['state/ca'\]$", + _remove_the_weight_matrix, + "^region 'constituency/E14001063' reweights households with " + "parliamentary_constituency_weights, which the uk release does not " + "certify$", ), - (_remove_national_region, "^us national region is absent$"), ( - _give_national_region_another_dataset, - "^uk national region does not use the reviewed dataset$", + _take_the_weight_matrix_from_another_release, + "^region 'constituency/E14001063' reweights households with " + "parliamentary_constituency_weights, which is not certified in the " + "release of microcosm_uk_2024_25$", ), + (_drop_a_filter_value, "^region 'state/ca' has no filter value$"), ( - _give_filter_an_unsupported_value, - "^region 'state/ca' has an unsupported filter value$", + _give_the_national_region_a_filter_value, + "^region 'us' has a filter value, but a national region selects every " + "household$", ), ( - _give_filter_additional_filters, - "^region 'country/england' has unsupported additional filters$", + _place_a_district_under_a_place, + "^region 'congressional_district/CA-01' cannot be published because its " + "parent 'place/CA-44000' is left out$", + ), + ( + _remove_a_country_from_the_references, + "^region references are absent for: uk$", ), ], ) -def test_inconsistent_public_models_fail_with_a_specific_message( - break_models, +def test_inconsistent_regions_fail_with_a_specific_message( + break_sources, message: str, ) -> None: models = source_models() - break_models(models) + references = region_references() + break_sources(models, references) with pytest.raises(CatalogExtractionError, match=message): - _extract(models) + extract_catalog( + region_references=references, + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) -def test_national_region_without_a_dataset_path_uses_the_reviewed_dataset() -> None: +def test_national_region_without_a_release_dataset_uses_the_reviewed_dataset() -> None: models = source_models() - models["us"].region_registry[0].dataset_path = None + del models["us"].release_manifest.region_datasets["national"] - extract_catalog( + catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, installed_version=installed_version, ) + us = catalog.country("us") + national = next(region for region in us.regions if region.code == "us") + datasets = {dataset.id: dataset.name for dataset in us.datasets} + assert datasets[national.default_dataset_id] == "populace_us_2024" + assert us.fallback_summaries == () + class _ScalarWrapper: """A numpy-like scalar whose item() returns a plain Python value.""" diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index a3cbe0d65..7bb733ec0 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -251,6 +251,7 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: entity_counts={"parameters": 12}, fallback_summaries=(("us", "state", 3),), elapsed_seconds=2.3456, + omitted_region_summaries=(("us", "place", 333),), ).as_dict() assert evidence == { @@ -261,6 +262,9 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: "fallback_summaries": [ {"country_id": "us", "region_type": "state", "count": 3} ], + "omitted_region_summaries": [ + {"country_id": "us", "region_type": "place", "count": 333} + ], "elapsed_seconds": 2.346, } assert ( From 6cf90bcfd756e46c59629ddf24978c169c4c6a88 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:31:09 +0400 Subject: [PATCH 020/118] Refuse to publish regions whose filter column or weight-matrix row is missing Before it publishes a catalog version that is not stored yet, the publication command checks each region against the certified artifact its filter reads, and stops with a clear error if: - a row filter's column (state_fips, congressional_district_geoid, region) is not a household column of the region's dataset; or - a weight matrix's row count differs from its row lookup, or the lookup has no row for a constituency's or local authority's ONS code. Compute finds a region's matrix row in the same lookup files (constituencies_2024.csv, local_authorities_2021.csv). The artifacts are read from Hugging Face at their pinned revisions with ranged requests, so only HDF5 metadata and the column-name arrays are downloaded: about a minute for the 9.8 GB US file. Pandas fixed and table formats, one frame per year, and policyengine-core files are understood. A read failure names only the artifact and the error type. Versions already stored are not read again, since publication only compares them. The seed workflow's publication step receives the organization secret PE_UK_PRIVATE_HF_READ_TOKEN as HUGGING_FACE_TOKEN, because the UK data repository is private. The release workflow passes it through its existing `secrets: inherit`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/seed-v2-database.yml | 3 + ...n-1f-123-region-reference-files.changed.md | 2 +- .../uk-private-hugging-face-credential.md | 5 + docs/migration/stage-9-v2-metadata.md | 60 +++-- .../v2/catalog/certified_data_inspection.py | 171 ++++++++++++++ .../data/v2/catalog/extraction.py | 2 + .../data/v2/catalog/initialization.py | 56 ++++- .../data/v2/catalog/publication.py | 17 ++ policyengine_api/data/v2/catalog/records.py | 18 ++ .../v2/catalog/region_data_verification.py | 193 ++++++++++++++++ .../data/v2/catalog/region_scoping.py | 31 ++- .../test_v2_catalog_publication.py | 14 ++ tests/unit/v2/test_catalog_initialization.py | 103 +++++++++ .../unit/v2/test_certified_data_inspection.py | 218 ++++++++++++++++++ tests/unit/v2/test_metadata_deployment.py | 9 + .../unit/v2/test_region_data_verification.py | 208 +++++++++++++++++ 16 files changed, 1078 insertions(+), 32 deletions(-) create mode 100644 policyengine_api/data/v2/catalog/certified_data_inspection.py create mode 100644 policyengine_api/data/v2/catalog/region_data_verification.py create mode 100644 tests/unit/v2/test_certified_data_inspection.py create mode 100644 tests/unit/v2/test_region_data_verification.py diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index 634352775..61af8978c 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -66,3 +66,6 @@ jobs: run: uv run python scripts/publish_v2_metadata_catalog.py env: V2_DATA_WRITE_DATABASE_URL: ${{ secrets.V2_DATA_WRITE_DATABASE_URL }} + # Reads the certified UK data release, whose repository is private, + # to check each new catalog version's region filters. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md index 589a50194..3b2933649 100644 --- a/changelog.d/v2plan-1f-123-region-reference-files.changed.md +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -1 +1 @@ -Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index 1bc1c6134..6d02b014e 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -51,6 +51,11 @@ Cloud Run candidate resolver verifies that the exact deployed revision binds `HUGGING_FACE_TOKEN` to `pe-uk-private-hf-read-token:latest`, without printing the credential. +The v2 catalog publication step of `.github/workflows/seed-v2-database.yml` +also receives the organization secret as `HUGGING_FACE_TOKEN`, step-scoped. It +reads the HDF5 metadata of the certified UK dataset and weight matrices to +check each new catalog version's region filters, and nothing else. + The simulation-entry Cloud Run service does not receive this credential. It submits requests to Modal and does not download model data. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8ee9dc4b4..b6f0bd994 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -64,8 +64,12 @@ must execute in this order: Alembic chain, confirms all heads are current, and detects ungenerated model changes. 3. Remove the migration URL from the command environment. -4. Supply only `V2_DATA_WRITE_DATABASE_URL` and the same target identity, then - run `uv run python scripts/publish_v2_metadata_catalog.py`. +4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity and + `HUGGING_FACE_TOKEN` (from the organization secret + `PE_UK_PRIVATE_HF_READ_TOKEN`), then run + `uv run python scripts/publish_v2_metadata_catalog.py`. Before publishing a + catalog version that is not stored yet, the command reads the certified data + artifacts its regions' filters use and stops if one cannot select a region. 5. Require the command to finish successfully and retain its non-secret JSON evidence before creating the candidate revision. 6. Deploy the Cloud Run candidate with the target identity and @@ -101,20 +105,42 @@ does not reorder, deduplicate, or repair package output. A value superseded on the immediately following day therefore has equal start and end dates and is a valid one-day interval. -The US national region uses `populace_us_2024`. A US state, -congressional-district, or place region uses its PolicyEngine.py regional -alternative when one is available. Until a later reviewed change removes the -fallback, a subnational region without such an alternative uses -`populace_us_2024`. The command emits one summary warning containing only the -affected region types and counts. The UK national and country regions use -`microcosm_uk_2024_25`, the default certified by PolicyEngine.py 6.10.0; a -country filters that national file. UK constituency and local-authority -regions use the dataset PolicyEngine.py routes them to, -`microcosm_uk_2024_25_dense`, because the national file has no area codes. - -Review the fallback summary on every new PolicyEngine.py release. A changed -count may identify an upstream catalog change that needs a reviewed dataset -selection even when publication otherwise succeeds. +Regions come from the Public API's checked-in reference files, one per country +(`policyengine_api/data/v2/catalog/region_reference_files/`), not from +PolicyEngine.py's region registry. Each row lists a region's code, label, type, +parent, US state, the value that selects its households, and the v1 identifier +it replaces. A scoping rule per country and region type gives each region its +dataset from the certified data release and its filter: + +| Regions | Dataset | Filter | +| --- | --- | --- | +| US national | the release's national dataset (`populace_us_2024`) | none | +| US states and congressional districts | the release's state and district dataset | `row_filter` on `state_fips` or `congressional_district_geoid` | +| UK national | `enhanced_frs_2024_25` | none | +| UK nations | `enhanced_frs_2024_25` | `row_filter` on `region` | +| UK constituencies and local authorities | `enhanced_frs_2024_25` | `weight_replacement`: field `parliamentary_constituency_weights` or `local_authority_weights`, value the ONS code | + +A weight matrix must come from the same release as the dataset it reweights. +If the release names no dataset for a sub-national type, its regions use the +default dataset and the command emits one summary warning with the affected +region types and counts. Region types without a rule, such as US places until +the data can select them, are left out and counted in the evidence's +`omitted_region_summaries`. + +Before a new catalog version is published, the command reads the HDF5 metadata +of each artifact the filters use, with ranged requests at the pinned revision: +publication stops if a row filter's column is missing from its dataset's +household columns, or if a weight matrix's row count differs from its row +lookup (`constituencies_2024.csv`, `local_authorities_2021.csv`) or the lookup +lacks a region's code. + +Review the fallback and omitted-region summaries on every new PolicyEngine.py +release. A changed count may identify an upstream catalog change that needs a +reviewed dataset selection even when publication otherwise succeeds. + +A catalog version is immutable: republishing a stored PolicyEngine.py version +whose regions differ from the stored ones fails. Region reference changes +therefore reach the catalog with the next PolicyEngine.py version. ## Validation evidence and retry @@ -123,7 +149,7 @@ Successful JSON evidence contains: - the canonical PolicyEngine.py version; - the dependency versions derived from its packaged manifest; - catalog entity counts; -- US fallback counts; +- fallback counts and omitted-region counts by country and region type; - elapsed publication time; and - a success outcome. diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py new file mode 100644 index 000000000..d0a1ebed5 --- /dev/null +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -0,0 +1,171 @@ +"""Read household column names and weight-matrix sizes from certified HDF5 files. + +The artifacts are read from Hugging Face at their pinned revisions through +ranged requests, so only the HDF5 metadata and the small column-name arrays +are downloaded, never the households themselves. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, BinaryIO, Protocol, TypeVar +from urllib.parse import quote + +import h5py +from huggingface_hub import HfFileSystem + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + + from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +HOUSEHOLD_TABLE = "household" +# PyTables names the columns a table stores together "values_block_"; their +# real names live only in pickled attributes. +VALUE_BLOCK_PREFIX = "values_block_" +TABLE_INDEX_COLUMN = "index" + +_Result = TypeVar("_Result") + + +class CertifiedDataReadError(RuntimeError): + """Raised when a certified artifact does not have the structure expected.""" + + +# HDF5 metadata is read in many small, scattered pieces; caching whole blocks of +# this size keeps the number of ranged requests low. +READ_BLOCK_BYTES = 1 << 20 + + +class ArtifactFileSystem(Protocol): + """Opens a file in a Hugging Face repository at a revision for reading.""" + + def open( + self, path: str, mode: str, *, block_size: int, cache_type: str + ) -> AbstractContextManager[BinaryIO]: + """Return a readable, seekable binary file that caches blocks it reads.""" + ... + + +def huggingface_path(artifact: CertifiedArtifact) -> str: + """Return the Hugging Face file system path of an artifact at its revision.""" + prefix = "datasets/" if artifact.repo_type == "dataset" else "" + revision = quote(artifact.revision, safe="") + return f"{prefix}{artifact.repo_id}@{revision}/{artifact.path}" + + +def _text(value: object) -> str: + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def _is_frame(group: h5py.Group) -> bool: + return "axis0" in group or "table" in group + + +def _frame_columns(group: h5py.Group) -> frozenset[str]: + """Return the column names of one pandas data frame stored in HDF5.""" + if "axis0" in group: + return frozenset(_text(value) for value in group["axis0"][()]) + names = group["table"].dtype.names or () + if any(name.startswith(VALUE_BLOCK_PREFIX) for name in names): + msg = ( + f"household table {group.name.lstrip('/')} stores columns in value " + "blocks, whose names cannot be read without unpickling" + ) + raise CertifiedDataReadError(msg) + return frozenset(name for name in names if name != TABLE_INDEX_COLUMN) + + +def household_columns_in_hdf5(h5: h5py.File) -> frozenset[str]: + """Return the household column names of a dataset file. + + A pandas file stores the household frame directly, or one frame per year + (then only columns every year has count). A policyengine-core file stores + one top-level entry per variable. + """ + if HOUSEHOLD_TABLE not in h5: + return frozenset(h5.keys()) + household = h5[HOUSEHOLD_TABLE] + if not isinstance(household, h5py.Group): + msg = "household entry is not an HDF5 group" + raise CertifiedDataReadError(msg) + if _is_frame(household): + return _frame_columns(household) + frames = [ + entry + for entry in household.values() + if isinstance(entry, h5py.Group) and _is_frame(entry) + ] + if not frames: + msg = "household group holds no data frame" + raise CertifiedDataReadError(msg) + return frozenset.intersection(*(_frame_columns(frame) for frame in frames)) + + +def weight_matrix_rows_in_hdf5(h5: h5py.File) -> int: + """Return the number of region rows of a weight matrix. + + The file holds one two-dimensional array per year, of regions by + households, and every year must have the same regions. + """ + row_counts: set[int] = set() + for name, entry in h5.items(): + if not isinstance(entry, h5py.Dataset): + continue + if len(entry.shape) != 2: # noqa: PLR2004 # regions by households + msg = f"the weight matrix entry {name} is not two-dimensional" + raise CertifiedDataReadError(msg) + row_counts.add(entry.shape[0]) + if not row_counts: + msg = "the weight matrix has no year datasets" + raise CertifiedDataReadError(msg) + if len(row_counts) > 1: + msg = ( + "the weight matrix years disagree on the number of regions: " + f"{sorted(row_counts)}" + ) + raise CertifiedDataReadError(msg) + return row_counts.pop() + + +def _hugging_face_file_system(token: str | None) -> ArtifactFileSystem: + return HfFileSystem(token=token) + + +class HuggingFaceDataInspector: + """Reads certified artifacts from Hugging Face with an optional read token.""" + + def __init__( + self, + *, + token: str | None, + file_system_factory: Callable[[str | None], ArtifactFileSystem] = ( + _hugging_face_file_system + ), + ) -> None: + """Build the file system once; a token of None reads public data only.""" + self._file_system = file_system_factory(token) + + def _read( + self, + artifact: CertifiedArtifact, + read: Callable[[h5py.File], _Result], + ) -> _Result: + with ( + self._file_system.open( + huggingface_path(artifact), + "rb", + block_size=READ_BLOCK_BYTES, + cache_type="blockcache", + ) as handle, + h5py.File(handle, "r") as h5, + ): + return read(h5) + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + """Return the household column names of a dataset artifact.""" + return self._read(artifact, household_columns_in_hdf5) + + def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: + """Return how many region rows a weight-matrix artifact holds.""" + return self._read(artifact, weight_matrix_rows_in_hdf5) diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 76895175e..410340cea 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -32,6 +32,7 @@ ) from policyengine_api.data.v2.catalog.region_scoping import ( ScopedRegion, + filtered_artifacts, scope_regions, ) @@ -663,6 +664,7 @@ def _normalize_country( OmittedRegionSummary(region_type=region_type, count=count) for region_type, count in scoping.omitted_counts.items() ), + certified_artifacts=filtered_artifacts(scoping), ) diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 10efdda6f..e62098bf9 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -3,12 +3,16 @@ from __future__ import annotations import json +import os import sys from typing import TYPE_CHECKING from sqlalchemy import Engine, create_engine from sqlalchemy.pool import NullPool +from policyengine_api.data.v2.catalog.certified_data_inspection import ( + HuggingFaceDataInspector, +) from policyengine_api.data.v2.catalog.extraction import ( CatalogExtractionError, extract_installed_catalog, @@ -17,6 +21,10 @@ CatalogPublicationError, PublicationEvidence, publish_catalog, + unpublished_countries, +) +from policyengine_api.data.v2.catalog.region_data_verification import ( + verify_region_data, ) from policyengine_api.data.v2.settings import ( V2ConfigurationError, @@ -25,16 +33,56 @@ ) if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Mapping, Sequence from policyengine_api.data.v2.catalog.records import NormalizedCatalog +# The environment variable holding the Hugging Face read token; PolicyEngine +# Core's data downloads read the same variable. +HUGGING_FACE_TOKEN_VARIABLE = "HUGGING_FACE_TOKEN" # noqa: S105 # an environment variable name, not a secret + + def build_data_write_engine(settings: V2DatabaseSettings) -> Engine: """Build an isolated, unpooled engine for one initialization attempt.""" return create_engine(settings.connection.url, poolclass=NullPool) +def verify_installed_region_data( + catalog: NormalizedCatalog, + countries: Sequence[str], + environ: Mapping[str, str] | None = None, +) -> None: + """Check the named countries' regions against their certified artifacts. + + The artifacts are read from Hugging Face with ``HUGGING_FACE_TOKEN``, + which the UK's private data repository needs. + """ + token = (os.environ if environ is None else environ).get( + HUGGING_FACE_TOKEN_VARIABLE + ) + verify_region_data( + catalog, + HuggingFaceDataInspector(token=token or None), + countries=countries, + ) + + +def verify_unpublished_region_data( + engine: Engine, + catalog: NormalizedCatalog, + environ: Mapping[str, str] | None = None, +) -> None: + """Check the regions of countries whose catalog version is not stored yet. + + Publication writes nothing for the other countries and only compares them + with what is stored, so their artifacts are not read. + """ + verify_installed_region_data( + catalog, unpublished_countries(engine, catalog), environ + ) + + def initialize_catalog( environ: Mapping[str, str] | None = None, *, @@ -43,12 +91,16 @@ def initialize_catalog( publisher: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( publish_catalog ), + region_data_verifier: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = verify_unpublished_region_data, ) -> PublicationEvidence: - """Load only row-write settings, extract fully, then publish explicitly.""" + """Load only row-write settings, extract fully, verify, then publish explicitly.""" settings = load_v2_data_write_database_settings(environ) catalog = extractor() engine = engine_builder(settings) try: + region_data_verifier(engine, catalog, environ) return publisher(engine, catalog) finally: engine.dispose() diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 88da73310..04256af4f 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -56,6 +56,7 @@ "CatalogPublicationError", "PublicationEvidence", "publish_catalog", + "unpublished_countries", ] @@ -87,6 +88,22 @@ def _protected_row_counts(connection: Connection) -> tuple[int, int, int, int]: ) +def unpublished_countries( + engine: Engine, catalog: NormalizedCatalog +) -> tuple[str, ...]: + """Return the countries whose catalog version is not stored yet, in catalog order. + + A read outside publication's transaction: a version another publisher adds + meanwhile is still compared with the staged one when this one publishes. + """ + with engine.connect() as connection: + return tuple( + country.country_id + for country in catalog.countries + if not version_exists(connection, country) + ) + + def publish_catalog( engine: Engine, catalog: NormalizedCatalog, diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index 5be00f616..0b22334fd 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -126,6 +126,21 @@ class RegionRecord: state_name: str | None +@dataclass(frozen=True, slots=True) +class CertifiedArtifact: + """One artifact of a country's certified data release, pinned to a revision. + + ``repo_type`` is the Hugging Face repository type, ``model`` or ``dataset``. + """ + + name: str + repo_id: str + repo_type: str + path: str + revision: str + sha256: str | None + + @dataclass(frozen=True, slots=True) class FallbackSummary: """How many sub-national regions of one type fell back to the default dataset. @@ -178,6 +193,9 @@ class CountryCatalog: regions: tuple[RegionRecord, ...] fallback_summaries: tuple[FallbackSummary, ...] omitted_region_summaries: tuple[OmittedRegionSummary, ...] = () + # The release artifacts the regions' filters read: the datasets of row + # filters and the weight matrices of weight-replacement filters. + certified_artifacts: tuple[CertifiedArtifact, ...] = () def parameter_value_batches( self, diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py new file mode 100644 index 000000000..803698467 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -0,0 +1,193 @@ +"""Refuse to publish regions whose households the certified data cannot select. + +A row-filtered region needs its filter column among the household columns of +its dataset. A weight-replacement region needs a row in its weight matrix: the +matrix's rows follow the order of the row lookup that compute uses to find a +region's row, so the lookup must list exactly as many codes as the matrix has +rows and include the region's code. +""" + +from __future__ import annotations + +import csv +from collections import defaultdict +from functools import partial +from pathlib import Path +from typing import TYPE_CHECKING, Protocol, TypeVar + +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.region_scoping import ( + ROW_FILTER, + WEIGHT_REPLACEMENT, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from policyengine_api.data.v2.catalog.records import ( + CertifiedArtifact, + CountryCatalog, + NormalizedCatalog, + RegionRecord, + ) + +# The Public API's copies of the row lookups compute reads to find a region's +# row in each UK weight matrix (the files compute downloads under the same names). +_DATA_DIRECTORY = Path(__file__).parents[2] +WEIGHT_MATRIX_ROW_LOOKUPS: Mapping[str, Path] = { + "parliamentary_constituency_weights": _DATA_DIRECTORY / "constituencies_2024.csv", + "local_authority_weights": _DATA_DIRECTORY / "local_authorities_2021.csv", +} +# Region codes named in an error message, at most. +MAX_LISTED_REGIONS = 3 + +_Result = TypeVar("_Result") + + +class RegionDataVerificationError(CatalogExtractionError): + """Raised when the certified data cannot select a region's households.""" + + +class CertifiedDataInspector(Protocol): + """Reads what verification needs from one certified artifact.""" + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + """Return the household column names of a dataset artifact.""" + ... + + def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: + """Return how many region rows a weight-matrix artifact holds.""" + ... + + +def load_weight_matrix_row_codes(path: Path) -> tuple[str, ...]: + """Return a row lookup's region codes in matrix row order.""" + with path.open(encoding="utf-8", newline="") as handle: + return tuple(row["code"] for row in csv.DictReader(handle)) + + +def _listed(codes: Sequence[str]) -> str: + shown = ", ".join(codes[:MAX_LISTED_REGIONS]) + more = len(codes) - MAX_LISTED_REGIONS + return f"{shown} and {more} more" if more > 0 else shown + + +def _noun(count: int) -> str: + return "region" if count == 1 else "regions" + + +def _read(artifact: CertifiedArtifact, read: Callable[[], _Result]) -> _Result: + """Run one read of an artifact, reporting a failure without its error text.""" + try: + return read() + except RegionDataVerificationError: + raise + except Exception as error: + # Only the artifact's name and the error type: the error text may hold + # its location, and command output names no dataset release. + msg = ( + f"could not read the certified artifact {artifact.name}: " + f"{type(error).__name__}" + ) + raise RegionDataVerificationError(msg) from error + + +def _verify_row_filters( + country: CountryCatalog, + artifacts: Mapping[str, CertifiedArtifact], + inspector: CertifiedDataInspector, +) -> None: + dataset_names = {dataset.id: dataset.name for dataset in country.datasets} + regions_by_column: dict[tuple[str, str], list[str]] = defaultdict(list) + for region in country.regions: + if region.filter_strategy == ROW_FILTER and region.filter_field: + dataset_name = dataset_names[region.default_dataset_id] + regions_by_column[dataset_name, region.filter_field].append(region.code) + columns_by_dataset: dict[str, frozenset[str]] = {} + for (dataset_name, column), codes in sorted(regions_by_column.items()): + if dataset_name not in columns_by_dataset: + artifact = artifacts[dataset_name] + columns_by_dataset[dataset_name] = _read( + artifact, + partial(inspector.household_columns, artifact), + ) + if column not in columns_by_dataset[dataset_name]: + msg = ( + f"the certified dataset {dataset_name} has no household column " + f"{column}, which {len(codes)} {country.country_id} " + f"{_noun(len(codes))} {'filters' if len(codes) == 1 else 'filter'} " + f"on ({_listed(codes)})" + ) + raise RegionDataVerificationError(msg) + + +def _weight_replacement_regions( + regions: Sequence[RegionRecord], +) -> dict[str, list[RegionRecord]]: + regions_by_matrix: dict[str, list[RegionRecord]] = defaultdict(list) + for region in regions: + if region.filter_strategy == WEIGHT_REPLACEMENT and region.filter_field: + regions_by_matrix[region.filter_field].append(region) + return regions_by_matrix + + +def _verify_weight_matrices( + country: CountryCatalog, + artifacts: Mapping[str, CertifiedArtifact], + inspector: CertifiedDataInspector, + weight_matrix_row_codes: Mapping[str, Sequence[str]], +) -> None: + for matrix_name, regions in sorted( + _weight_replacement_regions(country.regions).items() + ): + row_codes = weight_matrix_row_codes.get(matrix_name) + if row_codes is None: + msg = f"no row lookup names the regions of the weight matrix {matrix_name}" + raise RegionDataVerificationError(msg) + artifact = artifacts[matrix_name] + row_count = _read( + artifact, + partial(inspector.weight_matrix_row_count, artifact), + ) + if row_count != len(row_codes): + msg = ( + f"the weight matrix {matrix_name} has {row_count} region rows, but its " + f"row lookup lists {len(row_codes)} codes" + ) + raise RegionDataVerificationError(msg) + known_codes = set(row_codes) + missing = [ + region.code for region in regions if region.filter_value not in known_codes + ] + if missing: + msg = ( + f"the weight matrix {matrix_name} has no row for {len(missing)} " + f"{country.country_id} {_noun(len(missing))} ({_listed(missing)})" + ) + raise RegionDataVerificationError(msg) + + +def verify_region_data( + catalog: NormalizedCatalog, + inspector: CertifiedDataInspector, + *, + countries: Sequence[str], + weight_matrix_row_codes: Mapping[str, Sequence[str]] | None = None, +) -> None: + """Check the regions of the named countries against their certified artifacts. + + Each artifact is read at most once. Without explicit row codes, the row + lookups in ``WEIGHT_MATRIX_ROW_LOOKUPS`` are read. + """ + if weight_matrix_row_codes is None: + weight_matrix_row_codes = { + name: load_weight_matrix_row_codes(path) + for name, path in WEIGHT_MATRIX_ROW_LOOKUPS.items() + } + for country_id in countries: + country = catalog.country(country_id) + artifacts = { + artifact.name: artifact for artifact in country.certified_artifacts + } + _verify_row_filters(country, artifacts, inspector) + _verify_weight_matrices(country, artifacts, inspector, weight_matrix_row_codes) diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py index d3ea6629d..541973bc6 100644 --- a/policyengine_api/data/v2/catalog/region_scoping.py +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.records import CertifiedArtifact if TYPE_CHECKING: from policyengine_api.data.v2.catalog.region_references import RegionReference @@ -58,18 +59,6 @@ class RegionScopingRule: } -@dataclass(frozen=True, slots=True) -class CertifiedArtifact: - """One artifact of a country's certified data release, pinned to a revision.""" - - name: str - repo_id: str - repo_type: str - path: str - revision: str - sha256: str | None - - @dataclass(frozen=True, slots=True) class ScopedRegion: """A reference region with its dataset and the filter selecting its households.""" @@ -311,3 +300,21 @@ def scope_regions( fallback_counts=dict(sorted(fallback_counts.items())), omitted_counts=dict(sorted(Counter(omitted.values()).items())), ) + + +def filtered_artifacts(scoping: RegionScoping) -> tuple[CertifiedArtifact, ...]: + """Return, by name, the artifacts the regions' filters read. + + Those are the datasets of row-filtered regions and the weight matrices of + weight-replacement regions. + """ + names = { + region.dataset_name + for region in scoping.regions + if region.filter_strategy == ROW_FILTER + } | { + region.filter_field + for region in scoping.regions + if region.filter_strategy == WEIGHT_REPLACEMENT and region.filter_field + } + return tuple(scoping.certified_artifacts[name] for name in sorted(names)) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index c894c8b45..011b3d29c 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -18,6 +18,7 @@ from policyengine_api.data.v2.catalog.publication import ( CatalogPublicationError, publish_catalog, + unpublished_countries, ) from policyengine_api.data.v2.models import ( Dataset, @@ -173,6 +174,19 @@ def test_publish_and_republish_preserve_complete_catalog( } +def test_lists_only_countries_whose_catalog_version_is_not_stored( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + newer = normalized_catalog(policyengine_version="4.21.0") + + assert unpublished_countries(publication_engine, catalog) == ("us", "uk") + publish_catalog(publication_engine, catalog) + + assert unpublished_countries(publication_engine, catalog) == () + assert unpublished_countries(publication_engine, newer) == ("us", "uk") + + @pytest.mark.parametrize( "failure_point", ["during_copy", "after_reconciliation", "after_validation"], diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index e77681fcd..c998c3006 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -9,6 +9,9 @@ from policyengine_api.data.v2.catalog import initialization from policyengine_api.data.v2.catalog.publication import PublicationEvidence +from policyengine_api.data.v2.catalog.region_data_verification import ( + RegionDataVerificationError, +) from policyengine_api.data.v2.settings import ( V2_DATA_WRITE_DATABASE_URL, V2_SUPABASE_ENVIRONMENT, @@ -65,22 +68,122 @@ def publish(selected_engine, catalog): observed["version"] = catalog.policyengine_version return _evidence() + def verify(selected_engine, catalog, environ): + assert "engine" not in observed + observed["verified"] = ( + selected_engine, + catalog.policyengine_version, + environ, + ) + result = initialization.initialize_catalog( ENVIRONMENT, extractor=normalized_catalog, engine_builder=build, publisher=publish, + region_data_verifier=verify, ) assert result == _evidence() assert observed == { "username": "data-writer", + "verified": (engine, POLICYENGINE_VERSION, ENVIRONMENT), "engine": engine, "version": POLICYENGINE_VERSION, } assert engine.disposed +def test_region_data_failure_publishes_nothing_and_disposes_engine() -> None: + engine = FakeEngine() + + def reject(_engine, _catalog, _environ): + msg = "the certified dataset d has no household column c" + raise RegionDataVerificationError(msg) + + def refuse_publication(_engine, _catalog): + msg = "publication must not start" + raise AssertionError(msg) + + with pytest.raises(RegionDataVerificationError, match="no household column"): + initialization.initialize_catalog( + ENVIRONMENT, + extractor=normalized_catalog, + engine_builder=lambda _settings: engine, + publisher=refuse_publication, + region_data_verifier=reject, + ) + assert engine.disposed + + +def test_installed_verification_reads_artifacts_with_the_hugging_face_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed = {} + + class RecordingInspector: + def __init__(self, *, token): + observed["token"] = token + + def record(catalog, inspector, *, countries): + observed["call"] = (catalog.policyengine_version, type(inspector), countries) + + monkeypatch.setattr(initialization, "HuggingFaceDataInspector", RecordingInspector) + monkeypatch.setattr(initialization, "verify_region_data", record) + + initialization.verify_installed_region_data( + normalized_catalog(), ("uk",), {"HUGGING_FACE_TOKEN": "read-token"} + ) + assert observed == { + "token": "read-token", + "call": (POLICYENGINE_VERSION, RecordingInspector, ("uk",)), + } + + initialization.verify_installed_region_data(normalized_catalog(), ("uk",), {}) + assert observed["token"] is None + + +def test_default_verification_checks_only_countries_not_yet_published( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine = FakeEngine() + catalog = normalized_catalog() + observed = {} + + def find_unpublished(selected_engine, selected_catalog): + observed["found"] = (selected_engine, selected_catalog) + return ("uk",) + + def verify(selected_catalog, countries, environ): + observed["verified"] = (selected_catalog, countries, environ) + + monkeypatch.setattr(initialization, "unpublished_countries", find_unpublished) + monkeypatch.setattr(initialization, "verify_installed_region_data", verify) + + initialization.verify_unpublished_region_data(engine, catalog, ENVIRONMENT) + + assert observed == { + "found": (engine, catalog), + "verified": (catalog, ("uk",), ENVIRONMENT), + } + + +def test_main_reports_a_region_data_failure_with_its_message( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail(): + msg = "the weight matrix m has no row for 1 uk region (constituency/x)" + raise RegionDataVerificationError(msg) + + monkeypatch.setattr(initialization, "initialize_catalog", fail) + + assert initialization.main() == 1 + failure = capsys.readouterr() + assert '"type": "RegionDataVerificationError"' in failure.err + assert "has no row for 1 uk region" in failure.err + + def test_extraction_failure_opens_no_engine() -> None: def fail_extraction(): msg = "extraction failed" diff --git a/tests/unit/v2/test_certified_data_inspection.py b/tests/unit/v2/test_certified_data_inspection.py new file mode 100644 index 000000000..6ff69191e --- /dev/null +++ b/tests/unit/v2/test_certified_data_inspection.py @@ -0,0 +1,218 @@ +"""Reading column names and weight-matrix sizes from certified HDF5 artifacts.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import h5py +import numpy as np +import pandas as pd +import pytest + +from policyengine_api.data.v2.catalog.certified_data_inspection import ( + CertifiedDataReadError, + HuggingFaceDataInspector, + household_columns_in_hdf5, + huggingface_path, + weight_matrix_rows_in_hdf5, +) +from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +if TYPE_CHECKING: + from pathlib import Path + +HOUSEHOLDS = pd.DataFrame( + { + "household_id": [1, 2], + "household_weight": [1.5, 2.5], + "state_fips": [6, 36], + } +) + + +def _artifact( + *, + name: str = "populace_us_2024_acs_local", + repo_type: str = "dataset", + revision: str = "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", +) -> CertifiedArtifact: + return CertifiedArtifact( + name=name, + repo_id="policyengine/populace-us", + repo_type=repo_type, + path=f"{name}.h5", + revision=revision, + sha256=None, + ) + + +def _columns(path: Path) -> frozenset[str]: + with h5py.File(path, "r") as h5: + return household_columns_in_hdf5(h5) + + +def _rows(path: Path) -> int: + with h5py.File(path, "r") as h5: + return weight_matrix_rows_in_hdf5(h5) + + +def test_reads_columns_of_a_fixed_format_household_frame(tmp_path: Path) -> None: + path = tmp_path / "fixed.h5" + HOUSEHOLDS.to_hdf(path, key="household", format="fixed") + + assert _columns(path) == {"household_id", "household_weight", "state_fips"} + + +def test_reads_data_columns_of_a_table_format_household_frame(tmp_path: Path) -> None: + path = tmp_path / "table.h5" + frame = HOUSEHOLDS.assign(region=pd.Categorical(["LONDON", "WALES"])) + frame.to_hdf(path, key="household", format="table", data_columns=True) + + assert _columns(path) == { + "household_id", + "household_weight", + "state_fips", + "region", + } + + +# PyTables warns that year keys are not Python identifiers; policyengine-uk's +# multi-year files use the same keys. +@pytest.mark.filterwarnings("ignore:object name is not a valid Python identifier") +def test_keeps_only_columns_every_year_of_a_multi_year_file_has( + tmp_path: Path, +) -> None: + path = tmp_path / "years.h5" + HOUSEHOLDS.to_hdf(path, key="household/2024", format="table", data_columns=True) + HOUSEHOLDS.drop(columns="state_fips").to_hdf( + path, key="household/2025", format="table", data_columns=True + ) + + assert _columns(path) == {"household_id", "household_weight"} + + +def test_reads_variable_names_of_a_core_format_file(tmp_path: Path) -> None: + path = tmp_path / "core.h5" + with h5py.File(path, "w") as h5: + h5.create_dataset("state_fips/2024", data=np.array([6, 36])) + h5.create_dataset("household_weight/2024", data=np.array([1.5, 2.5])) + + assert _columns(path) == {"state_fips", "household_weight"} + + +def test_refuses_a_table_whose_columns_are_stored_in_value_blocks( + tmp_path: Path, +) -> None: + path = tmp_path / "blocks.h5" + HOUSEHOLDS.to_hdf(path, key="household", format="table") + + with pytest.raises( + CertifiedDataReadError, + match=r"^household table household stores columns in value blocks, whose " + r"names cannot be read without unpickling$", + ): + _columns(path) + + +def test_refuses_a_household_group_that_holds_no_frame(tmp_path: Path) -> None: + path = tmp_path / "empty.h5" + with h5py.File(path, "w") as h5: + h5.create_group("household") + + with pytest.raises( + CertifiedDataReadError, match=r"^household group holds no data frame$" + ): + _columns(path) + + +def test_counts_the_region_rows_of_a_weight_matrix(tmp_path: Path) -> None: + path = tmp_path / "matrix.h5" + with h5py.File(path, "w") as h5: + h5.create_dataset("2024", data=np.ones((3, 5))) + h5.create_dataset("2025", data=np.ones((3, 5))) + + assert _rows(path) == 3 + + +@pytest.mark.parametrize( + ("shapes", "message"), + [ + ({}, r"^the weight matrix has no year datasets$"), + ({"2024": (3,)}, r"^the weight matrix entry 2024 is not two-dimensional$"), + ( + {"2024": (3, 5), "2025": (4, 5)}, + r"^the weight matrix years disagree on the number of regions: \[3, 4\]$", + ), + ], +) +def test_refuses_an_inconsistent_weight_matrix( + tmp_path: Path, + shapes: dict[str, tuple[int, ...]], + message: str, +) -> None: + path = tmp_path / "matrix.h5" + with h5py.File(path, "w") as h5: + for year, shape in shapes.items(): + h5.create_dataset(year, data=np.ones(shape)) + + with pytest.raises(CertifiedDataReadError, match=message): + _rows(path) + + +def test_names_dataset_and_model_repository_files_at_their_revision() -> None: + assert huggingface_path(_artifact()) == ( + "datasets/policyengine/populace-us@" + "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/" + "populace_us_2024_acs_local.h5" + ) + assert huggingface_path( + _artifact(name="enhanced_frs_2024_25", repo_type="model", revision="1.58.1") + ) == ("policyengine/populace-us@1.58.1/enhanced_frs_2024_25.h5") + assert huggingface_path(_artifact(revision="refs/pr/1")).startswith( + "datasets/policyengine/populace-us@refs%2Fpr%2F1/" + ) + + +class FakeFileSystem: + """Serves local files for Hugging Face paths and records how it was built.""" + + def __init__(self, files: dict[str, Path]) -> None: + self.files = files + self.opened: list[str] = [] + + def open(self, path: str, mode: str, *, block_size: int, cache_type: str): + assert (mode, block_size, cache_type) == ("rb", 1 << 20, "blockcache") + self.opened.append(path) + return self.files[path].open("rb") + + +def test_inspector_reads_artifacts_through_an_authenticated_file_system( + tmp_path: Path, +) -> None: + dataset_path = tmp_path / "dataset.h5" + HOUSEHOLDS.to_hdf(dataset_path, key="household", format="fixed") + matrix_path = tmp_path / "matrix.h5" + with h5py.File(matrix_path, "w") as h5: + h5.create_dataset("2025", data=np.ones((2, 2))) + dataset = _artifact() + matrix = _artifact(name="parliamentary_constituency_weights", repo_type="model") + file_system = FakeFileSystem( + { + huggingface_path(dataset): dataset_path, + huggingface_path(matrix): matrix_path, + } + ) + tokens: list[str | None] = [] + + def build_file_system(token: str | None) -> FakeFileSystem: + tokens.append(token) + return file_system + + inspector = HuggingFaceDataInspector( + token="read-token", file_system_factory=build_file_system + ) + + assert "state_fips" in inspector.household_columns(dataset) + assert inspector.weight_matrix_row_count(matrix) == 2 + assert tokens == ["read-token"] + assert file_system.opened == [huggingface_path(dataset), huggingface_path(matrix)] diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index c752b46c0..a5e51e864 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -171,3 +171,12 @@ def test_stage_9_deployment_shell_script_is_syntax_valid() -> None: ) assert result.returncode == 0, result.stderr + + +def test_only_catalog_publication_receives_the_hugging_face_read_token() -> None: + workflow = _read(".github/workflows/seed-v2-database.yml") + publication = _step(workflow, "Seed and validate the v2 metadata catalog") + token = "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}" + + assert token in publication + assert workflow.count("HUGGING_FACE_TOKEN") == 1 diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py new file mode 100644 index 000000000..b5a4d89b7 --- /dev/null +++ b/tests/unit/v2/test_region_data_verification.py @@ -0,0 +1,208 @@ +"""Publication refuses regions the certified data cannot select.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from policyengine_api.data.v2.catalog.region_data_verification import ( + WEIGHT_MATRIX_ROW_LOOKUPS, + RegionDataVerificationError, + load_weight_matrix_row_codes, + verify_region_data, +) +from tests.fixtures.v2_catalog import normalized_catalog + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +ALL_COLUMNS = { + "populace_us_2024_acs_local": frozenset( + {"state_fips", "congressional_district_geoid", "household_weight"} + ), + "microcosm_uk_2024_25": frozenset({"region", "household_weight"}), +} +CONSTITUENCY_ROW_CODES = ("E14001063", "E14001064") + + +class FakeInspector: + """Answers from fixed columns and matrix sizes, and records each artifact read.""" + + def __init__( + self, + *, + columns: dict[str, frozenset[str]] | None = None, + matrix_rows: dict[str, int] | None = None, + failure: Exception | None = None, + ) -> None: + self.columns = ALL_COLUMNS if columns is None else columns + self.matrix_rows = ( + {"parliamentary_constituency_weights": len(CONSTITUENCY_ROW_CODES)} + if matrix_rows is None + else matrix_rows + ) + self.failure = failure + self.reads: list[tuple[str, str]] = [] + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + self.reads.append(("columns", artifact.name)) + if self.failure is not None: + raise self.failure + return self.columns[artifact.name] + + def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: + self.reads.append(("rows", artifact.name)) + if self.failure is not None: + raise self.failure + return self.matrix_rows[artifact.name] + + +def _verify( + inspector: FakeInspector, + *, + countries: tuple[str, ...] = ("us", "uk"), + row_codes: tuple[str, ...] = CONSTITUENCY_ROW_CODES, +) -> None: + verify_region_data( + normalized_catalog(), + inspector, + countries=countries, + weight_matrix_row_codes={"parliamentary_constituency_weights": row_codes}, + ) + + +def test_catalog_records_only_the_artifacts_its_filters_read() -> None: + catalog = normalized_catalog() + + assert [ + (artifact.name, artifact.repo_id, artifact.repo_type, artifact.revision) + for artifact in catalog.country("us").certified_artifacts + ] == [ + ( + "populace_us_2024_acs_local", + "policyengine/populace-us", + "dataset", + "fixture", + ) + ] + assert [ + (artifact.name, artifact.path, artifact.repo_type) + for artifact in catalog.country("uk").certified_artifacts + ] == [ + ("microcosm_uk_2024_25", "microcosm_uk_2024_25.h5", "model"), + ( + "parliamentary_constituency_weights", + "parliamentary_constituency_weights.h5", + "model", + ), + ] + + +def test_accepts_regions_whose_columns_and_matrix_rows_exist() -> None: + inspector = FakeInspector() + + _verify(inspector) + + assert sorted(inspector.reads) == [ + ("columns", "microcosm_uk_2024_25"), + ("columns", "populace_us_2024_acs_local"), + ("rows", "parliamentary_constituency_weights"), + ] + + +def test_reads_only_the_artifacts_of_the_countries_it_verifies() -> None: + inspector = FakeInspector() + + _verify(inspector, countries=("us",)) + + assert inspector.reads == [("columns", "populace_us_2024_acs_local")] + + +def test_verifies_nothing_when_every_version_is_already_published() -> None: + inspector = FakeInspector(failure=AssertionError("must not read")) + + _verify(inspector, countries=()) + + assert inspector.reads == [] + + +def test_rejects_a_filter_column_missing_from_the_dataset() -> None: + inspector = FakeInspector( + columns={ + **ALL_COLUMNS, + "populace_us_2024_acs_local": frozenset({"state_fips"}), + } + ) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the certified dataset populace_us_2024_acs_local has no household " + r"column congressional_district_geoid, which 1 us region filters on " + r"\(congressional_district/CA-01\)$", + ): + _verify(inspector) + + +def test_rejects_a_region_code_missing_from_the_weight_matrix() -> None: + with pytest.raises( + RegionDataVerificationError, + match=r"^the weight matrix parliamentary_constituency_weights has no row for " + r"1 uk region \(constituency/E14001063\)$", + ): + _verify(FakeInspector(), row_codes=("E14001064", "E14001065")) + + +def test_rejects_a_weight_matrix_whose_rows_differ_from_its_lookup() -> None: + inspector = FakeInspector(matrix_rows={"parliamentary_constituency_weights": 3}) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the weight matrix parliamentary_constituency_weights has 3 region " + r"rows, but its row lookup lists 2 codes$", + ): + _verify(inspector) + + +def test_rejects_a_weight_matrix_without_a_row_lookup() -> None: + with pytest.raises( + RegionDataVerificationError, + match=r"^no row lookup names the regions of the weight matrix " + r"parliamentary_constituency_weights$", + ): + verify_region_data( + normalized_catalog(), + FakeInspector(), + countries=("uk",), + weight_matrix_row_codes={}, + ) + + +def test_reports_an_unreadable_artifact_without_its_error_text() -> None: + inspector = FakeInspector( + failure=PermissionError("403 for https://example.test/?token=do-not-print") + ) + + with pytest.raises(RegionDataVerificationError) as raised: + _verify(inspector, countries=("us",)) + + assert str(raised.value) == ( + "could not read the certified artifact populace_us_2024_acs_local: " + "PermissionError" + ) + assert "do-not-print" not in str(raised.value) + assert "fixture" not in str(raised.value) + + +def test_row_lookups_list_every_reference_code_in_matrix_order() -> None: + constituencies = load_weight_matrix_row_codes( + WEIGHT_MATRIX_ROW_LOOKUPS["parliamentary_constituency_weights"] + ) + local_authorities = load_weight_matrix_row_codes( + WEIGHT_MATRIX_ROW_LOOKUPS["local_authority_weights"] + ) + + assert len(constituencies) == len(set(constituencies)) == 650 + assert len(local_authorities) == len(set(local_authorities)) == 361 + assert constituencies[0] == "E14001063" + assert local_authorities[0] == "E06000001" From 47902c1295d1ba973d6d49de9a6cb3e4a7d7a648 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:40:02 +0400 Subject: [PATCH 021/118] Keep a stored catalog version's regions when the reference files change A catalog version is identified by its country and PolicyEngine.py version and never changes once published, but its regions now come from the Public API's reference files. Republishing a stored version compared every table, regions included, so a reference file change under an unchanged PolicyEngine.py version (6.7.5 today, whose stored version has the 826 regions of PolicyEngine.py's registry) made the seed job fail. For a stored version, publication now compares every table except regions exactly as before, keeps the stored region rows unchanged, and logs kept_published_catalog_regions with the country, the version and the counts of staged region rows the stored version lacks and of stored rows not staged. A version not yet stored still publishes, and is checked against, the reference-file regions. Region file changes therefore take effect in the next catalog version that is published. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-123-region-reference-files.changed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 15 ++- .../data/v2/catalog/publication.py | 36 ++++- .../v2/catalog/publication_reconciliation.py | 55 ++++++-- .../test_v2_catalog_publication.py | 125 ++++++++++++++++++ 5 files changed, 214 insertions(+), 19 deletions(-) diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md index 3b2933649..0b762cece 100644 --- a/changelog.d/v2plan-1f-123-region-reference-files.changed.md +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -1 +1 @@ -Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, and every other table must still match. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index b6f0bd994..b5ef519db 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -138,9 +138,14 @@ Review the fallback and omitted-region summaries on every new PolicyEngine.py release. A changed count may identify an upstream catalog change that needs a reviewed dataset selection even when publication otherwise succeeds. -A catalog version is immutable: republishing a stored PolicyEngine.py version -whose regions differ from the stored ones fails. Region reference changes -therefore reach the catalog with the next PolicyEngine.py version. +A catalog version is identified by its country and PolicyEngine.py version and +never changes once published. When the command republishes a stored version +whose staged regions differ from the stored ones, it keeps the stored regions +unchanged, logs `kept_published_catalog_regions` with the country, the version +and the counts of staged region rows the stored version lacks and of stored +rows not staged, and continues. Every other table must still match exactly. +Region reference changes therefore take effect in the next catalog version +that is published. ## Validation evidence and retry @@ -169,7 +174,9 @@ A failed attempt rolls back all catalog changes. Retry the same artifact only after correcting the external failure. A matching retry preserves identifiers, content, and row counts. Different normalized content under the same PolicyEngine.py version fails for operator review; it is never silently -replaced. A later PolicyEngine.py version adds a catalog version without +replaced. Regions are the exception: they come from the Public API's reference +files, so a stored version keeps the regions it was published with (see +"Source and regional dataset behavior"). A later PolicyEngine.py version adds a catalog version without deleting or modifying earlier versioned rows, including their logical datasets, regions, model descriptions, current-law IDs, and time-period options. diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 04256af4f..d58ddd665 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -12,6 +12,7 @@ from policyengine_api.data.v2.catalog.publication_reconciliation import ( assert_canonical_value_uniqueness, assert_country_matches, + count_region_differences, publish_new_country, version_exists, ) @@ -34,7 +35,10 @@ if TYPE_CHECKING: from collections.abc import Callable - from policyengine_api.data.v2.catalog.records import NormalizedCatalog + from policyengine_api.data.v2.catalog.records import ( + CountryCatalog, + NormalizedCatalog, + ) # Stable application-defined PostgreSQL lock ID shared by all v2 catalog publishers. PUBLICATION_ADVISORY_LOCK_KEY = 8_629_020_026_090_001 @@ -121,10 +125,14 @@ def publish_catalog( if checkpoint is not None: checkpoint("after_copy", connection) + # A stored catalog version never changes. Its regions stay as they + # were published even when the reference files differ now; every + # other table must still match exactly. existing: set[str] = set() for country in catalog.countries: if version_exists(connection, country): - assert_country_matches(connection, country) + assert_country_matches(connection, country, include_regions=False) + _log_kept_regions(connection, country) existing.add(country.country_id) for country in catalog.countries: @@ -134,7 +142,11 @@ def publish_catalog( checkpoint("after_reconciliation", connection) for country in catalog.countries: - assert_country_matches(connection, country) + assert_country_matches( + connection, + country, + include_regions=country.country_id not in existing, + ) assert_canonical_value_uniqueness(connection) if _protected_row_counts(connection) != before: msg = "publication changed simulation, report, or dataset-version rows" @@ -162,6 +174,24 @@ def publish_catalog( ) +def _log_kept_regions(connection: Connection, country: CountryCatalog) -> None: + """Log, with counts only, how a stored version's regions differ from the staged.""" + staged_not_stored, stored_not_staged = count_region_differences(connection, country) + if not staged_not_stored and not stored_not_staged: + return + LOGGER.warning( + "kept_published_catalog_regions", + extra={ + "policyengine_attributes": { + "country_id": country.country_id, + "policyengine_version": country.model_version.version, + "staged_region_rows_not_stored": staged_not_stored, + "stored_region_rows_not_staged": stored_not_staged, + } + }, + ) + + def _log_fallback_warning( fallback_summaries: tuple[tuple[str, str, int], ...], ) -> None: diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 0977676c6..f8e378235 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -220,6 +220,22 @@ def _comparison_pairs( ) ) + return ( + (staged_model_version, actual_model_version), + (staged_variables, actual_variables), + (staged_parameter_nodes, actual_parameter_nodes), + (staged_parameters, actual_parameters), + (staged_parameter_values, actual_parameter_values), + (staged_datasets, actual_datasets), + ) + + +def _region_comparison_pair(country: CountryCatalog) -> tuple[Select, Select]: + """Return the staged and stored region rows of one country's catalog version.""" + selected_version = sa.and_( + MODELS.c.name == country.model.name, + MODEL_VERSIONS.c.version == country.model_version.version, + ) staged_regions = sa.select( STAGE_CATALOG_REGIONS.c.code, STAGE_CATALOG_REGIONS.c.label, @@ -256,15 +272,7 @@ def _comparison_pairs( .where(selected_version) ) - return ( - (staged_model_version, actual_model_version), - (staged_variables, actual_variables), - (staged_parameter_nodes, actual_parameter_nodes), - (staged_parameters, actual_parameters), - (staged_parameter_values, actual_parameter_values), - (staged_datasets, actual_datasets), - (staged_regions, actual_regions), - ) + return staged_regions, actual_regions def _sets_differ( @@ -284,18 +292,43 @@ def _sets_differ( def assert_country_matches( connection: Connection, country: CountryCatalog, + *, + include_regions: bool = True, ) -> None: """Raise CatalogPublicationError if the stored version differs from the staged one. Rows are compared as sets in both directions, so a missing row and an extra - row both fail. + row both fail. Without ``include_regions``, region rows are not compared: + they come from the Public API's reference files, which may change while + the PolicyEngine.py version stays the same. """ - for staged, actual in _comparison_pairs(country): + pairs = _comparison_pairs(country) + if include_regions: + pairs = (*pairs, _region_comparison_pair(country)) + for staged, actual in pairs: if _sets_differ(connection, staged, actual): msg = f"persisted {country.country_id} catalog differs from PolicyEngine.py" raise CatalogPublicationError(msg) +def count_region_differences( + connection: Connection, + country: CountryCatalog, +) -> tuple[int, int]: + """Count staged region rows the stored version lacks, and stored rows not staged.""" + staged, actual = _region_comparison_pair(country) + staged_not_stored = sa.except_(staged, actual).subquery() + stored_not_staged = sa.except_(actual, staged).subquery() + return ( + connection.execute( + sa.select(sa.func.count()).select_from(staged_not_stored) + ).scalar_one(), + connection.execute( + sa.select(sa.func.count()).select_from(stored_not_staged) + ).scalar_one(), + ) + + INSERT_MODEL = ( postgresql_insert(MODELS) .from_select( diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 011b3d29c..92d572bba 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -15,6 +15,7 @@ from sqlalchemy.pool import NullPool from sqlmodel import Session, select +from policyengine_api.data.v2.catalog import publication as publication_module from policyengine_api.data.v2.catalog.publication import ( CatalogPublicationError, publish_catalog, @@ -232,6 +233,130 @@ def test_same_version_with_changed_content_fails_without_mutation( assert _counts(publication_engine) == before_counts +def _region_rows(engine: Engine) -> list[tuple[object, ...]]: + with engine.connect() as connection: + return [ + tuple(row) + for row in connection.execute( + text( + """ + SELECT id, code, label, region_type, requires_filter, + filter_field, filter_value, filter_strategy, + parent_code, state_code, state_name, default_dataset_id + FROM regions + ORDER BY id + """ + ) + ) + ] + + +def _with_changed_us_regions(catalog): + """Relabel state/ca and drop the district, as a reference-file change would.""" + us = catalog.country("us") + regions = tuple( + replace(region, label="Changed California") + if region.code == "state/ca" + else region + for region in us.regions + if region.code != "congressional_district/CA-01" + ) + return replace( + catalog, + countries=(replace(us, regions=regions), catalog.country("uk")), + ) + + +def test_stored_version_keeps_its_published_regions_and_logs_the_difference( + publication_engine: Engine, + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_regions = _region_rows(publication_engine) + before_ids = _identifiers(publication_engine) + before_counts = _counts(publication_engine) + # Alembic's logging configuration disables existing loggers, so the + # warnings are recorded where they are emitted. + warnings: list[tuple[str, dict]] = [] + monkeypatch.setattr( + publication_module.LOGGER, + "warning", + lambda message, *args, **kwargs: warnings.append((message, kwargs)), + ) + + evidence = publish_catalog(publication_engine, _with_changed_us_regions(catalog)) + + assert evidence.as_dict()["outcome"] == "ok" + assert _region_rows(publication_engine) == before_regions + assert _identifiers(publication_engine) == before_ids + assert _counts(publication_engine) == before_counts + notices = [ + keywords["extra"] + for message, keywords in warnings + if message == "kept_published_catalog_regions" + ] + assert [notice["policyengine_attributes"] for notice in notices] == [ + { + "country_id": "us", + "policyengine_version": catalog.policyengine_version, + "staged_region_rows_not_stored": 1, + "stored_region_rows_not_staged": 2, + } + ] + + +def test_stored_version_with_changed_regions_still_fails_on_other_tables( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_regions = _region_rows(publication_engine) + changed = _with_changed_us_regions(catalog) + us = changed.country("us") + changed = replace( + changed, + countries=( + replace(us, variables=(replace(us.variables[0], label="Changed"),)), + changed.country("uk"), + ), + ) + + with pytest.raises(CatalogPublicationError, match="differs"): + publish_catalog(publication_engine, changed) + + assert _region_rows(publication_engine) == before_regions + + +def test_new_version_publishes_the_reference_file_regions( + publication_engine: Engine, +) -> None: + publish_catalog(publication_engine, normalized_catalog()) + newer = _with_changed_us_regions(normalized_catalog(policyengine_version="4.21.0")) + + publish_catalog(publication_engine, newer) + + with publication_engine.connect() as connection: + newer_us_regions = dict( + connection.execute( + text( + """ + SELECT region.code, region.label + FROM regions AS region + JOIN tax_benefit_model_versions AS version + ON version.id = region.tax_benefit_model_version_id + JOIN tax_benefit_models AS model ON model.id = version.model_id + WHERE version.version = '4.21.0' AND model.name = 'policyengine-us' + """ + ) + ).all() + ) + assert newer_us_regions == { + "state/ca": "Changed California", + "us": "United States", + } + + def test_republish_preserves_existing_run_and_dataset_version_records( publication_engine: Engine, ) -> None: From 5670b182ee61b2bbee7ebdb897871c7b7fa2db7d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:40:43 +0400 Subject: [PATCH 022/118] Pin the UK catalog fixture's placeholder repository in the artifact test The artifact test now checks that the UK release fixture's artifacts name the placeholder repository example-org/example-uk-data. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_region_data_verification.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py index b5a4d89b7..74e67a5cb 100644 --- a/tests/unit/v2/test_region_data_verification.py +++ b/tests/unit/v2/test_region_data_verification.py @@ -87,12 +87,18 @@ def test_catalog_records_only_the_artifacts_its_filters_read() -> None: ) ] assert [ - (artifact.name, artifact.path, artifact.repo_type) + (artifact.name, artifact.repo_id, artifact.path, artifact.repo_type) for artifact in catalog.country("uk").certified_artifacts ] == [ - ("microcosm_uk_2024_25", "microcosm_uk_2024_25.h5", "model"), + ( + "microcosm_uk_2024_25", + "example-org/example-uk-data", + "microcosm_uk_2024_25.h5", + "model", + ), ( "parliamentary_constituency_weights", + "example-org/example-uk-data", "parliamentary_constituency_weights.h5", "model", ), From 15d84e2618818fba626f5336a92d3914a133ed3a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:54:10 +0400 Subject: [PATCH 023/118] Depend directly on h5py and huggingface-hub Catalog publication's data reader imports both, which the project had only through PolicyEngine.py and PolicyEngine Core. They are now direct dependencies, constrained to keep the locked versions (h5py 3.16.0, huggingface-hub 1.12.2). uv.lock, refreshed with `uv lock` (uv 0.13.4, as the release workflow uses), changes only the project's dependency metadata. A test requires every third-party package the reader imports to be a direct dependency. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 3 ++ .../unit/v2/test_certified_data_inspection.py | 40 +++++++++++++++++-- uv.lock | 4 ++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ef1bee9d6..9285d2383 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,10 @@ dependencies = [ "google-cloud-logging>=3,<4", "google-cloud-secret-manager>=2,<3", "gunicorn", + # Catalog publication reads certified HDF5 artifacts from Hugging Face. + "h5py>=3.16,<4", "httpx>=0.27.0", + "huggingface-hub>=1.12,<2", "markupsafe>=3,<4", "microdf_python>=1.0.0", "openai", diff --git a/tests/unit/v2/test_certified_data_inspection.py b/tests/unit/v2/test_certified_data_inspection.py index 6ff69191e..d159fb820 100644 --- a/tests/unit/v2/test_certified_data_inspection.py +++ b/tests/unit/v2/test_certified_data_inspection.py @@ -2,12 +2,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import ast +import importlib.metadata +import tomllib +from pathlib import Path import h5py import numpy as np import pandas as pd import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from policyengine_api.data.v2.catalog.certified_data_inspection import ( CertifiedDataReadError, @@ -18,9 +23,6 @@ ) from policyengine_api.data.v2.catalog.records import CertifiedArtifact -if TYPE_CHECKING: - from pathlib import Path - HOUSEHOLDS = pd.DataFrame( { "household_id": [1, 2], @@ -216,3 +218,33 @@ def build_file_system(token: str | None) -> FakeFileSystem: assert inspector.weight_matrix_row_count(matrix) == 2 assert tokens == ["read-token"] assert file_system.opened == [huggingface_path(dataset), huggingface_path(matrix)] + + +def test_imports_only_packages_the_project_depends_on_directly() -> None: + repo = Path(__file__).parents[3] + module = repo / "policyengine_api/data/v2/catalog/certified_data_inspection.py" + tree = ast.parse(module.read_text(encoding="utf-8")) + top_level_imports = { + alias.name.split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } | { + (node.module or "").split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level == 0 + } + distributions = importlib.metadata.packages_distributions() + third_party = { + canonicalize_name(distributions[name][0]) + for name in top_level_imports + if name in distributions and name != "policyengine_api" + } + project = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8")) + direct = { + canonicalize_name(Requirement(requirement).name) + for requirement in project["project"]["dependencies"] + } + + assert third_party == {"h5py", "huggingface-hub"} + assert third_party <= direct diff --git a/uv.lock b/uv.lock index 094aec459..14d4b7346 100644 --- a/uv.lock +++ b/uv.lock @@ -2786,7 +2786,9 @@ dependencies = [ { name = "google-cloud-logging" }, { name = "google-cloud-secret-manager" }, { name = "gunicorn" }, + { name = "h5py" }, { name = "httpx" }, + { name = "huggingface-hub" }, { name = "markupsafe" }, { name = "microdf-python" }, { name = "openai" }, @@ -2840,7 +2842,9 @@ requires-dist = [ { name = "google-cloud-logging", specifier = ">=3,<4" }, { name = "google-cloud-secret-manager", specifier = ">=2,<3" }, { name = "gunicorn" }, + { name = "h5py", specifier = ">=3.16,<4" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "huggingface-hub", specifier = ">=1.12,<2" }, { name = "markupsafe", specifier = ">=3,<4" }, { name = "microdf-python", specifier = ">=1.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, From 4abc8a087c9356939b6af0d3234fffd31c514305 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:17:00 +0400 Subject: [PATCH 024/118] Print the kept-regions warning's country, version and counts The seed command configures no logging handler, so Python's last-resort handler printed only the warning's bare event name and dropped the structured attributes. The message now carries the country, the PolicyEngine.py version and both region-row counts; the attributes stay for structured handlers. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 6 +-- .../data/v2/catalog/publication.py | 9 +++- .../test_v2_catalog_publication.py | 13 +++-- tests/unit/v2/test_catalog_publication.py | 50 +++++++++++++++++++ 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index b5ef519db..d8deb827f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -141,9 +141,9 @@ reviewed dataset selection even when publication otherwise succeeds. A catalog version is identified by its country and PolicyEngine.py version and never changes once published. When the command republishes a stored version whose staged regions differ from the stored ones, it keeps the stored regions -unchanged, logs `kept_published_catalog_regions` with the country, the version -and the counts of staged region rows the stored version lacks and of stored -rows not staged, and continues. Every other table must still match exactly. +unchanged, prints a warning with the country, the version and the counts of +staged region rows the stored version lacks and of stored rows not staged, and +continues. Every other table must still match exactly. Region reference changes therefore take effect in the next catalog version that is published. diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index d58ddd665..108a41af0 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -179,8 +179,15 @@ def _log_kept_regions(connection: Connection, country: CountryCatalog) -> None: staged_not_stored, stored_not_staged = count_region_differences(connection, country) if not staged_not_stored and not stored_not_staged: return + # The seed command configures no logging handler, so the details are in the + # message itself; the attributes serve structured handlers. LOGGER.warning( - "kept_published_catalog_regions", + "kept the published regions of the stored %s catalog version %s: %d staged " + "region rows are not stored and %d stored region rows are not staged", + country.country_id, + country.model_version.version, + staged_not_stored, + stored_not_staged, extra={ "policyengine_attributes": { "country_id": country.country_id, diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 92d572bba..87d88a326 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -282,7 +282,7 @@ def test_stored_version_keeps_its_published_regions_and_logs_the_difference( monkeypatch.setattr( publication_module.LOGGER, "warning", - lambda message, *args, **kwargs: warnings.append((message, kwargs)), + lambda message, *args, **kwargs: warnings.append((message % args, kwargs)), ) evidence = publish_catalog(publication_engine, _with_changed_us_regions(catalog)) @@ -292,11 +292,16 @@ def test_stored_version_keeps_its_published_regions_and_logs_the_difference( assert _identifiers(publication_engine) == before_ids assert _counts(publication_engine) == before_counts notices = [ - keywords["extra"] + (message, keywords["extra"]) for message, keywords in warnings - if message == "kept_published_catalog_regions" + if message.startswith("kept the published regions") ] - assert [notice["policyengine_attributes"] for notice in notices] == [ + assert [message for message, _extra in notices] == [ + "kept the published regions of the stored us catalog version " + f"{catalog.policyengine_version}: 1 staged region rows are not stored and " + "2 stored region rows are not staged" + ] + assert [extra["policyengine_attributes"] for _message, extra in notices] == [ { "country_id": "us", "policyengine_version": catalog.policyengine_version, diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 7bb733ec0..abd519d06 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from io import StringIO from pathlib import Path from unittest.mock import patch @@ -297,3 +298,52 @@ def test_fallback_summary_emits_one_non_secret_warning() -> None: rendered_message = message_template % (logged_summaries,) assert "regional dataset fallback summary" in rendered_message assert "postgresql://" not in rendered_message + + +def _kept_region_records( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + differences: tuple[int, int], +) -> list[logging.LogRecord]: + monkeypatch.setattr( + publication, + "count_region_differences", + lambda _connection, _country: differences, + ) + # An earlier test's Alembic logging setup may have disabled existing loggers. + monkeypatch.setattr(publication.LOGGER, "disabled", False) + with caplog.at_level(logging.WARNING, logger=publication.LOGGER.name): + publication._log_kept_regions(object(), normalized_catalog().country("us")) + return [ + record for record in caplog.records if record.name == publication.LOGGER.name + ] + + +def test_kept_region_warning_prints_country_version_and_counts_without_logging_setup( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + [record] = _kept_region_records(monkeypatch, caplog, (1, 2)) + + # The seed command configures no logging, so Python's last-resort handler + # prints the record; this is the text the job output shows. + assert logging.lastResort is not None + assert logging.lastResort.format(record) == ( + "kept the published regions of the stored us catalog version 5.0.4: " + "1 staged region rows are not stored and 2 stored region rows are not " + "staged" + ) + assert record.levelno == logging.WARNING + assert record.policyengine_attributes == { + "country_id": "us", + "policyengine_version": "5.0.4", + "staged_region_rows_not_stored": 1, + "stored_region_rows_not_staged": 2, + } + + +def test_matching_kept_regions_log_nothing( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + assert _kept_region_records(monkeypatch, caplog, (0, 0)) == [] From 103cf3bd9a8f2b758f087bab297ec44e92afc537 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:21:23 +0400 Subject: [PATCH 025/118] Name the table that differs when a stored catalog version does not match Republishing a stored catalog version with different content failed with "persisted catalog differs from PolicyEngine.py", even when the difference came from the Public API's own selections, such as a dataset. The error now names the country, the version and the first table that differs (model version, variables, parameter nodes, parameters, parameter values, datasets or regions), and still no row data. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 4 +-- .../v2/catalog/publication_reconciliation.py | 32 ++++++++++------- .../test_v2_catalog_publication.py | 34 ++++++++++++++++++- tests/unit/v2/test_catalog_publication.py | 14 ++++++-- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d8deb827f..6568ad9e3 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -173,8 +173,8 @@ confirms existing simulation and report record counts are unchanged. A failed attempt rolls back all catalog changes. Retry the same artifact only after correcting the external failure. A matching retry preserves identifiers, content, and row counts. Different normalized content under the same -PolicyEngine.py version fails for operator review; it is never silently -replaced. Regions are the exception: they come from the Public API's reference +PolicyEngine.py version fails for operator review, with an error naming the +first table that differs; it is never silently replaced. Regions are the exception: they come from the Public API's reference files, so a stored version keeps the regions it was published with (see "Source and regional dataset behavior"). A later PolicyEngine.py version adds a catalog version without deleting or modifying earlier versioned rows, including their logical datasets, diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index f8e378235..1debc3d98 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -78,7 +78,8 @@ def _versioned_table_from(table: sa.Table) -> sa.Join: def _comparison_pairs( country: CountryCatalog, -) -> tuple[tuple[Select, Select], ...]: +) -> tuple[tuple[str, Select, Select], ...]: + """Return each compared table's name with its staged and stored rows.""" staged_country = STAGE_CATALOG_MODELS.c.country_id == country.country_id selected_version = sa.and_( MODELS.c.name == country.model.name, @@ -221,12 +222,12 @@ def _comparison_pairs( ) return ( - (staged_model_version, actual_model_version), - (staged_variables, actual_variables), - (staged_parameter_nodes, actual_parameter_nodes), - (staged_parameters, actual_parameters), - (staged_parameter_values, actual_parameter_values), - (staged_datasets, actual_datasets), + ("model version", staged_model_version, actual_model_version), + ("variables", staged_variables, actual_variables), + ("parameter nodes", staged_parameter_nodes, actual_parameter_nodes), + ("parameters", staged_parameters, actual_parameters), + ("parameter values", staged_parameter_values, actual_parameter_values), + ("datasets", staged_datasets, actual_datasets), ) @@ -298,16 +299,21 @@ def assert_country_matches( """Raise CatalogPublicationError if the stored version differs from the staged one. Rows are compared as sets in both directions, so a missing row and an extra - row both fail. Without ``include_regions``, region rows are not compared: - they come from the Public API's reference files, which may change while - the PolicyEngine.py version stays the same. + row both fail; the error names the first table that differs, never its + rows. Without ``include_regions``, region rows are not compared: they come + from the Public API's reference files, which may change while the + PolicyEngine.py version stays the same. """ pairs = _comparison_pairs(country) if include_regions: - pairs = (*pairs, _region_comparison_pair(country)) - for staged, actual in pairs: + pairs = (*pairs, ("regions", *_region_comparison_pair(country))) + for table_name, staged, actual in pairs: if _sets_differ(connection, staged, actual): - msg = f"persisted {country.country_id} catalog differs from PolicyEngine.py" + msg = ( + f"the stored {country.country_id} catalog version " + f"{country.model_version.version} differs from the staged catalog " + f"in its {table_name}" + ) raise CatalogPublicationError(msg) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 87d88a326..db16da1c8 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -226,13 +226,45 @@ def test_same_version_with_changed_content_fails_without_mutation( countries=(changed_us, catalog.country("uk")), ) - with pytest.raises(CatalogPublicationError, match="differs"): + with pytest.raises( + CatalogPublicationError, + match=r"^the stored us catalog version 5\.0\.4 differs from the staged " + r"catalog in its variables$", + ): publish_catalog(publication_engine, changed) assert _identifiers(publication_engine) == before_ids assert _counts(publication_engine) == before_counts +def test_stored_version_with_a_different_dataset_names_the_datasets( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + uk = catalog.country("uk") + changed = replace( + catalog, + countries=( + catalog.country("us"), + replace( + uk, + datasets=tuple( + replace(dataset, description="Another dataset selection") + for dataset in uk.datasets + ), + ), + ), + ) + + with pytest.raises( + CatalogPublicationError, + match=r"^the stored uk catalog version 5\.0\.4 differs from the staged " + r"catalog in its datasets$", + ): + publish_catalog(publication_engine, changed) + + def _region_rows(engine: Engine) -> list[tuple[object, ...]]: with engine.connect() as connection: return [ diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index abd519d06..911948618 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -215,10 +215,20 @@ def test_publication_uses_mapped_tables_and_sqlalchemy_statements() -> None: catalog = normalized_catalog() comparisons = publication_reconciliation._comparison_pairs(catalog.country("us")) + assert [table_name for table_name, _staged, _stored in comparisons] == [ + "model version", + "variables", + "parameter nodes", + "parameters", + "parameter values", + "datasets", + ] + statements = [ + statement for _table_name, *pair in comparisons for statement in pair + ] + list(publication_reconciliation._region_comparison_pair(catalog.country("us"))) assert all( isinstance(statement, sa.sql.Select) and not isinstance(statement, TextClause) - for pair in comparisons - for statement in pair + for statement in statements ) class Result: From 089713dc397d8f96f880fd4de663175682a2188b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:26:16 +0400 Subject: [PATCH 026/118] Report a certified artifact's structure error with its own message Verification reduced every read failure to the artifact's name and the error type, so a structure error such as "household group holds no data frame" reached the operator only as "CertifiedDataReadError". That error, which names only HDF5 entries and counts, now keeps its message; any other failure is still reduced to the name and type, since its text may hold a location. The error class moves to the catalog's errors module so verification can recognize it. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/certified_data_inspection.py | 6 ++---- policyengine_api/data/v2/catalog/errors.py | 9 ++++++++- .../data/v2/catalog/region_data_verification.py | 15 +++++++++++++-- tests/unit/v2/test_region_data_verification.py | 14 ++++++++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py index d0a1ebed5..80fd29ba3 100644 --- a/policyengine_api/data/v2/catalog/certified_data_inspection.py +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -13,6 +13,8 @@ import h5py from huggingface_hub import HfFileSystem +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError + if TYPE_CHECKING: from collections.abc import Callable from contextlib import AbstractContextManager @@ -28,10 +30,6 @@ _Result = TypeVar("_Result") -class CertifiedDataReadError(RuntimeError): - """Raised when a certified artifact does not have the structure expected.""" - - # HDF5 metadata is read in many small, scattered pieces; caching whole blocks of # this size keeps the number of ranged requests low. READ_BLOCK_BYTES = 1 << 20 diff --git a/policyengine_api/data/v2/catalog/errors.py b/policyengine_api/data/v2/catalog/errors.py index 53be90f51..e33762b81 100644 --- a/policyengine_api/data/v2/catalog/errors.py +++ b/policyengine_api/data/v2/catalog/errors.py @@ -1,7 +1,14 @@ -"""The error raised when catalog sources cannot be normalized.""" +"""Errors raised while catalog sources are read and normalized.""" from __future__ import annotations class CatalogExtractionError(RuntimeError): """Raised before database access when source metadata cannot be normalized.""" + + +class CertifiedDataReadError(RuntimeError): + """Raised when a certified artifact does not have the structure expected. + + Its message names only HDF5 entries and counts, so it is safe to print. + """ diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py index 803698467..e3f1a8489 100644 --- a/policyengine_api/data/v2/catalog/region_data_verification.py +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -15,7 +15,10 @@ from pathlib import Path from typing import TYPE_CHECKING, Protocol, TypeVar -from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.errors import ( + CatalogExtractionError, + CertifiedDataReadError, +) from policyengine_api.data.v2.catalog.region_scoping import ( ROW_FILTER, WEIGHT_REPLACEMENT, @@ -77,11 +80,19 @@ def _noun(count: int) -> str: def _read(artifact: CertifiedArtifact, read: Callable[[], _Result]) -> _Result: - """Run one read of an artifact, reporting a failure without its error text.""" + """Run one read of an artifact, reporting a failure safely. + + A structure error keeps its own message; any other failure is reduced to + the artifact's name and the error type, since its text may hold a location. + """ try: return read() except RegionDataVerificationError: raise + except CertifiedDataReadError as error: + # Its message names only HDF5 entries and counts. + msg = f"the certified artifact {artifact.name} cannot be checked: {error}" + raise RegionDataVerificationError(msg) from error except Exception as error: # Only the artifact's name and the error type: the error text may hold # its location, and command output names no dataset release. diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py index 74e67a5cb..0735d39b0 100644 --- a/tests/unit/v2/test_region_data_verification.py +++ b/tests/unit/v2/test_region_data_verification.py @@ -6,6 +6,7 @@ import pytest +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError from policyengine_api.data.v2.catalog.region_data_verification import ( WEIGHT_MATRIX_ROW_LOOKUPS, RegionDataVerificationError, @@ -212,3 +213,16 @@ def test_row_lookups_list_every_reference_code_in_matrix_order() -> None: assert len(local_authorities) == len(set(local_authorities)) == 361 assert constituencies[0] == "E14001063" assert local_authorities[0] == "E06000001" + + +def test_reports_an_artifact_structure_error_with_its_own_message() -> None: + inspector = FakeInspector( + failure=CertifiedDataReadError("household group holds no data frame") + ) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the certified artifact populace_us_2024_acs_local cannot be " + r"checked: household group holds no data frame$", + ): + _verify(inspector, countries=("us",)) From c27bd92880d7d391a778db8af55a117eebd45d84 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:26:58 +0400 Subject: [PATCH 027/118] Check the v2 revision before reading which catalog versions are stored The region data check reads stored catalog versions before publication checks the database revision, so on a database without the v2 schema the command failed with "catalog initialization failed unexpectedly" instead of the specific revision error. unpublished_countries now requires the expected Alembic revision first, as publication does. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/data/v2/catalog/publication.py | 7 +++++-- tests/integration/test_v2_catalog_publication.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 108a41af0..12b6ef059 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -97,10 +97,13 @@ def unpublished_countries( ) -> tuple[str, ...]: """Return the countries whose catalog version is not stored yet, in catalog order. - A read outside publication's transaction: a version another publisher adds - meanwhile is still compared with the staged one when this one publishes. + It first requires the expected v2 revision, as publication does, so a + missing or older schema is reported as such. A read outside publication's + transaction: a version another publisher adds meanwhile is still compared + with the staged one when this one publishes. """ with engine.connect() as connection: + _verify_expected_revision(connection) return tuple( country.country_id for country in catalog.countries diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index db16da1c8..e89323431 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -175,6 +175,20 @@ def test_publish_and_republish_preserve_complete_catalog( } +def test_unpublished_country_check_reports_a_missing_schema_specifically() -> None: + # The same server's maintenance database has no v2 schema at all. + schema_less = make_url(_disposable_url()).set(database="postgres") + engine = create_engine(schema_less, poolclass=NullPool) + try: + with pytest.raises( + CatalogPublicationError, + match=r"^the v2 Alembic revision table is absent$", + ): + unpublished_countries(engine, normalized_catalog()) + finally: + engine.dispose() + + def test_lists_only_countries_whose_catalog_version_is_not_stored( publication_engine: Engine, ) -> None: From 052ef12d00961d43364512456b97951640ed8931 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:27:26 +0400 Subject: [PATCH 028/118] State exactly what the lookup copies and the read token cover The UK credential document said the catalog publication step reads only UK artifacts, but the same authenticated file system also reads the public US dataset. The weight-matrix lookup comment and the catalog document now say that the certified data release lists no lookup file, so the check uses the Public API's own copies, and that on 2026-10-22 both copies were byte-identical to the files compute downloads. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/uk-private-hugging-face-credential.md | 9 ++++++--- docs/migration/stage-9-v2-metadata.md | 4 +++- .../data/v2/catalog/region_data_verification.py | 10 +++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index 6d02b014e..afbefac48 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -52,9 +52,12 @@ Cloud Run candidate resolver verifies that the exact deployed revision binds the credential. The v2 catalog publication step of `.github/workflows/seed-v2-database.yml` -also receives the organization secret as `HUGGING_FACE_TOKEN`, step-scoped. It -reads the HDF5 metadata of the certified UK dataset and weight matrices to -check each new catalog version's region filters, and nothing else. +also receives the organization secret as `HUGGING_FACE_TOKEN`, step-scoped. +Before it publishes a new catalog version, the step reads, through one +authenticated Hugging Face file system, the HDF5 metadata of the certified +artifacts that version's region filters use: the UK dataset and weight +matrices in the private UK data repository, and the public US dataset. It +downloads no household data. The simulation-entry Cloud Run service does not receive this credential. It submits requests to Modal and does not download model data. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 6568ad9e3..91792d6ff 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -132,7 +132,9 @@ of each artifact the filters use, with ranged requests at the pinned revision: publication stops if a row filter's column is missing from its dataset's household columns, or if a weight matrix's row count differs from its row lookup (`constituencies_2024.csv`, `local_authorities_2021.csv`) or the lookup -lacks a region's code. +lacks a region's code. The certified data release lists no lookup file, so these +are the Public API's own copies of the lookups compute reads; on 2026-10-22 both +were byte-identical to the files compute downloads. Review the fallback and omitted-region summaries on every new PolicyEngine.py release. A changed count may identify an upstream catalog change that needs a diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py index e3f1a8489..a2ad6515a 100644 --- a/policyengine_api/data/v2/catalog/region_data_verification.py +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -4,7 +4,8 @@ its dataset. A weight-replacement region needs a row in its weight matrix: the matrix's rows follow the order of the row lookup that compute uses to find a region's row, so the lookup must list exactly as many codes as the matrix has -rows and include the region's code. +rows and include the region's code. The release certifies no lookup file, so +the Public API's own copies of the lookups stand in for it. """ from __future__ import annotations @@ -34,8 +35,11 @@ RegionRecord, ) -# The Public API's copies of the row lookups compute reads to find a region's -# row in each UK weight matrix (the files compute downloads under the same names). +# The Public API's own copies of the row lookups compute reads to find a +# region's row in each UK weight matrix. The certified data release lists no +# lookup file, so the matrix check compares each matrix with these copies. On +# 2026-10-22 both were byte-identical to the files compute downloads under the +# same names (last changed 2025-04-29). _DATA_DIRECTORY = Path(__file__).parents[2] WEIGHT_MATRIX_ROW_LOOKUPS: Mapping[str, Path] = { "parliamentary_constituency_weights": _DATA_DIRECTORY / "constituencies_2024.csv", From a9160d92bc6128a823eba5f3dbe1c83217c5623d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:27:35 +0400 Subject: [PATCH 029/118] Name the weight matrix's dimension count instead of suppressing the lint Co-Authored-By: Claude Opus 5.5 --- policyengine_api/data/v2/catalog/certified_data_inspection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py index 80fd29ba3..b1ce180b8 100644 --- a/policyengine_api/data/v2/catalog/certified_data_inspection.py +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -26,6 +26,8 @@ # real names live only in pickled attributes. VALUE_BLOCK_PREFIX = "values_block_" TABLE_INDEX_COLUMN = "index" +# Each year of a weight matrix is one array of regions by households. +WEIGHT_MATRIX_DIMENSIONS = 2 _Result = TypeVar("_Result") @@ -110,7 +112,7 @@ def weight_matrix_rows_in_hdf5(h5: h5py.File) -> int: for name, entry in h5.items(): if not isinstance(entry, h5py.Dataset): continue - if len(entry.shape) != 2: # noqa: PLR2004 # regions by households + if len(entry.shape) != WEIGHT_MATRIX_DIMENSIONS: msg = f"the weight matrix entry {name} is not two-dimensional" raise CertifiedDataReadError(msg) row_counts.add(entry.shape[0]) From d96b11520f09421237826a5069f39178ecc01ce6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:21:05 +0400 Subject: [PATCH 030/118] Report the stored regions in the evidence when a version keeps them When a stored catalog version kept its region rows, the publication evidence still counted the staged regions and reported the staged fallbacks and omissions, so the record kept before a release described regions that were never published. The evidence now counts the stored region rows for such a country, leaves its staged fallback and omitted summaries out, and lists it under kept_regions with the stored and staged region counts. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-123-region-reference-files.changed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 9 ++- .../data/v2/catalog/publication.py | 79 ++++++++++++++++--- .../v2/catalog/publication_reconciliation.py | 8 ++ .../data/v2/catalog/publication_types.py | 12 +++ .../test_v2_catalog_publication.py | 16 ++++ tests/unit/v2/test_catalog_publication.py | 14 +++- 7 files changed, 123 insertions(+), 17 deletions(-) diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md index 0b762cece..8434bbdb6 100644 --- a/changelog.d/v2plan-1f-123-region-reference-files.changed.md +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -1 +1 @@ -Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, and every other table must still match. +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, its evidence counts the stored regions and lists the kept version under kept_regions, and every other table must still match. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 91792d6ff..0b231ac12 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -145,7 +145,10 @@ never changes once published. When the command republishes a stored version whose staged regions differ from the stored ones, it keeps the stored regions unchanged, prints a warning with the country, the version and the counts of staged region rows the stored version lacks and of stored rows not staged, and -continues. Every other table must still match exactly. +continues. Its evidence then describes what is stored: the region count is the +stored one, `kept_regions` lists the country, version, stored and staged region +counts, and the fallback and omitted-region summaries leave that country out, +since they describe staged regions that were not published. Every other table must still match exactly. Region reference changes therefore take effect in the next catalog version that is published. @@ -156,7 +159,9 @@ Successful JSON evidence contains: - the canonical PolicyEngine.py version; - the dependency versions derived from its packaged manifest; - catalog entity counts; -- fallback counts and omitted-region counts by country and region type; +- fallback counts and omitted-region counts by country and region type, and + the stored and staged region counts of any stored version whose regions were + kept; - elapsed publication time; and - a success outcome. diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 12b6ef059..a8951d6b5 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -13,6 +13,7 @@ assert_canonical_value_uniqueness, assert_country_matches, count_region_differences, + count_stored_regions, publish_new_country, version_exists, ) @@ -128,15 +129,7 @@ def publish_catalog( if checkpoint is not None: checkpoint("after_copy", connection) - # A stored catalog version never changes. Its regions stay as they - # were published even when the reference files differ now; every - # other table must still match exactly. - existing: set[str] = set() - for country in catalog.countries: - if version_exists(connection, country): - assert_country_matches(connection, country, include_regions=False) - _log_kept_regions(connection, country) - existing.add(country.country_id) + existing, kept_region_summaries = _check_stored_versions(connection, catalog) for country in catalog.countries: if country.country_id not in existing: @@ -157,31 +150,85 @@ def publish_catalog( if checkpoint is not None: checkpoint("after_validation", connection) + return _evidence(catalog, kept_region_summaries, started=started) + + +def _check_stored_versions( + connection: Connection, + catalog: NormalizedCatalog, +) -> tuple[set[str], tuple[tuple[str, str, int, int], ...]]: + """Compare each stored catalog version and return its countries and kept regions. + + A stored catalog version never changes. Its regions stay as they were + published even when the reference files differ now; every other table + must still match exactly. + """ + existing: set[str] = set() + kept_region_summaries: list[tuple[str, str, int, int]] = [] + for country in catalog.countries: + if version_exists(connection, country): + assert_country_matches(connection, country, include_regions=False) + kept = _kept_region_summary(connection, country) + if kept is not None: + kept_region_summaries.append(kept) + existing.add(country.country_id) + return existing, tuple(kept_region_summaries) + + +def _evidence( + catalog: NormalizedCatalog, + kept_region_summaries: tuple[tuple[str, str, int, int], ...], + *, + started: float, +) -> PublicationEvidence: + """Describe what is stored after publication. + + Every table of a stored version matches the staged catalog except regions + that were kept: for those countries the evidence counts the stored region + rows and reports none of the staged fallbacks or omissions, which describe + rows that were not published. + """ + kept_countries = {country_id for country_id, *_counts in kept_region_summaries} + entity_counts = catalog.entity_counts() + entity_counts["regions"] += sum( + stored - staged for _country, _version, stored, staged in kept_region_summaries + ) fallback_summaries = tuple( (country.country_id, summary.region_type, summary.count) for country in catalog.countries + if country.country_id not in kept_countries for summary in country.fallback_summaries ) _log_fallback_warning(fallback_summaries) return PublicationEvidence( policyengine_version=catalog.policyengine_version, dependency_versions=catalog.dependency_versions, - entity_counts=catalog.entity_counts(), + entity_counts=entity_counts, fallback_summaries=fallback_summaries, elapsed_seconds=time.monotonic() - started, omitted_region_summaries=tuple( (country.country_id, summary.region_type, summary.count) for country in catalog.countries + if country.country_id not in kept_countries for summary in country.omitted_region_summaries ), + kept_region_summaries=kept_region_summaries, ) -def _log_kept_regions(connection: Connection, country: CountryCatalog) -> None: - """Log, with counts only, how a stored version's regions differ from the staged.""" +def _kept_region_summary( + connection: Connection, + country: CountryCatalog, +) -> tuple[str, str, int, int] | None: + """Return and log how a stored version's kept regions differ from the staged. + + The summary is the country, the version and the stored and staged region + counts; there is none when the regions are the same. The log names counts + only. + """ staged_not_stored, stored_not_staged = count_region_differences(connection, country) if not staged_not_stored and not stored_not_staged: - return + return None # The seed command configures no logging handler, so the details are in the # message itself; the attributes serve structured handlers. LOGGER.warning( @@ -200,6 +247,12 @@ def _log_kept_regions(connection: Connection, country: CountryCatalog) -> None: } }, ) + return ( + country.country_id, + country.model_version.version, + count_stored_regions(connection, country), + len(country.regions), + ) def _log_fallback_warning( diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 1debc3d98..8a0c9c0aa 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -317,6 +317,14 @@ def assert_country_matches( raise CatalogPublicationError(msg) +def count_stored_regions(connection: Connection, country: CountryCatalog) -> int: + """Count the region rows stored for one country's catalog version.""" + _staged, actual = _region_comparison_pair(country) + return connection.execute( + sa.select(sa.func.count()).select_from(actual.subquery()) + ).scalar_one() + + def count_region_differences( connection: Connection, country: CountryCatalog, diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index 51865f7c7..515232560 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -19,6 +19,9 @@ class PublicationEvidence: fallback_summaries: tuple[tuple[str, str, int], ...] elapsed_seconds: float omitted_region_summaries: tuple[tuple[str, str, int], ...] = () + # Stored catalog versions whose region rows were kept although the staged + # regions differ: country, version, stored and staged region counts. + kept_region_summaries: tuple[tuple[str, str, int, int], ...] = () def as_dict(self) -> dict[str, object]: """Return the evidence as the JSON object printed after a publication.""" @@ -43,5 +46,14 @@ def as_dict(self) -> dict[str, object]: } for country_id, region_type, count in self.omitted_region_summaries ], + "kept_regions": [ + { + "country_id": country_id, + "policyengine_version": version, + "stored_regions": stored, + "staged_regions": staged, + } + for country_id, version, stored, staged in self.kept_region_summaries + ], "elapsed_seconds": round(self.elapsed_seconds, 3), } diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index e89323431..70a275895 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -337,6 +337,22 @@ def test_stored_version_keeps_its_published_regions_and_logs_the_difference( assert _region_rows(publication_engine) == before_regions assert _identifiers(publication_engine) == before_ids assert _counts(publication_engine) == before_counts + # The evidence describes the stored rows, not the staged regions. + assert evidence.entity_counts == { + "models": before_counts["tax_benefit_models"], + "model_versions": before_counts["tax_benefit_model_versions"], + "variables": before_counts["variables"], + "parameter_nodes": before_counts["parameter_nodes"], + "parameters": before_counts["parameters"], + "parameter_values": before_counts["parameter_values"], + "datasets": before_counts["datasets"], + "regions": before_counts["regions"], + } + assert evidence.kept_region_summaries == ( + ("us", catalog.policyengine_version, 3, 2), + ) + assert evidence.omitted_region_summaries == () + assert evidence.fallback_summaries == () notices = [ (message, keywords["extra"]) for message, keywords in warnings diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 911948618..2341003dc 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -263,6 +263,7 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: fallback_summaries=(("us", "state", 3),), elapsed_seconds=2.3456, omitted_region_summaries=(("us", "place", 333),), + kept_region_summaries=(("uk", "6.7.5", 5, 1_015),), ).as_dict() assert evidence == { @@ -276,6 +277,14 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: "omitted_region_summaries": [ {"country_id": "us", "region_type": "place", "count": 333} ], + "kept_regions": [ + { + "country_id": "uk", + "policyengine_version": "6.7.5", + "stored_regions": 5, + "staged_regions": 1_015, + } + ], "elapsed_seconds": 2.346, } assert ( @@ -320,10 +329,13 @@ def _kept_region_records( "count_region_differences", lambda _connection, _country: differences, ) + monkeypatch.setattr(publication, "count_stored_regions", lambda *_args: 4) # An earlier test's Alembic logging setup may have disabled existing loggers. monkeypatch.setattr(publication.LOGGER, "disabled", False) + us = normalized_catalog().country("us") with caplog.at_level(logging.WARNING, logger=publication.LOGGER.name): - publication._log_kept_regions(object(), normalized_catalog().country("us")) + summary = publication._kept_region_summary(object(), us) + assert summary == (None if differences == (0, 0) else ("us", "5.0.4", 4, 3)) return [ record for record in caplog.records if record.name == publication.LOGGER.name ] From 2a01b4c0c248de74d6415f7e80426ae313cb9b5a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:21:29 +0400 Subject: [PATCH 031/118] Name the missing ancestor when a region's parent chain breaks higher up When a region's parent existed but a region further up its chain did not, the reference file check named the region's direct parent as unknown. It now names the missing code and the region that names it as parent. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/region_references.py | 18 +++++++++++++----- tests/unit/v2/test_region_references.py | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/policyengine_api/data/v2/catalog/region_references.py b/policyengine_api/data/v2/catalog/region_references.py index ce37a5908..966d24612 100644 --- a/policyengine_api/data/v2/catalog/region_references.py +++ b/policyengine_api/data/v2/catalog/region_references.py @@ -187,19 +187,27 @@ def _check_ancestry( parents = {reference.code: reference.parent_code for reference in references} for reference in references: visited = {reference.code} + child = reference.code ancestor = reference.parent_code while ancestor is not None and ancestor != country_id: if ancestor not in parents: - msg = ( - f"region {reference.code!r} names an unknown parent " - f"{reference.parent_code!r}" + raise CatalogExtractionError( + _unknown_ancestor_message(reference.code, child, ancestor) ) - raise CatalogExtractionError(msg) if ancestor in visited: msg = f"region {reference.code!r} has a parent cycle" raise CatalogExtractionError(msg) visited.add(ancestor) - ancestor = parents[ancestor] + child, ancestor = ancestor, parents[ancestor] + + +def _unknown_ancestor_message(code: str, child: str, ancestor: str) -> str: + """Name the missing region and, if it is further up, the region it parents.""" + if child == code: + return f"region {code!r} names an unknown parent {ancestor!r}" + return ( + f"region {code!r} descends from {child!r}, whose parent {ancestor!r} is unknown" + ) def parse_region_references( diff --git a/tests/unit/v2/test_region_references.py b/tests/unit/v2/test_region_references.py index dd97be4b4..c8c17d9aa 100644 --- a/tests/unit/v2/test_region_references.py +++ b/tests/unit/v2/test_region_references.py @@ -298,6 +298,25 @@ def test_rejects_an_inconsistent_reference_file(change, message: str) -> None: parse_region_references("us", rows) +def test_names_the_missing_ancestor_further_up_a_regions_chain() -> None: + rows = [ + _row("us", "national"), + _row( + "congressional_district/CA-01", + "congressional_district", + parent_code="state/ca", + ), + _row("state/ca", "state", parent_code="state/zz"), + ] + + with pytest.raises( + CatalogExtractionError, + match=r"^region 'congressional_district/CA-01' descends from 'state/ca', " + r"whose parent 'state/zz' is unknown$", + ): + parse_region_references("us", rows) + + def test_rejects_a_file_whose_columns_differ(tmp_path: Path) -> None: (tmp_path / "us.csv").write_text( "code,label,region_type\nus,United States,national\n", encoding="utf-8" From 765c69ba4ea912d0c84df3d5a57b39473884709f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:22:06 +0400 Subject: [PATCH 032/118] Require the Hugging Face token before checking the UK's private data Without HUGGING_FACE_TOKEN, the UK artifact reads failed with "could not read the certified artifact enhanced_frs_2024_25: FileNotFoundError", which gave no sign of the cause. The installed check now stops before any read when a country whose certified data repository is private (the UK) needs checking and the token is unset, with a message naming the variable, never its value. The public US data is still read without it. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 2 ++ .../data/v2/catalog/initialization.py | 17 +++++++++-- tests/unit/v2/test_catalog_initialization.py | 29 +++++++++++++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0b231ac12..adb8e59b9 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -70,6 +70,8 @@ must execute in this order: `uv run python scripts/publish_v2_metadata_catalog.py`. Before publishing a catalog version that is not stored yet, the command reads the certified data artifacts its regions' filters use and stops if one cannot select a region. + It stops before reading anything if a UK version needs checking and + `HUGGING_FACE_TOKEN` is unset, because the UK data repository is private. 5. Require the command to finish successfully and retain its non-secret JSON evidence before creating the candidate revision. 6. Deploy the Cloud Run candidate with the target identity and diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index e62098bf9..ce5c00585 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -24,6 +24,7 @@ unpublished_countries, ) from policyengine_api.data.v2.catalog.region_data_verification import ( + RegionDataVerificationError, verify_region_data, ) from policyengine_api.data.v2.settings import ( @@ -41,6 +42,8 @@ # The environment variable holding the Hugging Face read token; PolicyEngine # Core's data downloads read the same variable. HUGGING_FACE_TOKEN_VARIABLE = "HUGGING_FACE_TOKEN" # noqa: S105 # an environment variable name, not a secret +# Countries whose certified data repository needs the read token. +PRIVATE_DATA_COUNTRIES = frozenset({"uk"}) def build_data_write_engine(settings: V2DatabaseSettings) -> Engine: @@ -55,12 +58,22 @@ def verify_installed_region_data( ) -> None: """Check the named countries' regions against their certified artifacts. - The artifacts are read from Hugging Face with ``HUGGING_FACE_TOKEN``, - which the UK's private data repository needs. + The artifacts are read from Hugging Face with ``HUGGING_FACE_TOKEN``. + Countries whose certified data repository is private need it, so their + check fails before any read when it is unset. """ token = (os.environ if environ is None else environ).get( HUGGING_FACE_TOKEN_VARIABLE ) + needing_token = [ + country for country in countries if country in PRIVATE_DATA_COUNTRIES + ] + if needing_token and not token: + msg = ( + f"{HUGGING_FACE_TOKEN_VARIABLE} must be set to read the certified " + f"{', '.join(needing_token)} data, whose repository is private" + ) + raise RegionDataVerificationError(msg) verify_region_data( catalog, HuggingFaceDataInspector(token=token or None), diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index c998c3006..1b57b0166 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -139,8 +139,33 @@ def record(catalog, inspector, *, countries): "call": (POLICYENGINE_VERSION, RecordingInspector, ("uk",)), } - initialization.verify_installed_region_data(normalized_catalog(), ("uk",), {}) - assert observed["token"] is None + initialization.verify_installed_region_data(normalized_catalog(), ("us",), {}) + assert observed == { + "token": None, + "call": (POLICYENGINE_VERSION, RecordingInspector, ("us",)), + } + + +@pytest.mark.parametrize("environ", [{}, {"HUGGING_FACE_TOKEN": ""}]) +def test_uk_verification_requires_the_hugging_face_token_before_reading( + monkeypatch: pytest.MonkeyPatch, + environ: dict[str, str], +) -> None: + def refuse(*_args, **_kwargs): + msg = "no artifact may be read without the token" + raise AssertionError(msg) + + monkeypatch.setattr(initialization, "HuggingFaceDataInspector", refuse) + monkeypatch.setattr(initialization, "verify_region_data", refuse) + + with pytest.raises( + RegionDataVerificationError, + match=r"^HUGGING_FACE_TOKEN must be set to read the certified uk data, " + r"whose repository is private$", + ): + initialization.verify_installed_region_data( + normalized_catalog(), ("us", "uk"), environ + ) def test_default_verification_checks_only_countries_not_yet_published( From 1a5beffee3d2fe75dc678d088a7a4f98004ee06e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:25:05 +0400 Subject: [PATCH 033/118] Test the remaining failure paths of region references, scoping and reading Adds tests for errors that had none: an invalid region code and a national region not named after its country; a release manifest without a dataset map or a region-dataset map, without the reviewed default dataset, with a region dataset without a path, and with an artifact without a path, revision or repository; a household entry that is not an HDF5 group. Also covers a weight matrix with a non-array entry, the default Hugging Face file system and its token, a verification error the inspector raises, and the default row lookups. Co-Authored-By: Claude Opus 5.5 --- .../v2/test_catalog_extraction_failures.py | 72 +++++++++++++++++++ .../unit/v2/test_certified_data_inspection.py | 21 ++++++ .../unit/v2/test_region_data_verification.py | 15 ++++ tests/unit/v2/test_region_references.py | 16 +++++ 4 files changed, 124 insertions(+) diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index 20dce3fe6..eeea7625d 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -157,9 +157,81 @@ def _remove_a_country_from_the_references( del references["uk"] +def _remove_the_dataset_map( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.datasets = None + + +def _remove_the_reviewed_default_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["us"].release_manifest.datasets["populace_us_2024"] + + +def _remove_the_region_dataset_map( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets = None + + +def _remove_a_region_dataset_path( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets["state"].path_template = None + + +def _remove_an_artifact_path( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.datasets["populace_us_2024"].path = None + + +def _remove_every_artifact_revision( + models: dict[str, SimpleNamespace], _references: References +) -> None: + manifest = models["us"].release_manifest + manifest.datasets["populace_us_2024"].revision = None + manifest.data_package.release_manifest_revision = None + manifest.data_package.version = None + + +def _remove_every_artifact_repository( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.data_package.repo_id = None + + @pytest.mark.parametrize( ("break_sources", "message"), [ + (_remove_the_dataset_map, "^us release manifest has no dataset map$"), + ( + _remove_the_reviewed_default_dataset, + "^us release does not certify populace_us_2024$", + ), + ( + _remove_the_region_dataset_map, + "^us release manifest has no region dataset map$", + ), + ( + _remove_a_region_dataset_path, + "^us state region dataset must be a non-empty string$", + ), + ( + _remove_an_artifact_path, + "^us release artifact populace_us_2024 path must be a non-empty string$", + ), + ( + _remove_every_artifact_revision, + "^us release artifact populace_us_2024 revision must be a non-empty " + "string$", + ), + ( + _remove_every_artifact_repository, + "^us release artifact populace_us_2024 repository must be a non-empty " + "string$", + ), ( _point_national_region_at_another_dataset, "^uk national region does not use the reviewed dataset$", diff --git a/tests/unit/v2/test_certified_data_inspection.py b/tests/unit/v2/test_certified_data_inspection.py index d159fb820..be0be2c7e 100644 --- a/tests/unit/v2/test_certified_data_inspection.py +++ b/tests/unit/v2/test_certified_data_inspection.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd import pytest +from huggingface_hub import HfFileSystem from packaging.requirements import Requirement from packaging.utils import canonicalize_name @@ -127,15 +128,35 @@ def test_refuses_a_household_group_that_holds_no_frame(tmp_path: Path) -> None: _columns(path) +def test_refuses_a_household_entry_that_is_not_a_group(tmp_path: Path) -> None: + path = tmp_path / "flat.h5" + with h5py.File(path, "w") as h5: + h5.create_dataset("household", data=np.array([1, 2])) + + with pytest.raises( + CertifiedDataReadError, match=r"^household entry is not an HDF5 group$" + ): + _columns(path) + + def test_counts_the_region_rows_of_a_weight_matrix(tmp_path: Path) -> None: path = tmp_path / "matrix.h5" with h5py.File(path, "w") as h5: h5.create_dataset("2024", data=np.ones((3, 5))) h5.create_dataset("2025", data=np.ones((3, 5))) + h5.create_group("metadata").create_dataset("note", data=np.ones(1)) assert _rows(path) == 3 +def test_default_file_system_is_hugging_face_with_the_token() -> None: + inspector = HuggingFaceDataInspector(token="read-token") + + file_system = inspector._file_system + assert isinstance(file_system, HfFileSystem) + assert file_system.token == "read-token" + + @pytest.mark.parametrize( ("shapes", "message"), [ diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py index 0735d39b0..d2d1cfa2d 100644 --- a/tests/unit/v2/test_region_data_verification.py +++ b/tests/unit/v2/test_region_data_verification.py @@ -226,3 +226,18 @@ def test_reports_an_artifact_structure_error_with_its_own_message() -> None: r"checked: household group holds no data frame$", ): _verify(inspector, countries=("us",)) + + +def test_passes_on_a_verification_error_the_inspector_raises() -> None: + inspector = FakeInspector(failure=RegionDataVerificationError("already explained")) + + with pytest.raises(RegionDataVerificationError, match=r"^already explained$"): + _verify(inspector, countries=("us",)) + + +def test_reads_the_checked_in_row_lookups_by_default() -> None: + inspector = FakeInspector(matrix_rows={"parliamentary_constituency_weights": 650}) + + verify_region_data(normalized_catalog(), inspector, countries=("uk",)) + + assert ("rows", "parliamentary_constituency_weights") in inspector.reads diff --git a/tests/unit/v2/test_region_references.py b/tests/unit/v2/test_region_references.py index c8c17d9aa..e2493a2d5 100644 --- a/tests/unit/v2/test_region_references.py +++ b/tests/unit/v2/test_region_references.py @@ -288,6 +288,22 @@ def test_parses_a_valid_minimal_file() -> None: lambda rows: rows[1].update(label=""), r"^region 'state/ca' label must be a non-empty string$", ), + ( + lambda rows: rows[1].update(code=""), + r"^us region reference file has an invalid code ''$", + ), + ( + lambda rows: rows[1].update(code="state/ca "), + r"^us region reference file has an invalid code 'state/ca '$", + ), + ( + lambda rows: rows[1].update(code="state/" + "c" * 250), + r"^us region reference file has an invalid code 'state/c+'$", + ), + ( + lambda rows: rows.append(_row("usa", "national")), + r"^national region 'usa' must be named 'us'$", + ), ], ) def test_rejects_an_inconsistent_reference_file(change, message: str) -> None: From dc263673217f7250a419ae0b3aa55dbc4bbe75b7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:25:21 +0400 Subject: [PATCH 034/118] Describe the lookup check without dating private storage The lookup comment and the catalog document now say only that on 2026-10-22 both copies were checked to be identical in content to the lookups the calculation workers download. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 3 ++- policyengine_api/data/v2/catalog/region_data_verification.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index adb8e59b9..2a2295a48 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -136,7 +136,8 @@ household columns, or if a weight matrix's row count differs from its row lookup (`constituencies_2024.csv`, `local_authorities_2021.csv`) or the lookup lacks a region's code. The certified data release lists no lookup file, so these are the Public API's own copies of the lookups compute reads; on 2026-10-22 both -were byte-identical to the files compute downloads. +copies were checked to be identical in content to the lookups the calculation +workers download. Review the fallback and omitted-region summaries on every new PolicyEngine.py release. A changed count may identify an upstream catalog change that needs a diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py index a2ad6515a..38493a306 100644 --- a/policyengine_api/data/v2/catalog/region_data_verification.py +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -38,8 +38,8 @@ # The Public API's own copies of the row lookups compute reads to find a # region's row in each UK weight matrix. The certified data release lists no # lookup file, so the matrix check compares each matrix with these copies. On -# 2026-10-22 both were byte-identical to the files compute downloads under the -# same names (last changed 2025-04-29). +# 2026-10-22 both copies were checked to be identical in content to the lookups +# the calculation workers download. _DATA_DIRECTORY = Path(__file__).parents[2] WEIGHT_MATRIX_ROW_LOOKUPS: Mapping[str, Path] = { "parliamentary_constituency_weights": _DATA_DIRECTORY / "constituencies_2024.csv", From d466c99b868629412460047bd1dae0dd7a9a4146 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:25:45 +0400 Subject: [PATCH 035/118] Describe exactly what the artifact reads download The data reader's docstring and the UK credential document said only HDF5 metadata and column names are downloaded. Reads go through a cache of 1 MB blocks, so they download the blocks holding that metadata, which can include neighbouring bytes; the household arrays are never read. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/uk-private-hugging-face-credential.md | 2 +- .../data/v2/catalog/certified_data_inspection.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index afbefac48..57d5621dd 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -57,7 +57,7 @@ Before it publishes a new catalog version, the step reads, through one authenticated Hugging Face file system, the HDF5 metadata of the certified artifacts that version's region filters use: the UK dataset and weight matrices in the private UK data repository, and the public US dataset. It -downloads no household data. +downloads only the 1 MB blocks that hold that metadata, not whole files. The simulation-entry Cloud Run service does not receive this credential. It submits requests to Modal and does not download model data. diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py index b1ce180b8..e18d42c58 100644 --- a/policyengine_api/data/v2/catalog/certified_data_inspection.py +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -1,8 +1,9 @@ """Read household column names and weight-matrix sizes from certified HDF5 files. The artifacts are read from Hugging Face at their pinned revisions through -ranged requests, so only the HDF5 metadata and the small column-name arrays -are downloaded, never the households themselves. +ranged requests. Only the 1 MB blocks holding the HDF5 metadata and the +column-name arrays are downloaded, a small part of each file; the household +arrays themselves are never read. """ from __future__ import annotations From b120d44de1e59f9dd46a795a293bff381127bb13 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:02:54 +0400 Subject: [PATCH 036/118] Ignore the coverage.py data and report files `make test` runs coverage, which writes .coverage (an SQLite database with local paths) and coverage.xml at the repository root; neither was ignored, so a broad `git add` could commit them. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index d69167e9d..18fe946f3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,8 @@ gha-creds-*.json ## vscode settings /.vscode *.db.init.lock + +# coverage.py data and reports written by `make test` +.coverage +.coverage.* +coverage.xml From 00c558cd9cbbf3a7316f754031c706e83868bb3f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:03:08 +0400 Subject: [PATCH 037/118] Rename the read-token variable constant so it needs no lint suppression The constant holding the environment variable's name is now HUGGING_FACE_CREDENTIAL_VARIABLE, which the hardcoded-password check does not flag, so its suppression is removed. The environment variable stays HUGGING_FACE_TOKEN. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/data/v2/catalog/initialization.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index ce5c00585..2bde74d25 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -41,7 +41,7 @@ # The environment variable holding the Hugging Face read token; PolicyEngine # Core's data downloads read the same variable. -HUGGING_FACE_TOKEN_VARIABLE = "HUGGING_FACE_TOKEN" # noqa: S105 # an environment variable name, not a secret +HUGGING_FACE_CREDENTIAL_VARIABLE = "HUGGING_FACE_TOKEN" # Countries whose certified data repository needs the read token. PRIVATE_DATA_COUNTRIES = frozenset({"uk"}) @@ -63,14 +63,14 @@ def verify_installed_region_data( check fails before any read when it is unset. """ token = (os.environ if environ is None else environ).get( - HUGGING_FACE_TOKEN_VARIABLE + HUGGING_FACE_CREDENTIAL_VARIABLE ) needing_token = [ country for country in countries if country in PRIVATE_DATA_COUNTRIES ] if needing_token and not token: msg = ( - f"{HUGGING_FACE_TOKEN_VARIABLE} must be set to read the certified " + f"{HUGGING_FACE_CREDENTIAL_VARIABLE} must be set to read the certified " f"{', '.join(needing_token)} data, whose repository is private" ) raise RegionDataVerificationError(msg) From 637b6284e30b3ffed95dad32c32380e2d4c5f054 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:05:55 +0400 Subject: [PATCH 038/118] Give region data and token failures error classes that match when they occur RegionDataVerificationError inherited from CatalogExtractionError, which is raised before any database access, although verification runs after the stored catalog versions are read and reads certified data over the network. It is now its own error, documented as such, and the command still prints its message. A missing HUGGING_FACE_TOKEN is now reported as the existing V2ConfigurationError, the error for absent configuration. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/initialization.py | 5 ++-- .../v2/catalog/region_data_verification.py | 14 ++++++---- tests/unit/v2/test_catalog_initialization.py | 28 ++++++++++++++++++- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 2bde74d25..978e40e47 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -60,7 +60,7 @@ def verify_installed_region_data( The artifacts are read from Hugging Face with ``HUGGING_FACE_TOKEN``. Countries whose certified data repository is private need it, so their - check fails before any read when it is unset. + check fails with a configuration error before any read when it is unset. """ token = (os.environ if environ is None else environ).get( HUGGING_FACE_CREDENTIAL_VARIABLE @@ -73,7 +73,7 @@ def verify_installed_region_data( f"{HUGGING_FACE_CREDENTIAL_VARIABLE} must be set to read the certified " f"{', '.join(needing_token)} data, whose repository is private" ) - raise RegionDataVerificationError(msg) + raise V2ConfigurationError(msg) verify_region_data( catalog, HuggingFaceDataInspector(token=token or None), @@ -123,6 +123,7 @@ def _error_payload(error: Exception) -> dict[str, object]: safe_errors = ( V2ConfigurationError, CatalogExtractionError, + RegionDataVerificationError, CatalogPublicationError, ) message = ( diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py index 38493a306..de2910a69 100644 --- a/policyengine_api/data/v2/catalog/region_data_verification.py +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -16,10 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Protocol, TypeVar -from policyengine_api.data.v2.catalog.errors import ( - CatalogExtractionError, - CertifiedDataReadError, -) +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError from policyengine_api.data.v2.catalog.region_scoping import ( ROW_FILTER, WEIGHT_REPLACEMENT, @@ -51,8 +48,13 @@ _Result = TypeVar("_Result") -class RegionDataVerificationError(CatalogExtractionError): - """Raised when the certified data cannot select a region's households.""" +class RegionDataVerificationError(RuntimeError): + """Raised when certified data cannot select a region's households. + + Also raised when an artifact cannot be read for that check. Verification + runs after the stored catalog versions are read and before anything is + published; its messages name only artifacts, columns, codes and counts. + """ class CertifiedDataInspector(Protocol): diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 1b57b0166..1870d431d 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -8,6 +8,7 @@ import pytest from policyengine_api.data.v2.catalog import initialization +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.publication import PublicationEvidence from policyengine_api.data.v2.catalog.region_data_verification import ( RegionDataVerificationError, @@ -16,6 +17,7 @@ V2_DATA_WRITE_DATABASE_URL, V2_SUPABASE_ENVIRONMENT, V2_SUPABASE_PROJECT_REF, + V2ConfigurationError, ) from tests.fixtures.v2_catalog import ( DEPENDENCY_VERSIONS, @@ -159,7 +161,7 @@ def refuse(*_args, **_kwargs): monkeypatch.setattr(initialization, "verify_region_data", refuse) with pytest.raises( - RegionDataVerificationError, + V2ConfigurationError, match=r"^HUGGING_FACE_TOKEN must be set to read the certified uk data, " r"whose repository is private$", ): @@ -270,3 +272,27 @@ def test_initializer_is_not_imported_by_application_startup_modules() -> None: module.startswith("policyengine_api.data.v2.catalog.initialization") for module in imported ) + + +def test_region_data_errors_are_not_reported_as_extraction_errors() -> None: + # Extraction errors are raised before any database access; verification + # runs after reading the stored versions and reads certified data over the + # network. + assert not issubclass(RegionDataVerificationError, CatalogExtractionError) + + +def test_main_reports_a_missing_hugging_face_token_with_its_message( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def verify_without_token(): + initialization.verify_installed_region_data(normalized_catalog(), ("uk",), {}) + + monkeypatch.setattr(initialization, "initialize_catalog", verify_without_token) + + assert initialization.main() == 1 + failure = capsys.readouterr() + assert '"type": "V2ConfigurationError"' in failure.err + assert "HUGGING_FACE_TOKEN must be set to read the certified uk data" in ( + failure.err + ) From 50c7dfce76a0060ddd2e8eb42ed9d5aa7a9f68cd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:07:30 +0400 Subject: [PATCH 039/118] Type the release manifest scoping reads with a small Protocol Region scoping typed the PolicyEngine.py release manifest as object and read its dataset map, region-dataset map and data package with getattr. A runtime-checkable Protocol now states those three attributes; a manifest without them fails with a message naming them, and their values are still checked where they are read. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/region_scoping.py | 42 ++++++++++++++++--- .../v2/test_catalog_extraction_failures.py | 10 +++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py index 541973bc6..2b43987cf 100644 --- a/policyengine_api/data/v2/catalog/region_scoping.py +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -14,7 +14,7 @@ from collections import Counter from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol, runtime_checkable from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.records import CertifiedArtifact @@ -92,22 +92,46 @@ def _manifest_text(value: object, *, description: str) -> str: return value +@runtime_checkable +class ReleaseManifest(Protocol): + """What scoping reads from a PolicyEngine.py country release manifest. + + The manifest is an untyped public object, so each value is checked where + it is read. + """ + + @property + def datasets(self) -> object: + """The certified artifacts by name.""" + ... + + @property + def region_datasets(self) -> object: + """The dataset path of each region type.""" + ... + + @property + def data_package(self) -> object: + """The repository and revision artifacts default to.""" + ... + + def certified_artifacts( country_id: str, - release_manifest: object, + release_manifest: ReleaseManifest, ) -> dict[str, CertifiedArtifact]: """Return the release manifest's artifacts, with the data package's defaults. An artifact without its own repository, repository type or revision is in the data package's repository at the release revision. """ - data_package = getattr(release_manifest, "data_package", None) + data_package = release_manifest.data_package default_repo_id = getattr(data_package, "repo_id", None) default_repo_type = getattr(data_package, "repo_type", None) or "model" default_revision = getattr( data_package, "release_manifest_revision", None ) or getattr(data_package, "version", None) - datasets = getattr(release_manifest, "datasets", None) + datasets = release_manifest.datasets if not isinstance(datasets, Mapping): msg = f"{country_id} release manifest has no dataset map" raise CatalogExtractionError(msg) @@ -138,11 +162,11 @@ def certified_artifacts( def _role_datasets( country_id: str, - release_manifest: object, + release_manifest: ReleaseManifest, artifacts: Mapping[str, CertifiedArtifact], ) -> dict[str, str]: """Map each dataset role the release names to its certified dataset's name.""" - region_datasets = getattr(release_manifest, "region_datasets", None) + region_datasets = release_manifest.region_datasets if not isinstance(region_datasets, Mapping): msg = f"{country_id} release manifest has no region dataset map" raise CatalogExtractionError(msg) @@ -243,6 +267,12 @@ def scope_regions( type has no scoping rule is left out, and a runnable region may not sit under a region that is left out. """ + if not isinstance(release_manifest, ReleaseManifest): + msg = ( + f"{country_id} release manifest lacks datasets, region_datasets or " + "data_package" + ) + raise CatalogExtractionError(msg) artifacts = certified_artifacts(country_id, release_manifest) if reviewed_default not in artifacts: msg = f"{country_id} release does not certify {reviewed_default}" diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index eeea7625d..74c9f4459 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -196,6 +196,12 @@ def _remove_every_artifact_revision( manifest.data_package.version = None +def _drop_the_region_dataset_attribute( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["us"].release_manifest.region_datasets + + def _remove_every_artifact_repository( models: dict[str, SimpleNamespace], _references: References ) -> None: @@ -206,6 +212,10 @@ def _remove_every_artifact_repository( ("break_sources", "message"), [ (_remove_the_dataset_map, "^us release manifest has no dataset map$"), + ( + _drop_the_region_dataset_attribute, + "^us release manifest lacks datasets, region_datasets or data_package$", + ), ( _remove_the_reviewed_default_dataset, "^us release does not certify populace_us_2024$", From 59f9ccc56038eab42b5d0aaf3cf434c356b96547 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:02:34 +0400 Subject: [PATCH 040/118] Row-filter UK constituencies and local authorities on the dataset the release routes them to The Public API now pins PolicyEngine.py 6.10.0. Its UK release certifies no constituency or local-authority weight matrix; it routes both region types to a local-area dataset (microcosm_uk_2024_25_dense) whose households carry constituency_code_oa and la_code_oa. UK constituencies and local authorities are now scoped like US states and districts: a row filter on that column, with the region's ONS code, on the dataset the release routes their type to. Because the national file has no area codes, they never fall back to it; a release that routes no dataset to their type stops publication, naming the region. The weight_replacement scoping, the weight-matrix artifact checks and the row-count reading of weight matrices are removed. No release the Public API can publish uses them: the pinned 6.10.0 release has none, stored 6.7.5 versions keep their regions without being scoped again, and the Public API's local-authority list now has 361 rows while 6.7.5's matrix had 360, so that check would be wrong for 6.7.5 anyway. The installed 6.10.0 catalog now extracts 1,504 regions (488 US, 1,016 UK) and 4 datasets. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-123-region-reference-files.changed.md | 2 +- .../uk-private-hugging-face-credential.md | 4 +- docs/migration/stage-9-v2-metadata.md | 31 +++-- .../v2/catalog/certified_data_inspection.py | 34 +----- policyengine_api/data/v2/catalog/records.py | 3 +- .../v2/catalog/region_data_verification.py | 90 +-------------- .../data/v2/catalog/region_references.py | 2 +- .../data/v2/catalog/region_scoping.py | 83 +++++--------- policyengine_api/data/v2/models/metadata.py | 4 +- .../services/v2/metadata/types.py | 4 +- tests/fixtures/v2_catalog.py | 11 +- .../integration/test_v2_catalog_installed.py | 76 +++++++------ .../test_v2_catalog_publication.py | 6 +- tests/unit/v2/test_catalog_extraction.py | 18 ++- .../v2/test_catalog_extraction_failures.py | 26 +---- tests/unit/v2/test_catalog_initialization.py | 7 +- .../unit/v2/test_certified_data_inspection.py | 65 +++-------- .../unit/v2/test_region_data_verification.py | 107 ++++-------------- 18 files changed, 169 insertions(+), 404 deletions(-) diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md index 8434bbdb6..2ec0371e0 100644 --- a/changelog.d/v2plan-1f-123-region-reference-files.changed.md +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -1 +1 @@ -Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and its filter from a scoping rule: a row filter on state_fips, congressional_district_geoid or the UK region column, or a weight_replacement filter naming the release's constituency or local-authority weight matrix and the region's ONS code. The UK now publishes 1,015 regions, with constituencies and local authorities by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified artifacts the filters use, from Hugging Face at their pinned revisions, and stops if a row filter's column is missing from its dataset or a weight matrix has no row for a region's code; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, its evidence counts the stored regions and lists the kept version under kept_regions, and every other table must still match. +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and a row filter from a scoping rule: state_fips or congressional_district_geoid for US states and districts, the UK region column for UK nations, and constituency_code_oa or la_code_oa on the local-area dataset the release routes UK constituencies and local authorities to. The UK now publishes 1,016 regions, with 650 constituencies and 361 local authorities (including Ards and North Down) by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified datasets the filters use, from Hugging Face at their pinned revisions, and stops if a filter's column is missing; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, its evidence counts the stored regions and lists the kept version under kept_regions, and every other table must still match. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index 57d5621dd..295be325c 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -55,8 +55,8 @@ The v2 catalog publication step of `.github/workflows/seed-v2-database.yml` also receives the organization secret as `HUGGING_FACE_TOKEN`, step-scoped. Before it publishes a new catalog version, the step reads, through one authenticated Hugging Face file system, the HDF5 metadata of the certified -artifacts that version's region filters use: the UK dataset and weight -matrices in the private UK data repository, and the public US dataset. It +datasets that version's region filters use: the UK national and local-area +datasets in the private UK data repository, and the public US dataset. It downloads only the 1 MB blocks that hold that metadata, not whole files. The simulation-entry Cloud Run service does not receive this credential. It diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 2a2295a48..61e37b49d 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -118,26 +118,23 @@ dataset from the certified data release and its filter: | --- | --- | --- | | US national | the release's national dataset (`populace_us_2024`) | none | | US states and congressional districts | the release's state and district dataset | `row_filter` on `state_fips` or `congressional_district_geoid` | -| UK national | `enhanced_frs_2024_25` | none | -| UK nations | `enhanced_frs_2024_25` | `row_filter` on `region` | -| UK constituencies and local authorities | `enhanced_frs_2024_25` | `weight_replacement`: field `parliamentary_constituency_weights` or `local_authority_weights`, value the ONS code | - -A weight matrix must come from the same release as the dataset it reweights. -If the release names no dataset for a sub-national type, its regions use the -default dataset and the command emits one summary warning with the affected -region types and counts. Region types without a rule, such as US places until -the data can select them, are left out and counted in the evidence's +| UK national | the release's national dataset (`microcosm_uk_2024_25`) | none | +| UK nations | the release's national dataset | `row_filter` on `region` | +| UK constituencies and local authorities | the release's local-area dataset (`microcosm_uk_2024_25_dense`), which carries their codes | `row_filter` on `constituency_code_oa` or `la_code_oa`, value the ONS code | + +If the release names no dataset for a US state or district, those regions use +the default dataset and the command emits one summary warning with the affected +region types and counts. A UK constituency or local authority cannot fall back, +since the national file has no area codes: a release that routes no dataset to +their type stops publication. Region types without a rule, such as US places +until the data can select them, are left out and counted in the evidence's `omitted_region_summaries`. Before a new catalog version is published, the command reads the HDF5 metadata -of each artifact the filters use, with ranged requests at the pinned revision: -publication stops if a row filter's column is missing from its dataset's -household columns, or if a weight matrix's row count differs from its row -lookup (`constituencies_2024.csv`, `local_authorities_2021.csv`) or the lookup -lacks a region's code. The certified data release lists no lookup file, so these -are the Public API's own copies of the lookups compute reads; on 2026-10-22 both -copies were checked to be identical in content to the lookups the calculation -workers download. +of each dataset the row filters use, with ranged requests at the pinned +revision, and stops if a filter's column is missing from its dataset's +household columns. It does not check that a column holds rows for every +region's value. Review the fallback and omitted-region summaries on every new PolicyEngine.py release. A changed count may identify an upstream catalog change that needs a diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py index e18d42c58..58f0164a4 100644 --- a/policyengine_api/data/v2/catalog/certified_data_inspection.py +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -1,4 +1,4 @@ -"""Read household column names and weight-matrix sizes from certified HDF5 files. +"""Read household column names from certified HDF5 dataset files. The artifacts are read from Hugging Face at their pinned revisions through ranged requests. Only the 1 MB blocks holding the HDF5 metadata and the @@ -27,8 +27,6 @@ # real names live only in pickled attributes. VALUE_BLOCK_PREFIX = "values_block_" TABLE_INDEX_COLUMN = "index" -# Each year of a weight matrix is one array of regions by households. -WEIGHT_MATRIX_DIMENSIONS = 2 _Result = TypeVar("_Result") @@ -103,32 +101,6 @@ def household_columns_in_hdf5(h5: h5py.File) -> frozenset[str]: return frozenset.intersection(*(_frame_columns(frame) for frame in frames)) -def weight_matrix_rows_in_hdf5(h5: h5py.File) -> int: - """Return the number of region rows of a weight matrix. - - The file holds one two-dimensional array per year, of regions by - households, and every year must have the same regions. - """ - row_counts: set[int] = set() - for name, entry in h5.items(): - if not isinstance(entry, h5py.Dataset): - continue - if len(entry.shape) != WEIGHT_MATRIX_DIMENSIONS: - msg = f"the weight matrix entry {name} is not two-dimensional" - raise CertifiedDataReadError(msg) - row_counts.add(entry.shape[0]) - if not row_counts: - msg = "the weight matrix has no year datasets" - raise CertifiedDataReadError(msg) - if len(row_counts) > 1: - msg = ( - "the weight matrix years disagree on the number of regions: " - f"{sorted(row_counts)}" - ) - raise CertifiedDataReadError(msg) - return row_counts.pop() - - def _hugging_face_file_system(token: str | None) -> ArtifactFileSystem: return HfFileSystem(token=token) @@ -166,7 +138,3 @@ def _read( def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: """Return the household column names of a dataset artifact.""" return self._read(artifact, household_columns_in_hdf5) - - def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: - """Return how many region rows a weight-matrix artifact holds.""" - return self._read(artifact, weight_matrix_rows_in_hdf5) diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index 0b22334fd..f05233202 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -193,8 +193,7 @@ class CountryCatalog: regions: tuple[RegionRecord, ...] fallback_summaries: tuple[FallbackSummary, ...] omitted_region_summaries: tuple[OmittedRegionSummary, ...] = () - # The release artifacts the regions' filters read: the datasets of row - # filters and the weight matrices of weight-replacement filters. + # The release datasets the regions' row filters read. certified_artifacts: tuple[CertifiedArtifact, ...] = () def parameter_value_batches( diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py index de2910a69..e7eda396a 100644 --- a/policyengine_api/data/v2/catalog/region_data_verification.py +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -1,26 +1,17 @@ """Refuse to publish regions whose households the certified data cannot select. A row-filtered region needs its filter column among the household columns of -its dataset. A weight-replacement region needs a row in its weight matrix: the -matrix's rows follow the order of the row lookup that compute uses to find a -region's row, so the lookup must list exactly as many codes as the matrix has -rows and include the region's code. The release certifies no lookup file, so -the Public API's own copies of the lookups stand in for it. +its dataset. """ from __future__ import annotations -import csv from collections import defaultdict from functools import partial -from pathlib import Path from typing import TYPE_CHECKING, Protocol, TypeVar from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError -from policyengine_api.data.v2.catalog.region_scoping import ( - ROW_FILTER, - WEIGHT_REPLACEMENT, -) +from policyengine_api.data.v2.catalog.region_scoping import ROW_FILTER if TYPE_CHECKING: from collections.abc import Callable, Mapping, Sequence @@ -29,19 +20,8 @@ CertifiedArtifact, CountryCatalog, NormalizedCatalog, - RegionRecord, ) -# The Public API's own copies of the row lookups compute reads to find a -# region's row in each UK weight matrix. The certified data release lists no -# lookup file, so the matrix check compares each matrix with these copies. On -# 2026-10-22 both copies were checked to be identical in content to the lookups -# the calculation workers download. -_DATA_DIRECTORY = Path(__file__).parents[2] -WEIGHT_MATRIX_ROW_LOOKUPS: Mapping[str, Path] = { - "parliamentary_constituency_weights": _DATA_DIRECTORY / "constituencies_2024.csv", - "local_authority_weights": _DATA_DIRECTORY / "local_authorities_2021.csv", -} # Region codes named in an error message, at most. MAX_LISTED_REGIONS = 3 @@ -64,16 +44,6 @@ def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: """Return the household column names of a dataset artifact.""" ... - def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: - """Return how many region rows a weight-matrix artifact holds.""" - ... - - -def load_weight_matrix_row_codes(path: Path) -> tuple[str, ...]: - """Return a row lookup's region codes in matrix row order.""" - with path.open(encoding="utf-8", newline="") as handle: - return tuple(row["code"] for row in csv.DictReader(handle)) - def _listed(codes: Sequence[str]) -> str: shown = ", ".join(codes[:MAX_LISTED_REGIONS]) @@ -138,73 +108,19 @@ def _verify_row_filters( raise RegionDataVerificationError(msg) -def _weight_replacement_regions( - regions: Sequence[RegionRecord], -) -> dict[str, list[RegionRecord]]: - regions_by_matrix: dict[str, list[RegionRecord]] = defaultdict(list) - for region in regions: - if region.filter_strategy == WEIGHT_REPLACEMENT and region.filter_field: - regions_by_matrix[region.filter_field].append(region) - return regions_by_matrix - - -def _verify_weight_matrices( - country: CountryCatalog, - artifacts: Mapping[str, CertifiedArtifact], - inspector: CertifiedDataInspector, - weight_matrix_row_codes: Mapping[str, Sequence[str]], -) -> None: - for matrix_name, regions in sorted( - _weight_replacement_regions(country.regions).items() - ): - row_codes = weight_matrix_row_codes.get(matrix_name) - if row_codes is None: - msg = f"no row lookup names the regions of the weight matrix {matrix_name}" - raise RegionDataVerificationError(msg) - artifact = artifacts[matrix_name] - row_count = _read( - artifact, - partial(inspector.weight_matrix_row_count, artifact), - ) - if row_count != len(row_codes): - msg = ( - f"the weight matrix {matrix_name} has {row_count} region rows, but its " - f"row lookup lists {len(row_codes)} codes" - ) - raise RegionDataVerificationError(msg) - known_codes = set(row_codes) - missing = [ - region.code for region in regions if region.filter_value not in known_codes - ] - if missing: - msg = ( - f"the weight matrix {matrix_name} has no row for {len(missing)} " - f"{country.country_id} {_noun(len(missing))} ({_listed(missing)})" - ) - raise RegionDataVerificationError(msg) - - def verify_region_data( catalog: NormalizedCatalog, inspector: CertifiedDataInspector, *, countries: Sequence[str], - weight_matrix_row_codes: Mapping[str, Sequence[str]] | None = None, ) -> None: """Check the regions of the named countries against their certified artifacts. - Each artifact is read at most once. Without explicit row codes, the row - lookups in ``WEIGHT_MATRIX_ROW_LOOKUPS`` are read. + Each artifact is read at most once. """ - if weight_matrix_row_codes is None: - weight_matrix_row_codes = { - name: load_weight_matrix_row_codes(path) - for name, path in WEIGHT_MATRIX_ROW_LOOKUPS.items() - } for country_id in countries: country = catalog.country(country_id) artifacts = { artifact.name: artifact for artifact in country.certified_artifacts } _verify_row_filters(country, artifacts, inspector) - _verify_weight_matrices(country, artifacts, inspector, weight_matrix_row_codes) diff --git a/policyengine_api/data/v2/catalog/region_references.py b/policyengine_api/data/v2/catalog/region_references.py index 966d24612..421e73622 100644 --- a/policyengine_api/data/v2/catalog/region_references.py +++ b/policyengine_api/data/v2/catalog/region_references.py @@ -57,7 +57,7 @@ class RegionReference: ``filter_value`` is the text that selects the region's households: a FIPS number or district GEOID for US states and districts, the stored region names of a UK nation, or the ONS code of a UK constituency or local - authority (its row in the weight matrix). The national region and regions + authority (its area code in the local-area dataset). The national region and regions that cannot run yet have none. ``v1_code`` is the v1 region identifier this region replaces, if v1 had one. """ diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py index 2b43987cf..3432dcb4c 100644 --- a/policyengine_api/data/v2/catalog/region_scoping.py +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -1,11 +1,9 @@ """Give each reference region its dataset and filter from the certified data release. A scoping rule per country and region type names the release's dataset role -(a key of the release manifest's ``region_datasets``) and how the region -selects its households: a row filter on one column of that dataset, or a -weight-replacement filter whose field names the release's weight-matrix -artifact and whose value is the region's row code in it. Region types without -a rule cannot run yet, so publication leaves them out and counts them. +(a key of the release manifest's ``region_datasets``) and the household column +whose value selects the region's households in that dataset. Region types +without a rule cannot run yet, so publication leaves them out and counts them. """ from __future__ import annotations @@ -23,7 +21,6 @@ from policyengine_api.data.v2.catalog.region_references import RegionReference ROW_FILTER = "row_filter" -WEIGHT_REPLACEMENT = "weight_replacement" NATIONAL_DATASET_ROLE = "national" @@ -31,13 +28,16 @@ class RegionScopingRule: """How regions of one type find their dataset and select their households. - ``filter_field`` is the dataset column of a row filter, or the release's - weight-matrix artifact name of a weight-replacement filter. + ``filter_field`` is the household column a row filter compares with the + region's filter value. When the release names no dataset for the role, + the regions use the default dataset, unless ``requires_release_dataset``: + then only the routed dataset carries the column, and scoping fails. """ dataset_role: str filter_strategy: str | None = None filter_field: str | None = None + requires_release_dataset: bool = False REGION_SCOPING_RULES: Mapping[tuple[str, str], RegionScopingRule] = { @@ -48,13 +48,19 @@ class RegionScopingRule: ), ("uk", "national"): RegionScopingRule(NATIONAL_DATASET_ROLE), ("uk", "country"): RegionScopingRule(NATIONAL_DATASET_ROLE, ROW_FILTER, "region"), + # The UK national file has no area codes; the release routes constituencies + # and local authorities to a local-area dataset that carries them. ("uk", "constituency"): RegionScopingRule( - NATIONAL_DATASET_ROLE, - WEIGHT_REPLACEMENT, - "parliamentary_constituency_weights", + "constituency", + ROW_FILTER, + "constituency_code_oa", + requires_release_dataset=True, ), ("uk", "local_authority"): RegionScopingRule( - NATIONAL_DATASET_ROLE, WEIGHT_REPLACEMENT, "local_authority_weights" + "local_authority", + ROW_FILTER, + "la_code_oa", + requires_release_dataset=True, ), } @@ -226,34 +232,6 @@ def _reject_constant(constant: str) -> object: raise ValueError(msg) -def _checked_weight_matrix( - country_id: str, - reference: RegionReference, - matrix_name: str, - dataset_name: str, - artifacts: Mapping[str, CertifiedArtifact], -) -> None: - """Require the weight matrix from the same release as the dataset it reweights. - - Weights attach to households by position, so only a matrix built with the - region's dataset lines up with its households. - """ - matrix = artifacts.get(matrix_name) - if matrix is None: - msg = ( - f"region {reference.code!r} reweights households with {matrix_name}, " - f"which the {country_id} release does not certify" - ) - raise CatalogExtractionError(msg) - dataset = artifacts[dataset_name] - if (matrix.repo_id, matrix.revision) != (dataset.repo_id, dataset.revision): - msg = ( - f"region {reference.code!r} reweights households with {matrix_name}, " - f"which is not certified in the release of {dataset_name}" - ) - raise CatalogExtractionError(msg) - - def scope_regions( country_id: str, references: tuple[RegionReference, ...], @@ -302,17 +280,16 @@ def scope_regions( _checked_filter_value(reference, rule) dataset_name = role_datasets.get(rule.dataset_role) if dataset_name is None: + if rule.requires_release_dataset: + msg = ( + f"the {country_id} release routes no dataset to " + f"{rule.dataset_role} regions, so region {reference.code!r} " + "cannot select its households" + ) + raise CatalogExtractionError(msg) dataset_name = reviewed_default if rule.dataset_role != NATIONAL_DATASET_ROLE: fallback_counts[reference.region_type] += 1 - if rule.filter_strategy == WEIGHT_REPLACEMENT: - _checked_weight_matrix( - country_id, - reference, - rule.filter_field or "", - dataset_name, - artifacts, - ) regions.append( ScopedRegion( reference=reference, @@ -333,18 +310,10 @@ def scope_regions( def filtered_artifacts(scoping: RegionScoping) -> tuple[CertifiedArtifact, ...]: - """Return, by name, the artifacts the regions' filters read. - - Those are the datasets of row-filtered regions and the weight matrices of - weight-replacement regions. - """ + """Return, by name, the datasets the regions' row filters read.""" names = { region.dataset_name for region in scoping.regions if region.filter_strategy == ROW_FILTER - } | { - region.filter_field - for region in scoping.regions - if region.filter_strategy == WEIGHT_REPLACEMENT and region.filter_field } return tuple(scoping.certified_artifacts[name] for name in sorted(names)) diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 324ad521b..37f04f7be 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -113,9 +113,7 @@ class Region(TimestampedModel, table=True): Its default dataset must belong to the same version. Filtered regions (``requires_filter``) also name how they select their households: a ``row_filter`` keeps the households whose ``filter_field`` column matches - ``filter_value``; a ``weight_replacement`` filter reweights every household - with row ``filter_value`` of the weight matrix ``filter_field`` names in the - certified data release. + ``filter_value``. """ __tablename__ = "regions" diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 97ec2ebeb..a7336d7e9 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -139,9 +139,7 @@ class MetadataRegion(StrictResponseModel): When requires_filter is true, filter_strategy says how the region selects households of its default dataset. With "row_filter", it keeps the households whose filter_field column equals filter_value (or is in it, - when filter_value is a JSON list). With "weight_replacement", it keeps - every household and gives each the weight in row filter_value of the - weight matrix that filter_field names in the certified data release. + when filter_value is a JSON list). """ id: UUID diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index e3de5fc62..60c44ff81 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -138,13 +138,18 @@ def _release_contents(country_id: str) -> tuple[dict, dict, SimpleNamespace]: return datasets, region_datasets, data_package datasets = { "microcosm_uk_2024_25": _certified_artifact("microcosm_uk_2024_25.h5"), - "parliamentary_constituency_weights": _certified_artifact( - "parliamentary_constituency_weights.h5" + "microcosm_uk_2024_25_dense": _certified_artifact( + "microcosm_uk_2024_25_dense.h5" ), - "local_authority_weights": _certified_artifact("local_authority_weights.h5"), } + # As in PolicyEngine.py 6.10.0, constituencies and local authorities are + # routed to the local-area dataset, which carries their codes. region_datasets = { "national": SimpleNamespace(path_template="microcosm_uk_2024_25.h5"), + "constituency": SimpleNamespace(path_template="microcosm_uk_2024_25_dense.h5"), + "local_authority": SimpleNamespace( + path_template="microcosm_uk_2024_25_dense.h5" + ), } data_package = SimpleNamespace( repo_id="example-org/example-uk-data", diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index 778b72d07..b705d68d0 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -6,12 +6,16 @@ import os from collections import Counter from importlib import metadata as importlib_metadata +from typing import TYPE_CHECKING import pytest from policyengine.bundle import get_current_bundle from policyengine_api.data.v2.catalog.extraction import extract_installed_catalog +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.records import CountryCatalog + pytestmark = pytest.mark.skipif( os.environ.get("RUN_V2_CATALOG_COMPATIBILITY") != "1", reason=("loads the complete public catalog; set RUN_V2_CATALOG_COMPATIBILITY=1"), @@ -39,10 +43,9 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: ) # Counts for the 6.10.0 bundle's models: US 2.2.1 and UK 2.132.6. Keep # this exact assertion so later package changes require explicit review. - # Since 6.10.0 the UK registry also lists its 650 constituencies and 361 - # local authorities, because the bundle routes them to a local-area - # release, so regions grow by 1,011 from 6.7.5's 826 and the catalog - # adds that release's dataset. + # Regions come from the checked-in reference files: 488 US (places are + # left out) and 1,016 UK, whose constituencies and local authorities use + # the local-area dataset the release routes them to. assert catalog.entity_counts() == { "models": 2, "model_versions": 2, @@ -50,8 +53,8 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "parameter_nodes": 29_307, "parameters": 104_172, "parameter_values": 1_194_360, - "datasets": 3, - "regions": 1_503, + "datasets": 4, + "regions": 1_504, } for country_id in ("us", "uk"): @@ -74,18 +77,18 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: total_values += len(batch) assert total_values == country.entity_counts()["parameter_values"] - assert {dataset.name for dataset in catalog.country("us").datasets} == { - "populace_us_2024", - "populace_us_2024_acs_local", - } - assert {dataset.name for dataset in catalog.country("uk").datasets} == { + _check_uk_regions(catalog.country("uk")) + _check_us_regions(catalog.country("us")) + + +def _check_uk_regions(uk_catalog: CountryCatalog) -> None: + assert {dataset.name for dataset in uk_catalog.datasets} == { "microcosm_uk_2024_25", "microcosm_uk_2024_25_dense", } # The national file has no area codes, so UK constituencies and local # authorities use the local-area dataset .py routes them to, while the # UK countries filter the national file. - uk_catalog = catalog.country("uk") uk_dataset_names = {dataset.id: dataset.name for dataset in uk_catalog.datasets} uk_region_type_datasets: dict[str, set[str]] = {} for region in uk_catalog.regions: @@ -100,9 +103,7 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: } assert uk_catalog.fallback_summaries == () england = next( - region - for region in catalog.country("uk").regions - if region.code == "country/england" + region for region in uk_catalog.regions if region.code == "country/england" ) assert england.filter_field == "region" assert england.filter_strategy == "row_filter" @@ -119,12 +120,39 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "SOUTH_WEST", ] + uk_types = Counter(region.region_type for region in uk_catalog.regions) + assert uk_types == { + "national": 1, + "country": 4, + "constituency": 650, + "local_authority": 361, + } + uk_regions = {region.code: region for region in uk_catalog.regions} + aldershot = uk_regions["constituency/E14001063"] + assert ( + aldershot.filter_strategy, + aldershot.filter_field, + aldershot.filter_value, + ) == ("row_filter", "constituency_code_oa", "E14001063") + ards = uk_regions["local_authority/N09000011"] + assert (ards.label, ards.filter_field, ards.filter_value) == ( + "Ards and North Down", + "la_code_oa", + "N09000011", + ) + assert uk_regions["uk"].label == "United Kingdom" + + +def _check_us_regions(us_catalog: CountryCatalog) -> None: + assert {dataset.name for dataset in us_catalog.datasets} == { + "populace_us_2024", + "populace_us_2024_acs_local", + } # Since 6.7.5, US state and congressional-district regions use the # ACS-local dataset again, on the PolicyEngine/microcosm#1263 rebuild # (PolicyEngine/policyengine.py#649, ending #604's national-file filter). # The national region uses the national dataset. Regions come from the # checked-in reference files; US places cannot run yet and are left out. - us_catalog = catalog.country("us") us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets} region_type_datasets: dict[str, set[str]] = {} for region in us_catalog.regions: @@ -142,22 +170,6 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: for summary in us_catalog.omitted_region_summaries ] == [("place", 333)] - uk_catalog = catalog.country("uk") - uk_types = Counter(region.region_type for region in uk_catalog.regions) - assert uk_types == { - "national": 1, - "country": 4, - "constituency": 650, - "local_authority": 360, - } - uk_regions = {region.code: region for region in uk_catalog.regions} - aldershot = uk_regions["constituency/E14001063"] - assert ( - aldershot.filter_strategy, - aldershot.filter_field, - aldershot.filter_value, - ) == ("weight_replacement", "parliamentary_constituency_weights", "E14001063") - assert uk_regions["uk"].label == "United Kingdom" assert {region.code: region.label for region in us_catalog.regions}["us"] == ( "United States" ) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 70a275895..71808bc6e 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -135,7 +135,7 @@ def test_publish_and_republish_preserve_complete_catalog( "parameter_nodes": 2, "parameters": 2, "parameter_values": 4, - "datasets": 3, + "datasets": 4, "dataset_versions": 0, "regions": 6, "simulations": 0, @@ -167,7 +167,7 @@ def test_publish_and_republish_preserve_complete_catalog( ) assert dict(region_defaults) == { "congressional_district/CA-01": "populace_us_2024_acs_local", - "constituency/E14001063": "microcosm_uk_2024_25", + "constituency/E14001063": "microcosm_uk_2024_25_dense", "country/england": "microcosm_uk_2024_25", "state/ca": "populace_us_2024_acs_local", "uk": "microcosm_uk_2024_25", @@ -541,7 +541,7 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "parameter_nodes": 4, "parameters": 4, "parameter_values": 8, - "datasets": 6, + "datasets": 8, "dataset_versions": 0, "regions": 12, "simulations": 0, diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index 8bd5cf519..8b7ecec7f 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -42,7 +42,10 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: "populace_us_2024", "populace_us_2024_acs_local", } - assert {dataset.name for dataset in uk.datasets} == {"microcosm_uk_2024_25"} + assert {dataset.name for dataset in uk.datasets} == { + "microcosm_uk_2024_25", + "microcosm_uk_2024_25_dense", + } assert all( not dataset.is_output_dataset for dataset in (*us.datasets, *uk.datasets) ) @@ -62,7 +65,14 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: (summary.region_type, summary.count) for summary in us.omitted_region_summaries ] == [("place", 1)] assert uk.omitted_region_summaries == () - assert all(region.default_dataset_id == uk.datasets[0].id for region in uk.regions) + uk_datasets = {dataset.id: dataset.name for dataset in uk.datasets} + assert { + region.code: uk_datasets[region.default_dataset_id] for region in uk.regions + } == { + "uk": "microcosm_uk_2024_25", + "country/england": "microcosm_uk_2024_25", + "constituency/E14001063": "microcosm_uk_2024_25_dense", + } def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> None: @@ -124,8 +134,8 @@ def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> "constituency", "country/england", True, - "weight_replacement", - "parliamentary_constituency_weights", + "row_filter", + "constituency_code_oa", "E14001063", None, None, diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index 74c9f4459..c0a314458 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -113,19 +113,10 @@ def _name_a_per_region_dataset_template( ].path_template = "populace_{state_code}.h5" -def _remove_the_weight_matrix( +def _stop_routing_constituencies( models: dict[str, SimpleNamespace], _references: References ) -> None: - del models["uk"].release_manifest.datasets["parliamentary_constituency_weights"] - - -def _take_the_weight_matrix_from_another_release( - models: dict[str, SimpleNamespace], _references: References -) -> None: - matrix = models["uk"].release_manifest.datasets[ - "parliamentary_constituency_weights" - ] - matrix.revision = "another-release" + del models["uk"].release_manifest.region_datasets["constituency"] def _drop_a_filter_value( @@ -257,16 +248,9 @@ def _remove_every_artifact_repository( r"'populace_\{state_code\}.h5', which the catalog does not support$", ), ( - _remove_the_weight_matrix, - "^region 'constituency/E14001063' reweights households with " - "parliamentary_constituency_weights, which the uk release does not " - "certify$", - ), - ( - _take_the_weight_matrix_from_another_release, - "^region 'constituency/E14001063' reweights households with " - "parliamentary_constituency_weights, which is not certified in the " - "release of microcosm_uk_2024_25$", + _stop_routing_constituencies, + "^the uk release routes no dataset to constituency regions, so region " + "'constituency/E14001063' cannot select its households$", ), (_drop_a_filter_value, "^region 'state/ca' has no filter value$"), ( diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index 1870d431d..d4d6c22c7 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -200,7 +200,10 @@ def test_main_reports_a_region_data_failure_with_its_message( capsys: pytest.CaptureFixture[str], ) -> None: def fail(): - msg = "the weight matrix m has no row for 1 uk region (constituency/x)" + msg = ( + "the certified dataset d has no household column c, which 1 uk region " + "filters on (constituency/x)" + ) raise RegionDataVerificationError(msg) monkeypatch.setattr(initialization, "initialize_catalog", fail) @@ -208,7 +211,7 @@ def fail(): assert initialization.main() == 1 failure = capsys.readouterr() assert '"type": "RegionDataVerificationError"' in failure.err - assert "has no row for 1 uk region" in failure.err + assert "which 1 uk region filters on" in failure.err def test_extraction_failure_opens_no_engine() -> None: diff --git a/tests/unit/v2/test_certified_data_inspection.py b/tests/unit/v2/test_certified_data_inspection.py index be0be2c7e..a0b375909 100644 --- a/tests/unit/v2/test_certified_data_inspection.py +++ b/tests/unit/v2/test_certified_data_inspection.py @@ -1,4 +1,4 @@ -"""Reading column names and weight-matrix sizes from certified HDF5 artifacts.""" +"""Reading household column names from certified HDF5 datasets.""" from __future__ import annotations @@ -20,7 +20,6 @@ HuggingFaceDataInspector, household_columns_in_hdf5, huggingface_path, - weight_matrix_rows_in_hdf5, ) from policyengine_api.data.v2.catalog.records import CertifiedArtifact @@ -54,11 +53,6 @@ def _columns(path: Path) -> frozenset[str]: return household_columns_in_hdf5(h5) -def _rows(path: Path) -> int: - with h5py.File(path, "r") as h5: - return weight_matrix_rows_in_hdf5(h5) - - def test_reads_columns_of_a_fixed_format_household_frame(tmp_path: Path) -> None: path = tmp_path / "fixed.h5" HOUSEHOLDS.to_hdf(path, key="household", format="fixed") @@ -139,16 +133,6 @@ def test_refuses_a_household_entry_that_is_not_a_group(tmp_path: Path) -> None: _columns(path) -def test_counts_the_region_rows_of_a_weight_matrix(tmp_path: Path) -> None: - path = tmp_path / "matrix.h5" - with h5py.File(path, "w") as h5: - h5.create_dataset("2024", data=np.ones((3, 5))) - h5.create_dataset("2025", data=np.ones((3, 5))) - h5.create_group("metadata").create_dataset("note", data=np.ones(1)) - - assert _rows(path) == 3 - - def test_default_file_system_is_hugging_face_with_the_token() -> None: inspector = HuggingFaceDataInspector(token="read-token") @@ -157,31 +141,6 @@ def test_default_file_system_is_hugging_face_with_the_token() -> None: assert file_system.token == "read-token" -@pytest.mark.parametrize( - ("shapes", "message"), - [ - ({}, r"^the weight matrix has no year datasets$"), - ({"2024": (3,)}, r"^the weight matrix entry 2024 is not two-dimensional$"), - ( - {"2024": (3, 5), "2025": (4, 5)}, - r"^the weight matrix years disagree on the number of regions: \[3, 4\]$", - ), - ], -) -def test_refuses_an_inconsistent_weight_matrix( - tmp_path: Path, - shapes: dict[str, tuple[int, ...]], - message: str, -) -> None: - path = tmp_path / "matrix.h5" - with h5py.File(path, "w") as h5: - for year, shape in shapes.items(): - h5.create_dataset(year, data=np.ones(shape)) - - with pytest.raises(CertifiedDataReadError, match=message): - _rows(path) - - def test_names_dataset_and_model_repository_files_at_their_revision() -> None: assert huggingface_path(_artifact()) == ( "datasets/policyengine/populace-us@" @@ -189,8 +148,8 @@ def test_names_dataset_and_model_repository_files_at_their_revision() -> None: "populace_us_2024_acs_local.h5" ) assert huggingface_path( - _artifact(name="enhanced_frs_2024_25", repo_type="model", revision="1.58.1") - ) == ("policyengine/populace-us@1.58.1/enhanced_frs_2024_25.h5") + _artifact(name="microcosm_uk_2024_25", repo_type="model", revision="1.0.0") + ) == ("policyengine/populace-us@1.0.0/microcosm_uk_2024_25.h5") assert huggingface_path(_artifact(revision="refs/pr/1")).startswith( "datasets/policyengine/populace-us@refs%2Fpr%2F1/" ) @@ -214,15 +173,16 @@ def test_inspector_reads_artifacts_through_an_authenticated_file_system( ) -> None: dataset_path = tmp_path / "dataset.h5" HOUSEHOLDS.to_hdf(dataset_path, key="household", format="fixed") - matrix_path = tmp_path / "matrix.h5" - with h5py.File(matrix_path, "w") as h5: - h5.create_dataset("2025", data=np.ones((2, 2))) + local_area_path = tmp_path / "local_area.h5" + HOUSEHOLDS.assign(la_code_oa=["N09000011", "E07000110"]).to_hdf( + local_area_path, key="household", format="table", data_columns=True + ) dataset = _artifact() - matrix = _artifact(name="parliamentary_constituency_weights", repo_type="model") + local_area = _artifact(name="microcosm_uk_2024_25_dense", repo_type="dataset") file_system = FakeFileSystem( { huggingface_path(dataset): dataset_path, - huggingface_path(matrix): matrix_path, + huggingface_path(local_area): local_area_path, } ) tokens: list[str | None] = [] @@ -236,9 +196,12 @@ def build_file_system(token: str | None) -> FakeFileSystem: ) assert "state_fips" in inspector.household_columns(dataset) - assert inspector.weight_matrix_row_count(matrix) == 2 + assert "la_code_oa" in inspector.household_columns(local_area) assert tokens == ["read-token"] - assert file_system.opened == [huggingface_path(dataset), huggingface_path(matrix)] + assert file_system.opened == [ + huggingface_path(dataset), + huggingface_path(local_area), + ] def test_imports_only_packages_the_project_depends_on_directly() -> None: diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py index d2d1cfa2d..11c719a41 100644 --- a/tests/unit/v2/test_region_data_verification.py +++ b/tests/unit/v2/test_region_data_verification.py @@ -8,9 +8,7 @@ from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError from policyengine_api.data.v2.catalog.region_data_verification import ( - WEIGHT_MATRIX_ROW_LOOKUPS, RegionDataVerificationError, - load_weight_matrix_row_codes, verify_region_data, ) from tests.fixtures.v2_catalog import normalized_catalog @@ -23,54 +21,38 @@ {"state_fips", "congressional_district_geoid", "household_weight"} ), "microcosm_uk_2024_25": frozenset({"region", "household_weight"}), + "microcosm_uk_2024_25_dense": frozenset( + {"constituency_code_oa", "la_code_oa", "household_weight"} + ), } -CONSTITUENCY_ROW_CODES = ("E14001063", "E14001064") class FakeInspector: - """Answers from fixed columns and matrix sizes, and records each artifact read.""" + """Answers from fixed columns, and records each artifact read.""" def __init__( self, *, columns: dict[str, frozenset[str]] | None = None, - matrix_rows: dict[str, int] | None = None, failure: Exception | None = None, ) -> None: self.columns = ALL_COLUMNS if columns is None else columns - self.matrix_rows = ( - {"parliamentary_constituency_weights": len(CONSTITUENCY_ROW_CODES)} - if matrix_rows is None - else matrix_rows - ) self.failure = failure - self.reads: list[tuple[str, str]] = [] + self.reads: list[str] = [] def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: - self.reads.append(("columns", artifact.name)) + self.reads.append(artifact.name) if self.failure is not None: raise self.failure return self.columns[artifact.name] - def weight_matrix_row_count(self, artifact: CertifiedArtifact) -> int: - self.reads.append(("rows", artifact.name)) - if self.failure is not None: - raise self.failure - return self.matrix_rows[artifact.name] - def _verify( inspector: FakeInspector, *, countries: tuple[str, ...] = ("us", "uk"), - row_codes: tuple[str, ...] = CONSTITUENCY_ROW_CODES, ) -> None: - verify_region_data( - normalized_catalog(), - inspector, - countries=countries, - weight_matrix_row_codes={"parliamentary_constituency_weights": row_codes}, - ) + verify_region_data(normalized_catalog(), inspector, countries=countries) def test_catalog_records_only_the_artifacts_its_filters_read() -> None: @@ -98,23 +80,23 @@ def test_catalog_records_only_the_artifacts_its_filters_read() -> None: "model", ), ( - "parliamentary_constituency_weights", + "microcosm_uk_2024_25_dense", "example-org/example-uk-data", - "parliamentary_constituency_weights.h5", + "microcosm_uk_2024_25_dense.h5", "model", ), ] -def test_accepts_regions_whose_columns_and_matrix_rows_exist() -> None: +def test_accepts_regions_whose_filter_columns_exist() -> None: inspector = FakeInspector() _verify(inspector) assert sorted(inspector.reads) == [ - ("columns", "microcosm_uk_2024_25"), - ("columns", "populace_us_2024_acs_local"), - ("rows", "parliamentary_constituency_weights"), + "microcosm_uk_2024_25", + "microcosm_uk_2024_25_dense", + "populace_us_2024_acs_local", ] @@ -123,7 +105,7 @@ def test_reads_only_the_artifacts_of_the_countries_it_verifies() -> None: _verify(inspector, countries=("us",)) - assert inspector.reads == [("columns", "populace_us_2024_acs_local")] + assert inspector.reads == ["populace_us_2024_acs_local"] def test_verifies_nothing_when_every_version_is_already_published() -> None: @@ -151,38 +133,21 @@ def test_rejects_a_filter_column_missing_from_the_dataset() -> None: _verify(inspector) -def test_rejects_a_region_code_missing_from_the_weight_matrix() -> None: - with pytest.raises( - RegionDataVerificationError, - match=r"^the weight matrix parliamentary_constituency_weights has no row for " - r"1 uk region \(constituency/E14001063\)$", - ): - _verify(FakeInspector(), row_codes=("E14001064", "E14001065")) - - -def test_rejects_a_weight_matrix_whose_rows_differ_from_its_lookup() -> None: - inspector = FakeInspector(matrix_rows={"parliamentary_constituency_weights": 3}) - - with pytest.raises( - RegionDataVerificationError, - match=r"^the weight matrix parliamentary_constituency_weights has 3 region " - r"rows, but its row lookup lists 2 codes$", - ): - _verify(inspector) - +def test_rejects_a_local_area_dataset_without_the_area_code_column() -> None: + inspector = FakeInspector( + columns={ + **ALL_COLUMNS, + "microcosm_uk_2024_25_dense": frozenset({"la_code_oa"}), + } + ) -def test_rejects_a_weight_matrix_without_a_row_lookup() -> None: with pytest.raises( RegionDataVerificationError, - match=r"^no row lookup names the regions of the weight matrix " - r"parliamentary_constituency_weights$", + match=r"^the certified dataset microcosm_uk_2024_25_dense has no household " + r"column constituency_code_oa, which 1 uk region filters on " + r"\(constituency/E14001063\)$", ): - verify_region_data( - normalized_catalog(), - FakeInspector(), - countries=("uk",), - weight_matrix_row_codes={}, - ) + _verify(inspector, countries=("uk",)) def test_reports_an_unreadable_artifact_without_its_error_text() -> None: @@ -201,20 +166,6 @@ def test_reports_an_unreadable_artifact_without_its_error_text() -> None: assert "fixture" not in str(raised.value) -def test_row_lookups_list_every_reference_code_in_matrix_order() -> None: - constituencies = load_weight_matrix_row_codes( - WEIGHT_MATRIX_ROW_LOOKUPS["parliamentary_constituency_weights"] - ) - local_authorities = load_weight_matrix_row_codes( - WEIGHT_MATRIX_ROW_LOOKUPS["local_authority_weights"] - ) - - assert len(constituencies) == len(set(constituencies)) == 650 - assert len(local_authorities) == len(set(local_authorities)) == 361 - assert constituencies[0] == "E14001063" - assert local_authorities[0] == "E06000001" - - def test_reports_an_artifact_structure_error_with_its_own_message() -> None: inspector = FakeInspector( failure=CertifiedDataReadError("household group holds no data frame") @@ -233,11 +184,3 @@ def test_passes_on_a_verification_error_the_inspector_raises() -> None: with pytest.raises(RegionDataVerificationError, match=r"^already explained$"): _verify(inspector, countries=("us",)) - - -def test_reads_the_checked_in_row_lookups_by_default() -> None: - inspector = FakeInspector(matrix_rows={"parliamentary_constituency_weights": 650}) - - verify_region_data(normalized_catalog(), inspector, countries=("uk",)) - - assert ("rows", "parliamentary_constituency_weights") in inspector.reads From cbb1656cf40aa59494c1ed793356bf1f3279b141 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:43:47 +0400 Subject: [PATCH 041/118] Cover UK local authorities and the UK's lack of fallback at unit level The catalog fixture's UK reference file now also lists Northern Ireland and Ards and North Down (local_authority/N09000011). The unit tests assert that the local authority is row-filtered on la_code_oa with its ONS code in the local-area dataset the release routes it to, that the UK records no fallback, and that a release routing no dataset to local authorities stops publication naming the region, as for constituencies. These replace, at unit level, master's registry-based test of the UK area regions' dataset. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/v2_catalog.py | 20 ++++++++++++++ .../test_v2_catalog_publication.py | 6 +++-- tests/unit/v2/test_catalog_extraction.py | 27 +++++++++++++++++++ .../v2/test_catalog_extraction_failures.py | 11 ++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 60c44ff81..61e569f95 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -99,6 +99,26 @@ def region_references() -> dict[str, tuple[RegionReference, ...]]: filter_value="E14001063", v1_code="constituency/Aldershot", ), + RegionReference( + code="country/northern_ireland", + label="Northern Ireland", + region_type="country", + parent_code="uk", + state_code=None, + state_name=None, + filter_value="NORTHERN_IRELAND", + v1_code="country/ni", + ), + RegionReference( + code="local_authority/N09000011", + label="Ards and North Down", + region_type="local_authority", + parent_code="country/northern_ireland", + state_code=None, + state_name=None, + filter_value="N09000011", + v1_code="local_authority/Ards and North Down", + ), ), } diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 71808bc6e..edef7b93e 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -137,7 +137,7 @@ def test_publish_and_republish_preserve_complete_catalog( "parameter_values": 4, "datasets": 4, "dataset_versions": 0, - "regions": 6, + "regions": 8, "simulations": 0, "reports": 0, "report_runs": 0, @@ -169,6 +169,8 @@ def test_publish_and_republish_preserve_complete_catalog( "congressional_district/CA-01": "populace_us_2024_acs_local", "constituency/E14001063": "microcosm_uk_2024_25_dense", "country/england": "microcosm_uk_2024_25", + "country/northern_ireland": "microcosm_uk_2024_25", + "local_authority/N09000011": "microcosm_uk_2024_25_dense", "state/ca": "populace_us_2024_acs_local", "uk": "microcosm_uk_2024_25", "us": "populace_us_2024", @@ -543,7 +545,7 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "parameter_values": 8, "datasets": 8, "dataset_versions": 0, - "regions": 12, + "regions": 16, "simulations": 0, "reports": 0, "report_runs": 0, diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index 8b7ecec7f..bc6d45eca 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -71,8 +71,13 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: } == { "uk": "microcosm_uk_2024_25", "country/england": "microcosm_uk_2024_25", + "country/northern_ireland": "microcosm_uk_2024_25", "constituency/E14001063": "microcosm_uk_2024_25_dense", + "local_authority/N09000011": "microcosm_uk_2024_25_dense", } + # UK nations filter the national dataset by design, and constituencies and + # local authorities use the dataset the release routes them to. + assert uk.fallback_summaries == () def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> None: @@ -140,6 +145,28 @@ def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> None, None, ), + "country/northern_ireland": ( + "Northern Ireland", + "country", + "uk", + True, + "row_filter", + "region", + "NORTHERN_IRELAND", + None, + None, + ), + "local_authority/N09000011": ( + "Ards and North Down", + "local_authority", + "country/northern_ireland", + True, + "row_filter", + "la_code_oa", + "N09000011", + None, + None, + ), } diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index c0a314458..70f5f954c 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -119,6 +119,12 @@ def _stop_routing_constituencies( del models["uk"].release_manifest.region_datasets["constituency"] +def _stop_routing_local_authorities( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["uk"].release_manifest.region_datasets["local_authority"] + + def _drop_a_filter_value( _models: dict[str, SimpleNamespace], references: References ) -> None: @@ -252,6 +258,11 @@ def _remove_every_artifact_repository( "^the uk release routes no dataset to constituency regions, so region " "'constituency/E14001063' cannot select its households$", ), + ( + _stop_routing_local_authorities, + "^the uk release routes no dataset to local_authority regions, so region " + "'local_authority/N09000011' cannot select its households$", + ), (_drop_a_filter_value, "^region 'state/ca' has no filter value$"), ( _give_the_national_region_a_filter_value, From d2f6c4ccbc524d4029652f995c0354613d8e2dab Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:44:13 +0400 Subject: [PATCH 042/118] Rewrap the Stage 9 region paragraphs like the surrounding text Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 61e37b49d..3825ad81c 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -148,9 +148,9 @@ staged region rows the stored version lacks and of stored rows not staged, and continues. Its evidence then describes what is stored: the region count is the stored one, `kept_regions` lists the country, version, stored and staged region counts, and the fallback and omitted-region summaries leave that country out, -since they describe staged regions that were not published. Every other table must still match exactly. -Region reference changes therefore take effect in the next catalog version -that is published. +since they describe staged regions that were not published. Every other table +must still match exactly. Region reference changes therefore take effect in the +next catalog version that is published. ## Validation evidence and retry @@ -181,9 +181,10 @@ A failed attempt rolls back all catalog changes. Retry the same artifact only after correcting the external failure. A matching retry preserves identifiers, content, and row counts. Different normalized content under the same PolicyEngine.py version fails for operator review, with an error naming the -first table that differs; it is never silently replaced. Regions are the exception: they come from the Public API's reference -files, so a stored version keeps the regions it was published with (see -"Source and regional dataset behavior"). A later PolicyEngine.py version adds a catalog version without +first table that differs; it is never silently replaced. Regions are the +exception: they come from the Public API's reference files, so a stored version +keeps the regions it was published with (see "Source and regional dataset +behavior"). A later PolicyEngine.py version adds a catalog version without deleting or modifying earlier versioned rows, including their logical datasets, regions, model descriptions, current-law IDs, and time-period options. From d6626794366003b53c2c1fe554a15306bdf2a98a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:04:55 +0400 Subject: [PATCH 043/118] Add nullable catalog format 2 columns for v3's metadata Catalog format 2 stores the metadata the app's v3 workspace reads, which PolicyEngine.py now publishes in its public records: - model versions: the catalog format, entities and basic inputs; - variables: the input and hidden-input flags, the module, the parameter named by adds or subtracts, and enum value labels; - parameters: the economy and household flags. Revision f5c1d9b4685b was autogenerated against the v2 configuration and adds every column as nullable with no default, so catalog versions published before format 2 keep them empty. It writes no data. Co-Authored-By: Claude Opus 5.5 --- ...9b4685b_store_catalog_format_2_metadata.py | 80 +++++++++++ policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/metadata.py | 17 +++ .../integration/test_alembic_v2_lifecycle.py | 131 ++++++++++++++++++ tests/unit/v2/test_alembic_v2.py | 50 +++++++ tests/unit/v2/test_models.py | 45 ++++++ 6 files changed, 324 insertions(+), 1 deletion(-) create mode 100644 migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py diff --git a/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py new file mode 100644 index 000000000..b0dc56d28 --- /dev/null +++ b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py @@ -0,0 +1,80 @@ +"""store catalog format 2 metadata + +Revision ID: f5c1d9b4685b +Revises: fa7fa1035444 +Create Date: 2026-10-22 18:59:48.186978 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "f5c1d9b4685b" +down_revision: Union[str, None] = "fa7fa1035444" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("parameters", sa.Column("economy", sa.Boolean(), nullable=True)) + op.add_column("parameters", sa.Column("household", sa.Boolean(), nullable=True)) + op.add_column( + "tax_benefit_model_versions", + sa.Column("catalog_format", sa.Integer(), nullable=True), + ) + op.add_column( + "tax_benefit_model_versions", sa.Column("entities", sa.JSON(), nullable=True) + ) + op.add_column( + "tax_benefit_model_versions", + sa.Column("basic_inputs", sa.JSON(), nullable=True), + ) + op.add_column( + "variables", sa.Column("is_input_variable", sa.Boolean(), nullable=True) + ) + op.add_column("variables", sa.Column("hidden_input", sa.Boolean(), nullable=True)) + op.add_column( + "variables", + sa.Column( + "module_name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True + ), + ) + op.add_column( + "variables", + sa.Column( + "adds_parameter_path", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + ) + op.add_column( + "variables", + sa.Column( + "subtracts_parameter_path", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + ) + op.add_column("variables", sa.Column("enum_values", sa.JSON(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("variables", "enum_values") + op.drop_column("variables", "subtracts_parameter_path") + op.drop_column("variables", "adds_parameter_path") + op.drop_column("variables", "module_name") + op.drop_column("variables", "hidden_input") + op.drop_column("variables", "is_input_variable") + op.drop_column("tax_benefit_model_versions", "basic_inputs") + op.drop_column("tax_benefit_model_versions", "entities") + op.drop_column("tax_benefit_model_versions", "catalog_format") + op.drop_column("parameters", "household") + op.drop_column("parameters", "economy") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index a57c2419c..d463fad16 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fa7fa1035444" +V2_ALEMBIC_HEAD_REVISION = "f5c1d9b4685b" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 37f04f7be..ca0153467 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -85,6 +85,11 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): description: str | None = None current_law_id: int metadata_time_periods: list[int] = Field(sa_type=sa.JSON) + # Catalog format 2 fields. Versions published before format 2 keep them + # null; entities are {key, plural, is_person} objects. + catalog_format: int | None = None + entities: list[dict[str, Any]] | None = Field(default=None, sa_type=sa.JSON) + basic_inputs: list[str] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") variables: list["Variable"] = Relationship( @@ -277,6 +282,15 @@ class Variable(IdentifiedModel, table=True): default_value: Any = Field(default=None, sa_type=sa.JSON) adds: list[str] | None = Field(default=None, sa_type=sa.JSON) subtracts: list[str] | None = Field(default=None, sa_type=sa.JSON) + # Catalog format 2 fields; null in versions published before it. A + # parameter path names the parameter whose dated value lists the variables + # this one adds or subtracts. Enum values are {value, label} objects. + is_input_variable: bool | None = None + hidden_input: bool | None = None + module_name: str | None = Field(default=None, max_length=512) + adds_parameter_path: str | None = Field(default=None, max_length=512) + subtracts_parameter_path: str | None = Field(default=None, max_length=512) + enum_values: list[dict[str, Any]] | None = Field(default=None, sa_type=sa.JSON) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", @@ -337,6 +351,9 @@ class Parameter(IdentifiedModel, table=True): description: str | None = None data_type: str | None = Field(default=None, max_length=128) unit: str | None = Field(default=None, max_length=128) + # Catalog format 2 fields; null in versions published before it. + economy: bool | None = None + household: bool | None = None tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 5ef7c6169..61ff7a72b 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -1414,3 +1414,134 @@ def test_previous_release_still_finds_user_policies() -> None: if saved is not None: _delete_saved_policy_rows(engine, parent, saved) engine.dispose() + + +CATALOG_FORMAT_2_PARENT_REVISION = "fa7fa1035444" +CATALOG_FORMAT_2_COLUMNS = { + "tax_benefit_model_versions": ("catalog_format", "entities", "basic_inputs"), + "variables": ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + ), + "parameters": ("economy", "household"), +} + + +def _catalog_version_published_before_format_2(connection: sa.Connection) -> UUID: + """Store a model version with one variable and one parameter; return its id.""" + model_id = uuid4() + version_id = uuid4() + connection.execute( + text( + "INSERT INTO tax_benefit_models (id, name) " + "VALUES (:model_id, 'catalog-format-probe')" + ), + {"model_id": model_id}, + ) + connection.execute( + text( + "INSERT INTO tax_benefit_model_versions " + "(id, model_id, version, current_law_id, metadata_time_periods) " + "VALUES (:version_id, :model_id, '0.0.1-probe', 1, '[2026]')" + ), + {"version_id": version_id, "model_id": model_id}, + ) + connection.execute( + text( + "INSERT INTO variables " + "(id, name, entity, default_value, tax_benefit_model_version_id) " + "VALUES (:variable_id, 'probe_income', 'person', '0', :version_id)" + ), + {"variable_id": uuid4(), "version_id": version_id}, + ) + connection.execute( + text( + "INSERT INTO parameters (id, name, tax_benefit_model_version_id) " + "VALUES (:parameter_id, 'gov.probe.rate', :version_id)" + ), + {"parameter_id": uuid4(), "version_id": version_id}, + ) + return version_id + + +def _format_2_values(connection: sa.Connection, version_id: UUID) -> list[tuple]: + """Return the format 2 columns of the version's model, variable and parameter.""" + rows = [] + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + key = ( + "id" + if table_name == "tax_benefit_model_versions" + else "tax_benefit_model_version_id" + ) + rows.append( + tuple( + connection.execute( + text( + f"SELECT {', '.join(columns)} FROM {table_name} " + f"WHERE {key} = :version_id" + ), + {"version_id": version_id}, + ).one() + ) + ) + return rows + + +def test_catalog_format_2_columns_leave_earlier_versions_empty() -> None: + """A version stored before catalog format 2 keeps its format 2 fields null. + + Upgrading adds the columns as nullable without writing to stored rows, and + downgrading drops them and keeps every row. + """ + config = _config() + engine = create_engine(_disposable_url()) + version_id = None + try: + command.downgrade(config, CATALOG_FORMAT_2_PARENT_REVISION) + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + assert set(columns).isdisjoint(_columns(engine, table_name)) + with engine.begin() as connection: + version_id = _catalog_version_published_before_format_2(connection) + + command.upgrade(config, "head") + + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + stored_columns = _columns(engine, table_name) + for column_name in columns: + assert stored_columns[column_name]["nullable"] + assert stored_columns[column_name]["default"] is None + with engine.connect() as connection: + assert _format_2_values(connection, version_id) == [ + (None, None, None), + (None, None, None, None, None, None), + (None, None), + ] + + command.downgrade(config, CATALOG_FORMAT_2_PARENT_REVISION) + + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + assert set(columns).isdisjoint(_columns(engine, table_name)) + with engine.connect() as connection: + counts = connection.execute( + text( + "SELECT (SELECT count(*) FROM variables " + "WHERE tax_benefit_model_version_id = :version_id), " + "(SELECT count(*) FROM parameters " + "WHERE tax_benefit_model_version_id = :version_id)" + ), + {"version_id": version_id}, + ).one() + assert tuple(counts) == (1, 1) + finally: + with engine.begin() as connection: + connection.execute( + text( + "DELETE FROM tax_benefit_models WHERE name = 'catalog-format-probe'" + ) + ) + command.upgrade(config, "head") + engine.dispose() diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 94d79f577..5a3d20de6 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -247,6 +247,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + SAVED_POLICY_RENAME_REVISION, SAVED_POLICY_RENAME_PARENT_REVISION, "91af4fc8d8c1", "60d6518b6a98", @@ -809,3 +810,52 @@ def test_no_other_v2_revision_runs_hand_written_sql() -> None: } assert revisions_with_sql == {SAVED_POLICY_RENAME_REVISION} + + +def test_catalog_format_2_revision_adds_only_nullable_columns() -> None: + """The revision after the saved-policy rename adds catalog format 2's columns. + + It is generated, adds each column as nullable so versions published earlier + keep them empty, drops them on downgrade, and writes no data. + """ + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + children = [ + revision + for revision in script.walk_revisions() + if revision.down_revision == SAVED_POLICY_RENAME_REVISION + ] + assert len(children) == 1 + revision = Path(children[0].path).read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + added_columns = { + "tax_benefit_model_versions": ("catalog_format", "entities", "basic_inputs"), + "variables": ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + ), + "parameters": ("economy", "household"), + } + column_count = sum(len(columns) for columns in added_columns.values()) + assert upgrade.count("op.add_column(") == column_count + assert upgrade.count("nullable=True") == column_count + assert downgrade.count("op.drop_column(") == column_count + for table_name, columns in added_columns.items(): + for column_name in columns: + assert f'"{column_name}"' in upgrade + assert f'op.drop_column("{table_name}", "{column_name}")' in downgrade + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert "server_default" not in body diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 17b87ec03..cdca61496 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -4,6 +4,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import configure_mappers from sqlalchemy.schema import CreateTable +from sqlmodel.sql.sqltypes import AutoString from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.models import ( @@ -676,3 +677,47 @@ def test_saved_policy_table_is_named_like_the_other_association_tables() -> None ("user_report_associations", "user_policy_id"), ("user_report_associations", "baseline_user_policy_id"), } + + +def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: + """Catalog format 2 stores v3's metadata; versions published before it keep null. + + The model version records its catalog format, entities and basic inputs; each + variable its input flags, module, parameter-named parts and enum labels; each + parameter its economy and household flags. + """ + expected_columns = { + "tax_benefit_model_versions": { + "catalog_format": sa.Integer, + "entities": sa.JSON, + "basic_inputs": sa.JSON, + }, + "variables": { + "is_input_variable": sa.Boolean, + "hidden_input": sa.Boolean, + "module_name": AutoString, + "adds_parameter_path": AutoString, + "subtracts_parameter_path": AutoString, + "enum_values": sa.JSON, + }, + "parameters": { + "economy": sa.Boolean, + "household": sa.Boolean, + }, + } + + for table_name, columns in expected_columns.items(): + table = V2_METADATA.tables[table_name] + for column_name, column_type in columns.items(): + column = table.c[column_name] + assert column.nullable, f"{table_name}.{column_name}" + assert column.server_default is None, f"{table_name}.{column_name}" + assert isinstance(column.type, column_type), f"{table_name}.{column_name}" + + variables = V2_METADATA.tables["variables"] + for column_name in ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + ): + assert variables.c[column_name].type.length == 512 From 14673dc46b23c5bf342ed61a76585ee8a533acec Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:14:39 +0400 Subject: [PATCH 044/118] Extract v3's metadata from PolicyEngine.py's public records as catalog format 2 The extractor now reads, only from PolicyEngine.py's public records: - each model version's entities (key, plural, person flag) and basic inputs; - each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, and its enum values' declared labels; - each parameter's economy and household flags. Bracket schedules already arrive as public parameter nodes, with their stored label or none. Extraction fails with a specific message when the public model publishes no entities or basic inputs, has other than one person entity, names a basic input or a parameter path that is not published, gives enum labels that do not match the possible values or are not JSON scalars, or gives a flag that is not a boolean. normalize_model_metadata normalizes this metadata apart from the certified data release, and an opt-in parity test compares every new field for the US and the UK with what /{country}/metadata builds. The v2 integration check runs it with the installed-catalog test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/v2-integration-check.yml | 4 +- .../data/v2/catalog/extraction.py | 266 ++++++++++++-- policyengine_api/data/v2/catalog/records.py | 54 ++- tests/fixtures/v2_catalog.py | 13 + .../test_v2_catalog_metadata_parity.py | 178 +++++++++ tests/unit/test_alembic_workflows.py | 1 + tests/unit/v2/test_catalog_metadata_fields.py | 337 ++++++++++++++++++ 7 files changed, 828 insertions(+), 25 deletions(-) create mode 100644 tests/integration/test_v2_catalog_metadata_parity.py create mode 100644 tests/unit/v2/test_catalog_metadata_fields.py diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 7959a83c2..9c4cd7493 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -73,8 +73,8 @@ jobs: run: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles - name: Verify the managed v2 roles hold their declared privileges run: uv run python scripts/reconcile_v2_privileges.py verify - - name: Verify the installed PolicyEngine.py catalog interface - run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py + - name: Verify the installed PolicyEngine.py catalog interface and its parity with v1 metadata + run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py tests/integration/test_v2_catalog_metadata_parity.py env: RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication and resource routes diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 410340cea..7e795cd70 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -5,6 +5,7 @@ import math import re from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from enum import Enum from importlib import metadata as importlib_metadata @@ -13,8 +14,11 @@ from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.records import ( + CATALOG_FORMAT, CountryCatalog, DatasetRecord, + EntityRecord, + EnumValueRecord, FallbackSummary, ModelRecord, ModelVersionRecord, @@ -91,6 +95,20 @@ def _optional_text( return value +def _optional_bool(value: object, *, field_name: str) -> bool | None: + if value is not None and not isinstance(value, bool): + msg = f"{field_name} must be a boolean or null" + raise CatalogExtractionError(msg) + return value + + +def _required_bool(value: object, *, field_name: str) -> bool: + if not isinstance(value, bool): + msg = f"{field_name} must be a boolean" + raise CatalogExtractionError(msg) + return value + + def _type_name(value: object) -> str | None: if value is None: return None @@ -277,6 +295,12 @@ def _normalize_variable( else: normalized_possible_values = None + enum_values = _normalize_enum_values( + name, + getattr(source, "enum_values", None), + possible_values=normalized_possible_values, + ) + def optional_names(attribute: str) -> list[str] | None: values = getattr(source, attribute, None) if values is None: @@ -316,9 +340,70 @@ def optional_names(attribute: str) -> list[str] | None: default_value=normalize_json_value(getattr(source, "default_value", None)), adds=optional_names("adds"), subtracts=optional_names("subtracts"), + is_input_variable=_optional_bool( + getattr(source, "is_input_variable", None), + field_name=f"variable {name} is_input_variable", + ), + hidden_input=_optional_bool( + getattr(source, "hidden_input", None), + field_name=f"variable {name} hidden_input", + ), + module_name=_optional_text( + getattr(source, "module_name", None), + field_name=f"variable {name} module_name", + maximum=512, + ), + adds_parameter_path=_optional_text( + getattr(source, "adds_parameter_path", None), + field_name=f"variable {name} adds_parameter_path", + maximum=512, + ), + subtracts_parameter_path=_optional_text( + getattr(source, "subtracts_parameter_path", None), + field_name=f"variable {name} subtracts_parameter_path", + maximum=512, + ), + enum_values=enum_values, ) +def _normalize_enum_values( + name: str, + source_values: object, + *, + possible_values: list[str] | None, +) -> tuple[EnumValueRecord, ...] | None: + """Normalize an enum variable's value labels, which must match its values. + + Each label is kept as declared when it is a JSON scalar. + """ + if source_values is None and possible_values is None: + return None + if ( + source_values is None + or not isinstance(source_values, Sequence) + or isinstance(source_values, (str, bytes)) + ): + msg = f"variable {name} enum values do not match its possible values" + raise CatalogExtractionError(msg) + records: list[EnumValueRecord] = [] + for source_value in source_values: + value = _required_text( + getattr(source_value, "value", None), + field_name=f"variable {name} enum value", + maximum=255, + ) + label = normalize_json_value(getattr(source_value, "label", None)) + if isinstance(label, (dict, list)): + msg = f"variable {name} enum label for {value} must be a JSON scalar" + raise CatalogExtractionError(msg) + records.append(EnumValueRecord(value=value, label=label)) + if [record.value for record in records] != possible_values: + msg = f"variable {name} enum values do not match its possible values" + raise CatalogExtractionError(msg) + return tuple(records) + + def _normalize_parameter( source: object, *, @@ -409,6 +494,14 @@ def _normalize_parameter( maximum=128, ), values=tuple(normalized_values), + economy=_required_bool( + getattr(source, "economy", None), + field_name=f"parameter {name} economy", + ), + household=_required_bool( + getattr(source, "household", None), + field_name=f"parameter {name} household", + ), ) @@ -526,6 +619,150 @@ def _normalize_parameter_nodes( return tuple(normalized_nodes) +def _normalize_entities(country_id: str, source: object) -> tuple[EntityRecord, ...]: + """Normalize the public model's entities, in its order, with one person entity.""" + source_entities = getattr(source, "entities", None) + if not isinstance(source_entities, Sequence) or isinstance( + source_entities, (str, bytes) + ): + msg = ( + f"{country_id} public model publishes no entities; catalog format 2 " + "needs PolicyEngine.py's public entity records" + ) + raise CatalogExtractionError(msg) + entities: list[EntityRecord] = [] + for source_entity in source_entities: + key = _required_text( + getattr(source_entity, "key", None), + field_name=f"{country_id} entity key", + maximum=128, + ) + if any(entity.key == key for entity in entities): + msg = f"{country_id} entity key {key!r} is repeated" + raise CatalogExtractionError(msg) + entities.append( + EntityRecord( + key=key, + plural=_required_text( + getattr(source_entity, "plural", None), + field_name=f"{country_id} entity {key} plural", + maximum=128, + ), + is_person=_required_bool( + getattr(source_entity, "is_person", None), + field_name=f"{country_id} entity {key} is_person", + ), + ) + ) + if sum(entity.is_person for entity in entities) != 1: + msg = f"{country_id} public model must have exactly one person entity" + raise CatalogExtractionError(msg) + return tuple(entities) + + +def _normalize_basic_inputs( + country_id: str, + source: object, + *, + variable_names: frozenset[str], +) -> tuple[str, ...]: + """Normalize the country's default household fields; each is a variable.""" + source_inputs = getattr(source, "basic_inputs", None) + if not isinstance(source_inputs, Sequence) or isinstance( + source_inputs, (str, bytes) + ): + msg = ( + f"{country_id} public model publishes no basic inputs; catalog format 2 " + "needs PolicyEngine.py's public basic inputs" + ) + raise CatalogExtractionError(msg) + basic_inputs = tuple( + _required_text(value, field_name=f"{country_id} basic input", maximum=512) + for value in source_inputs + ) + for name in basic_inputs: + if name not in variable_names: + msg = f"{country_id} basic input {name!r} is not a published variable" + raise CatalogExtractionError(msg) + return basic_inputs + + +def _verify_parameter_paths( + variables: tuple[VariableRecord, ...], + *, + parameter_names: frozenset[str], +) -> None: + """Require each parameter named by ``adds`` or ``subtracts`` to be published.""" + for variable in variables: + for field_name, path in ( + ("adds_parameter_path", variable.adds_parameter_path), + ("subtracts_parameter_path", variable.subtracts_parameter_path), + ): + if path is not None and path not in parameter_names: + msg = ( + f"variable {variable.name} {field_name} {path!r} is not a " + "published parameter" + ) + raise CatalogExtractionError(msg) + + +@dataclass(frozen=True, slots=True) +class ModelMetadata: + """The model metadata of one country's catalog, without regions or datasets.""" + + entities: tuple[EntityRecord, ...] + basic_inputs: tuple[str, ...] + variables: tuple[VariableRecord, ...] + parameter_nodes: tuple[ParameterNodeRecord, ...] + parameters: tuple[ParameterRecord, ...] + + +def normalize_model_metadata( + country_id: str, + source: object, + *, + model_version_id: UUID, +) -> ModelMetadata: + """Normalize a public model's entities, variables and parameter tree. + + Everything comes from PolicyEngine.py's public records. Separate from the + certified data release, so the metadata can be checked on its own. + """ + variables = tuple( + _normalize_variable(variable, model_version_id=model_version_id) + for variable in _public_named_records( + source, + attribute="variables_by_name", + entity_name="variable", + ) + ) + parameters = tuple( + _normalize_parameter(parameter, model_version_id=model_version_id) + for parameter in _public_named_records( + source, + attribute="parameters_by_name", + entity_name="parameter", + ) + ) + _verify_parameter_paths( + variables, + parameter_names=frozenset(parameter.name for parameter in parameters), + ) + return ModelMetadata( + entities=_normalize_entities(country_id, source), + basic_inputs=_normalize_basic_inputs( + country_id, + source, + variable_names=frozenset(variable.name for variable in variables), + ), + variables=variables, + parameter_nodes=_normalize_parameter_nodes( + source, model_version_id=model_version_id + ), + parameters=parameters, + ) + + def _verified_default_dataset(country_id: str, release_manifest: object) -> str: """Return the reviewed default dataset once the manifest names it as its default.""" reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] @@ -581,24 +818,8 @@ def _normalize_country( expected_country_package_version=expected_country_package_version, ) - variables = tuple( - _normalize_variable(variable, model_version_id=model_version_id) - for variable in _public_named_records( - source, - attribute="variables_by_name", - entity_name="variable", - ) - ) - parameter_nodes = _normalize_parameter_nodes( - source, model_version_id=model_version_id - ) - parameters = tuple( - _normalize_parameter(parameter, model_version_id=model_version_id) - for parameter in _public_named_records( - source, - attribute="parameters_by_name", - entity_name="parameter", - ) + metadata = normalize_model_metadata( + country_id, source, model_version_id=model_version_id ) reviewed_default = _verified_default_dataset(country_id, release_manifest) @@ -650,10 +871,13 @@ def _normalize_country( ), current_law_id=CURRENT_LAW_IDS[country_id], metadata_time_periods=METADATA_TIME_PERIODS[country_id], + catalog_format=CATALOG_FORMAT, + entities=metadata.entities, + basic_inputs=metadata.basic_inputs, ), - variables=variables, - parameter_nodes=parameter_nodes, - parameters=parameters, + variables=metadata.variables, + parameter_nodes=metadata.parameter_nodes, + parameters=metadata.parameters, datasets=datasets, regions=tuple(sorted(regions, key=lambda record: record.code)), fallback_summaries=tuple( diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index f05233202..56eaafc95 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -11,6 +11,37 @@ from uuid import UUID +CATALOG_FORMAT = 2 +"""The catalog content format this extractor publishes. + +Format 2 adds the metadata the app's v3 workspace reads, from PolicyEngine.py's +public records: entities and basic inputs; each variable's input and +hidden-input flags, module, the parameter its ``adds`` or ``subtracts`` names, +and enum value labels; each parameter's economy and household flags. Versions +published before it store null for these fields. +""" + + +@dataclass(frozen=True, slots=True) +class EntityRecord: + """A model entity: its key, plural name, and whether it is the person entity.""" + + key: str + plural: str + is_person: bool + + +@dataclass(frozen=True, slots=True) +class EnumValueRecord: + """One value of an enum variable: its stored name and its declared label. + + The label is kept as the country package declares it, usually a string. + """ + + value: str + label: str | int | float | bool | None + + @dataclass(frozen=True, slots=True) class ModelRecord: """One country's tax-benefit model as named in the catalog.""" @@ -35,11 +66,19 @@ class ModelVersionRecord: description: str | None current_law_id: int metadata_time_periods: tuple[int, ...] + catalog_format: int + entities: tuple[EntityRecord, ...] + basic_inputs: tuple[str, ...] @dataclass(frozen=True, slots=True) class VariableRecord: - """One model variable, with its entity, type and the variables it sums.""" + """One model variable, with its entity, type and the variables it sums. + + ``adds_parameter_path`` and ``subtracts_parameter_path`` name the parameter + whose dated value lists the variables summed; ``adds`` and ``subtracts`` + keep the list PolicyEngine.py resolved from it. + """ id: UUID model_version_id: UUID @@ -52,11 +91,20 @@ class VariableRecord: default_value: Any adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: tuple[EnumValueRecord, ...] | None @dataclass(frozen=True, slots=True) class ParameterNodeRecord: - """A non-leaf node of the parameter tree; it groups parameters and has no values.""" + """A non-leaf node of the parameter tree; it groups parameters and has no values. + + Bracket schedules are nodes too, with their stored label or none. + """ id: UUID model_version_id: UUID @@ -88,6 +136,8 @@ class ParameterRecord: data_type: str | None unit: str | None values: tuple[ParameterValueRecord, ...] + economy: bool + household: bool @dataclass(frozen=True, slots=True) diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 61e569f95..73f293237 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -221,6 +221,12 @@ def _model_source( default_value=0.0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name="household.income.employment_income", + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, ) parameter_node = SimpleNamespace( name="gov.example", @@ -233,6 +239,8 @@ def _model_source( description="An example rate", data_type=float, unit="/1", + economy=True, + household=False, parameter_values=[ SimpleNamespace( value=0.1, @@ -265,6 +273,11 @@ def _model_source( region_datasets=region_datasets, data_package=data_package, ) + version_holder.entities = [ + SimpleNamespace(key="person", plural="people", is_person=True), + SimpleNamespace(key="household", plural="households", is_person=False), + ] + version_holder.basic_inputs = ["employment_income"] version_holder.variables = [variable] version_holder.parameter_nodes = [parameter_node] version_holder.parameters = [parameter] diff --git a/tests/integration/test_v2_catalog_metadata_parity.py b/tests/integration/test_v2_catalog_metadata_parity.py new file mode 100644 index 000000000..9b9bd7790 --- /dev/null +++ b/tests/integration/test_v2_catalog_metadata_parity.py @@ -0,0 +1,178 @@ +"""Parity of catalog format 2's fields with the v1 ``/{country}/metadata`` document. + +The catalog reads these fields from PolicyEngine.py's public records; the v1 +document builds them from the country package's tax-benefit system. Each new +field must agree for the US and the UK. The check normalizes the installed +public models' metadata only, so it does not depend on the certified data +release that region publication reads. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest + +from policyengine_api.data.v2.catalog.extraction import normalize_model_metadata + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.extraction import ModelMetadata + +pytestmark = pytest.mark.skipif( + os.environ.get("RUN_V2_CATALOG_COMPATIBILITY") != "1", + reason=("loads the complete public catalog; set RUN_V2_CATALOG_COMPATIBILITY=1"), +) + +COUNTRY_PACKAGES = {"us": "policyengine_us", "uk": "policyengine_uk"} + + +@pytest.fixture(scope="module", params=("us", "uk")) +def country_pair(request: pytest.FixtureRequest) -> tuple[ModelMetadata, Any]: + """Return one country's normalized metadata and its v1 metadata builder.""" + import policyengine + + from policyengine_api.country import PolicyEngineCountry + + country_id = request.param + metadata = normalize_model_metadata( + country_id, + getattr(policyengine, country_id).model, + model_version_id=uuid4(), + ) + return metadata, PolicyEngineCountry(COUNTRY_PACKAGES[country_id], country_id) + + +def _report(mismatches: list[str]) -> str: + shown = "\n".join(mismatches[:20]) + return f"{len(mismatches)} mismatches, first {min(len(mismatches), 20)}:\n{shown}" + + +def test_entities_and_basic_inputs_match_v1( + country_pair: tuple[ModelMetadata, Any], +) -> None: + metadata, v1_country = country_pair + v1_entities = v1_country.metadata["entities"] + + assert [ + (entity.key, entity.plural, entity.is_person) for entity in metadata.entities + ] == [ + (key, entity["plural"], entity["is_person"]) + for key, entity in v1_entities.items() + ] + assert list(metadata.basic_inputs) == (v1_country.metadata["basicInputs"]) + + +def test_variable_flags_modules_enum_labels_and_parts_match_v1( + country_pair: tuple[ModelMetadata, Any], +) -> None: + metadata, v1_country = country_pair + v1_variables = v1_country.metadata["variables"] + mismatches: list[str] = [] + + assert {variable.name for variable in metadata.variables} == set(v1_variables) + for variable in metadata.variables: + v1_variable = v1_variables[variable.name] + enum_values = ( + [ + {"value": item.value, "label": item.label} + for item in variable.enum_values + ] + if variable.enum_values is not None + else None + ) + comparisons = { + "is_input_variable": ( + variable.is_input_variable, + v1_variable["isInputVariable"], + ), + "hidden_input": (variable.hidden_input, v1_variable["hidden_input"]), + "module_name": (variable.module_name, v1_variable["moduleName"]), + "enum_values": (enum_values, v1_variable.get("possibleValues")), + "adds_parameter_path": ( + variable.adds_parameter_path, + v1_variable["adds"] if isinstance(v1_variable["adds"], str) else None, + ), + "subtracts_parameter_path": ( + variable.subtracts_parameter_path, + v1_variable["subtracts"] + if isinstance(v1_variable["subtracts"], str) + else None, + ), + } + mismatches.extend( + f"{variable.name}.{field}: catalog {stored!r}, v1 {expected!r}" + for field, (stored, expected) in comparisons.items() + if stored != expected or type(stored) is not type(expected) + ) + + assert not mismatches, _report(mismatches) + + +def test_parameter_flags_match_v1(country_pair: tuple[ModelMetadata, Any]) -> None: + metadata, v1_country = country_pair + v1_parameters = { + name: entry + for name, entry in v1_country.metadata["parameters"].items() + if entry["type"] == "parameter" + } + catalog_parameters = { + parameter.name: parameter + for parameter in metadata.parameters + if parameter.name.startswith("gov") + } + mismatches: list[str] = [] + + assert set(catalog_parameters) == set(v1_parameters) + for name, parameter in catalog_parameters.items(): + expected = (v1_parameters[name]["economy"], v1_parameters[name]["household"]) + if (parameter.economy, parameter.household) != expected: + mismatches.append( + f"{name}: catalog {(parameter.economy, parameter.household)}, " + f"v1 {expected}" + ) + + assert not mismatches, _report(mismatches) + + +def test_bracket_schedules_are_nodes_with_the_stored_label_v1_shows( + country_pair: tuple[ModelMetadata, Any], +) -> None: + """Every schedule v1 lists is a catalog node; a stored label matches v1's. + + v1 falls back to the name's last part when a schedule has no stored label; + the catalog stores no label, and the routes generate one. + """ + from policyengine_core.parameters import ParameterScale + + metadata, v1_country = country_pair + nodes = {node.name: node for node in metadata.parameter_nodes} + v1_parameters = v1_country.metadata["parameters"] + schedules = [ + parameter + for parameter in v1_country.tax_benefit_system.parameters.get_descendants() + if isinstance(parameter, ParameterScale) and parameter.name.startswith("gov") + ] + mismatches: list[str] = [] + + assert schedules + for schedule in schedules: + node = nodes.get(schedule.name) + stored_label = schedule.metadata.get("label") + if node is None: + mismatches.append(f"{schedule.name}: not a catalog node") + elif node.label != stored_label: + mismatches.append( + f"{schedule.name}: catalog label {node.label!r}, " + f"stored label {stored_label!r}" + ) + elif stored_label is not None and ( + v1_parameters[schedule.name]["label"] != stored_label + ): + mismatches.append( + f"{schedule.name}: catalog label {node.label!r}, " + f"v1 label {v1_parameters[schedule.name]['label']!r}" + ) + + assert not mismatches, _report(mismatches) diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index a0e85c6c5..91992c01a 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -219,6 +219,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "alembic -c alembic-v2.ini upgrade head" in workflow assert "test_alembic_v2_lifecycle.sh" not in workflow assert "test_v2_catalog_installed.py" in workflow + assert "test_v2_catalog_metadata_parity.py" in workflow assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py new file mode 100644 index 000000000..cdab9464f --- /dev/null +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -0,0 +1,337 @@ +"""Catalog format 2: the metadata v3 reads, taken from .py's public records.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import pytest + +from policyengine_api.data.v2.catalog.extraction import ( + CatalogExtractionError, + extract_catalog, +) +from policyengine_api.data.v2.catalog.records import ( + CATALOG_FORMAT, + EntityRecord, + EnumValueRecord, + NormalizedCatalog, +) +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + bundle, + installed_version, + normalized_catalog, + region_references, + source_models, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + +def _extract(models: dict[str, SimpleNamespace]) -> NormalizedCatalog: + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + region_references=region_references(), + ) + + +def _add_variable(model: SimpleNamespace, **fields: object) -> None: + variable = SimpleNamespace( + label=None, + entity="person", + description=None, + data_type=float, + possible_values=None, + default_value=0.0, + adds=None, + subtracts=None, + is_input_variable=False, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + ) + for name, value in fields.items(): + setattr(variable, name, value) + model.variables.append(variable) + model.variables_by_name[variable.name] = variable + + +def _add_parameter(model: SimpleNamespace, name: str, *, value: object) -> None: + parameter = SimpleNamespace( + name=name, + label=None, + description=None, + data_type=type(value), + unit=None, + economy=True, + household=True, + parameter_values=[ + SimpleNamespace( + value=value, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ], + ) + model.parameters.append(parameter) + model.parameters_by_name[name] = parameter + + +def _add_parameter_node(model: SimpleNamespace, name: str, label: str | None) -> None: + node = SimpleNamespace(name=name, label=label, description=None) + model.parameter_nodes.append(node) + model.parameter_nodes_by_name[name] = node + + +def test_catalog_format_2_publishes_entities_and_basic_inputs() -> None: + catalog = normalized_catalog() + + for country_id in ("us", "uk"): + model_version = catalog.country(country_id).model_version + assert model_version.catalog_format == CATALOG_FORMAT == 2 + assert model_version.entities == ( + EntityRecord(key="person", plural="people", is_person=True), + EntityRecord(key="household", plural="households", is_person=False), + ) + assert model_version.basic_inputs == ("employment_income",) + + +def test_variables_carry_input_flags_module_and_unset_parts() -> None: + variable = normalized_catalog().country("us").variables[0] + + assert variable.name == "employment_income" + assert variable.is_input_variable is True + assert variable.hidden_input is False + assert variable.module_name == "household.income.employment_income" + assert variable.adds_parameter_path is None + assert variable.subtracts_parameter_path is None + assert variable.enum_values is None + + +def test_enum_values_keep_their_declared_labels_in_order() -> None: + models = source_models() + _add_variable( + models["us"], + name="filing_status", + data_type="Enum", + possible_values=["SINGLE", "JOINT", "EXEMPT"], + default_value="SINGLE", + is_input_variable=True, + enum_values=[ + SimpleNamespace(value="SINGLE", label="Single"), + SimpleNamespace(value="JOINT", label=2), + SimpleNamespace(value="EXEMPT", label=None), + ], + ) + + variables = { + variable.name: variable for variable in _extract(models).country("us").variables + } + + assert variables["filing_status"].enum_values == ( + EnumValueRecord(value="SINGLE", label="Single"), + EnumValueRecord(value="JOINT", label=2), + EnumValueRecord(value="EXEMPT", label=None), + ) + assert isinstance(variables["filing_status"].enum_values[1].label, int) + + +def test_parameter_named_parts_record_the_parameter_path_beside_the_list() -> None: + models = source_models() + _add_parameter( + models["us"], + "gov.example.countable_income", + value=["employment_income"], + ) + _add_parameter(models["us"], "gov.example.deductions", value=["employment_income"]) + _add_variable( + models["us"], + name="countable_income", + adds=["employment_income"], + subtracts=["employment_income"], + adds_parameter_path="gov.example.countable_income", + subtracts_parameter_path="gov.example.deductions", + ) + + variables = { + variable.name: variable for variable in _extract(models).country("us").variables + } + + countable_income = variables["countable_income"] + assert countable_income.adds_parameter_path == "gov.example.countable_income" + assert countable_income.subtracts_parameter_path == "gov.example.deductions" + assert countable_income.adds == ["employment_income"] + assert countable_income.subtracts == ["employment_income"] + + +def test_parameters_carry_their_economy_and_household_flags() -> None: + parameter = normalized_catalog().country("us").parameters[0] + + assert parameter.name == "gov.example.rate" + assert (parameter.economy, parameter.household) == (True, False) + + +def test_bracket_schedules_are_stored_as_nodes_with_their_stored_label() -> None: + models = source_models() + _add_parameter_node(models["uk"], "gov.example.rates", "Example rates") + _add_parameter_node(models["uk"], "gov.example.rates[0]", None) + _add_parameter_node(models["uk"], "gov.example.thresholds", None) + _add_parameter_node(models["uk"], "gov.example.thresholds[0]", None) + _add_parameter(models["uk"], "gov.example.rates[0].rate", value=0.2) + _add_parameter(models["uk"], "gov.example.thresholds[0].threshold", value=0) + + nodes = {node.name: node for node in _extract(models).country("uk").parameter_nodes} + + assert nodes["gov.example.rates"].label == "Example rates" + assert nodes["gov.example.thresholds"].label is None + assert "gov.example.rates[0]" in nodes + + +def _remove_entities(models: dict[str, SimpleNamespace]) -> None: + del models["us"].entities + + +def _make_entity_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities[0].is_person = "yes" + + +def _remove_person_entity(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities = models["us"].entities[1:] + + +def _repeat_entity_key(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities.append( + SimpleNamespace(key="household", plural="homes", is_person=False) + ) + + +def _remove_basic_inputs(models: dict[str, SimpleNamespace]) -> None: + del models["us"].basic_inputs + + +def _name_unknown_basic_input(models: dict[str, SimpleNamespace]) -> None: + models["us"].basic_inputs = ["employment_income", "unpublished_input"] + + +def _make_input_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].is_input_variable = "true" + + +def _make_hidden_flag_number(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].hidden_input = 0 + + +def _name_unknown_adds_parameter(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].adds_parameter_path = "gov.example.missing" + + +def _name_unknown_subtracts_parameter(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].subtracts_parameter_path = "gov.example.missing" + + +def _mismatch_enum_values(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE", "JOINT"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label="Single")] + + +def _give_enum_labels_without_values(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].enum_values = [ + SimpleNamespace(value="SINGLE", label="Single") + ] + + +def _make_enum_label_an_object(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label={"text": "Single"})] + + +def _make_economy_flag_missing(models: dict[str, SimpleNamespace]) -> None: + del models["us"].parameters[0].economy + + +def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].parameters[0].household = "false" + + +@pytest.mark.parametrize( + ("break_models", "message"), + [ + ( + _remove_entities, + "^us public model publishes no entities; catalog format 2 needs " + "PolicyEngine.py's public entity records$", + ), + (_make_entity_flag_text, "^us entity person is_person must be a boolean$"), + ( + _remove_person_entity, + "^us public model must have exactly one person entity$", + ), + (_repeat_entity_key, "^us entity key 'household' is repeated$"), + ( + _remove_basic_inputs, + "^us public model publishes no basic inputs; catalog format 2 needs " + "PolicyEngine.py's public basic inputs$", + ), + ( + _name_unknown_basic_input, + "^us basic input 'unpublished_input' is not a published variable$", + ), + ( + _make_input_flag_text, + "^variable employment_income is_input_variable must be a boolean or null$", + ), + ( + _make_hidden_flag_number, + "^variable employment_income hidden_input must be a boolean or null$", + ), + ( + _name_unknown_adds_parameter, + "^variable employment_income adds_parameter_path 'gov.example.missing' " + "is not a published parameter$", + ), + ( + _name_unknown_subtracts_parameter, + "^variable employment_income subtracts_parameter_path " + "'gov.example.missing' is not a published parameter$", + ), + ( + _mismatch_enum_values, + "^variable employment_income enum values do not match its possible values$", + ), + ( + _give_enum_labels_without_values, + "^variable employment_income enum values do not match its possible values$", + ), + ( + _make_enum_label_an_object, + "^variable employment_income enum label for SINGLE must be a JSON scalar$", + ), + ( + _make_economy_flag_missing, + "^parameter gov.example.rate economy must be a boolean$", + ), + ( + _make_household_flag_text, + "^parameter gov.example.rate household must be a boolean$", + ), + ], +) +def test_inconsistent_format_2_metadata_fails_with_a_specific_message( + break_models: Callable[[dict[str, SimpleNamespace]], None], + message: str, +) -> None: + models = source_models() + break_models(models) + + with pytest.raises(CatalogExtractionError, match=message): + _extract(models) From 07b7b0cfe3a060c25768f094c6532287057a6d27 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 19:26:22 +0400 Subject: [PATCH 045/118] Store catalog format 2 fields when a catalog version is published Publication now stages and inserts each model version's catalog format, entities and basic inputs, each variable's input and hidden-input flags, module, parameter paths and enum values (as ordered value and label objects), and each parameter's economy and household flags. Publication never rewrites a stored version. When the same PolicyEngine.py version is published again, a version stored before catalog format 2 (a null catalog_format) is compared without format 2's fields, so it keeps them empty, and a version stored with format 2 must match on them too. The Stage 9 metadata document describes the format, and the changelog fragment covers the whole change. Co-Authored-By: Claude Opus 5.5 --- ...an-1f-115-catalog-metadata-fields.added.md | 1 + docs/migration/stage-9-v2-metadata.md | 26 ++++ .../v2/catalog/publication_reconciliation.py | 102 +++++++++++++- .../data/v2/catalog/publication_staging.py | 41 ++++++ .../test_v2_catalog_publication.py | 128 ++++++++++++++++++ tests/unit/v2/test_catalog_publication.py | 50 +++++-- tests/unit/v2/test_catalog_staging_rows.py | 40 ++++++ 7 files changed, 370 insertions(+), 18 deletions(-) create mode 100644 changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md diff --git a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md new file mode 100644 index 000000000..7394dc074 --- /dev/null +++ b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md @@ -0,0 +1 @@ +Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, and its enum values' labels; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3825ad81c..3eb0a1731 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -99,6 +99,32 @@ metadata time-period options are stored on the model-version snapshot. PolicyEngine Core and country-package versions are derived diagnostic evidence and are not separately persisted. +Catalog format 2 adds the metadata the app's v3 workspace reads, taken only +from PolicyEngine.py's public records, which carry it from the release that +publishes entities, basic inputs and these flags: + +| Record | Fields | +| --- | --- | +| Model version | `catalog_format` (2), `entities` (`key`, `plural`, `is_person`, in the package's order), `basic_inputs` (the household builder's default fields) | +| Variable | `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path` and `subtracts_parameter_path` (the parameter whose dated value lists the variables summed; `adds` and `subtracts` keep the list PolicyEngine.py resolved), `enum_values` (`value` and declared `label`, in order) | +| Parameter | `economy`, `household` | + +Bracket schedules are parameter nodes with their stored label, or none when +the package stores none. Extraction stops if the public model publishes no +entities or basic inputs, has other than one person entity, names a basic +input or a parameter path that is not published, gives enum labels that do +not match the variable's possible values or are not JSON scalars, or gives a +flag that is not a boolean. An opt-in test +(`tests/integration/test_v2_catalog_metadata_parity.py`, run by the v2 +integration check) compares every format 2 field for the US and the UK with +what `/{country}/metadata` builds. + +Publication never rewrites a stored version. A version published before +format 2 has a null `catalog_format` and null format 2 fields, and later +publications of the same PolicyEngine.py version compare it without those +fields. The first PolicyEngine.py version published after format 2 deploys +carries them. + Parameter extraction requires the PolicyEngine.py public history contract from 5.0.4 and later: values are ordered from oldest to newest, bounded `end_date` values are inclusive and fall one day before the following `start_date`, and diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 8a0c9c0aa..9ddcad20d 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -76,10 +76,45 @@ def _versioned_table_from(table: sa.Table) -> sa.Join: ) +def stored_catalog_format( + connection: Connection, country: CountryCatalog +) -> int | None: + """Return the catalog format this country's stored version was published with. + + None means it was published before catalog format 2, so its format 2 + fields are empty. + """ + statement = ( + sa.select(MODEL_VERSIONS.c.catalog_format) + .select_from( + MODEL_VERSIONS.join( + MODELS, + MODELS.c.id == MODEL_VERSIONS.c.model_id, + ) + ) + .where( + MODELS.c.name == country.model.name, + MODEL_VERSIONS.c.version == country.model_version.version, + ) + ) + return connection.execute(statement).scalar_one_or_none() + + def _comparison_pairs( country: CountryCatalog, + *, + with_format_2: bool, ) -> tuple[tuple[str, Select, Select], ...]: - """Return each compared table's name with its staged and stored rows.""" + """Return each compared table's name with its staged and stored rows. + + Without ``with_format_2``, catalog format 2's fields are left out: a + version published before format 2 keeps them empty, and publication never + rewrites a stored version. + """ + + def format_2(*columns: sa.ColumnElement) -> tuple[sa.ColumnElement, ...]: + return columns if with_format_2 else () + staged_country = STAGE_CATALOG_MODELS.c.country_id == country.country_id selected_version = sa.and_( MODELS.c.name == country.model.name, @@ -90,12 +125,22 @@ def _comparison_pairs( STAGE_CATALOG_MODELS.c.version_description, STAGE_CATALOG_MODELS.c.current_law_id, STAGE_CATALOG_MODELS.c.metadata_time_periods, + *format_2( + STAGE_CATALOG_MODELS.c.catalog_format, + STAGE_CATALOG_MODELS.c.entities, + STAGE_CATALOG_MODELS.c.basic_inputs, + ), ).where(staged_country) actual_model_version = ( sa.select( MODEL_VERSIONS.c.description, MODEL_VERSIONS.c.current_law_id, sa.cast(MODEL_VERSIONS.c.metadata_time_periods, JSONB), + *format_2( + MODEL_VERSIONS.c.catalog_format, + sa.cast(MODEL_VERSIONS.c.entities, JSONB), + sa.cast(MODEL_VERSIONS.c.basic_inputs, JSONB), + ), ) .select_from( MODEL_VERSIONS.join( @@ -116,6 +161,14 @@ def _comparison_pairs( STAGE_CATALOG_VARIABLES.c.default_value, STAGE_CATALOG_VARIABLES.c.adds, STAGE_CATALOG_VARIABLES.c.subtracts, + *format_2( + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.adds_parameter_path, + STAGE_CATALOG_VARIABLES.c.subtracts_parameter_path, + STAGE_CATALOG_VARIABLES.c.enum_values, + ), ).where(STAGE_CATALOG_VARIABLES.c.country_id == country.country_id) actual_variables = ( sa.select( @@ -128,6 +181,14 @@ def _comparison_pairs( sa.cast(VARIABLES.c.default_value, JSONB), sa.cast(VARIABLES.c.adds, JSONB), sa.cast(VARIABLES.c.subtracts, JSONB), + *format_2( + VARIABLES.c.is_input_variable, + VARIABLES.c.hidden_input, + VARIABLES.c.module_name, + VARIABLES.c.adds_parameter_path, + VARIABLES.c.subtracts_parameter_path, + sa.cast(VARIABLES.c.enum_values, JSONB), + ), ) .select_from(_versioned_table_from(VARIABLES)) .where(selected_version) @@ -154,6 +215,10 @@ def _comparison_pairs( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + *format_2( + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, + ), ).where(STAGE_CATALOG_PARAMETERS.c.country_id == country.country_id) actual_parameters = ( sa.select( @@ -162,6 +227,7 @@ def _comparison_pairs( PARAMETERS.c.description, PARAMETERS.c.data_type, PARAMETERS.c.unit, + *format_2(PARAMETERS.c.economy, PARAMETERS.c.household), ) .select_from(_versioned_table_from(PARAMETERS)) .where(selected_version) @@ -302,9 +368,11 @@ def assert_country_matches( row both fail; the error names the first table that differs, never its rows. Without ``include_regions``, region rows are not compared: they come from the Public API's reference files, which may change while the - PolicyEngine.py version stays the same. + PolicyEngine.py version stays the same. A version published before catalog + format 2 is compared without format 2's fields. """ - pairs = _comparison_pairs(country) + stored_format = stored_catalog_format(connection, country) + pairs = _comparison_pairs(country, with_format_2=stored_format is not None) if include_regions: pairs = (*pairs, ("regions", *_region_comparison_pair(country))) for table_name, staged, actual in pairs: @@ -367,6 +435,9 @@ def count_region_differences( "description", "current_law_id", "metadata_time_periods", + "catalog_format", + "entities", + "basic_inputs", ], sa.select( STAGE_CATALOG_MODELS.c.version_id, @@ -379,6 +450,12 @@ def count_region_differences( STAGE_CATALOG_MODELS.c.metadata_time_periods, MODEL_VERSIONS.c.metadata_time_periods.type, ), + STAGE_CATALOG_MODELS.c.catalog_format, + sa.cast(STAGE_CATALOG_MODELS.c.entities, MODEL_VERSIONS.c.entities.type), + sa.cast( + STAGE_CATALOG_MODELS.c.basic_inputs, + MODEL_VERSIONS.c.basic_inputs.type, + ), ) .select_from( STAGE_CATALOG_MODELS.join( @@ -442,6 +519,12 @@ def count_region_differences( "default_value", "adds", "subtracts", + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", "tax_benefit_model_version_id", ], sa.select( @@ -465,6 +548,15 @@ def count_region_differences( STAGE_CATALOG_VARIABLES.c.subtracts, VARIABLES.c.subtracts.type, ), + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.adds_parameter_path, + STAGE_CATALOG_VARIABLES.c.subtracts_parameter_path, + sa.cast( + STAGE_CATALOG_VARIABLES.c.enum_values, + VARIABLES.c.enum_values.type, + ), MODEL_VERSIONS.c.id, ) .select_from( @@ -528,6 +620,8 @@ def count_region_differences( "description", "data_type", "unit", + "economy", + "household", "tax_benefit_model_version_id", ], sa.select( @@ -538,6 +632,8 @@ def count_region_differences( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, MODEL_VERSIONS.c.id, ) .select_from( diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index 76820b27b..e0dfa3c31 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -16,6 +16,7 @@ from policyengine_api.data.v2.catalog.records import ( CountryCatalog, NormalizedCatalog, + VariableRecord, iter_batches, ) @@ -39,6 +40,9 @@ sa.Column("version_description", sa.Text), sa.Column("current_law_id", sa.Integer, nullable=False), sa.Column("metadata_time_periods", JSONB, nullable=False), + sa.Column("catalog_format", sa.Integer, nullable=False), + sa.Column("entities", JSONB, nullable=False), + sa.Column("basic_inputs", JSONB, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -57,6 +61,12 @@ sa.Column("default_value", JSONB, nullable=False), sa.Column("adds", JSONB), sa.Column("subtracts", JSONB), + sa.Column("is_input_variable", sa.Boolean), + sa.Column("hidden_input", sa.Boolean), + sa.Column("module_name", sa.Text), + sa.Column("adds_parameter_path", sa.Text), + sa.Column("subtracts_parameter_path", sa.Text), + sa.Column("enum_values", JSONB), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -83,6 +93,8 @@ sa.Column("description", sa.Text), sa.Column("data_type", sa.Text), sa.Column("unit", sa.Text), + sa.Column("economy", sa.Boolean, nullable=False), + sa.Column("household", sa.Boolean, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -163,6 +175,27 @@ def _model_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: country.model_version.description, country.model_version.current_law_id, Jsonb(country.model_version.metadata_time_periods), + country.model_version.catalog_format, + Jsonb( + [ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ] + ), + Jsonb(list(country.model_version.basic_inputs)), + ) + + +def _enum_values_json(record: VariableRecord) -> Jsonb | None: + """Return an enum variable's values as ordered value and label objects.""" + if record.enum_values is None: + return None + return Jsonb( + [{"value": item.value, "label": item.label} for item in record.enum_values] ) @@ -182,6 +215,12 @@ def _variable_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: Jsonb(record.default_value), _optional_json(record.adds), _optional_json(record.subtracts), + record.is_input_variable, + record.hidden_input, + record.module_name, + record.adds_parameter_path, + record.subtracts_parameter_path, + _enum_values_json(record), ) @@ -213,6 +252,8 @@ def _parameter_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: record.description, record.data_type, record.unit, + record.economy, + record.household, ) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index edef7b93e..0e4033379 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -550,3 +550,131 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "reports": 0, "report_runs": 0, } + + +def _format_2_values(engine: Engine) -> dict[str, object]: + """Return the US version's stored catalog format 2 fields.""" + with engine.connect() as connection: + version = connection.execute( + text( + "SELECT version.catalog_format, version.entities, version.basic_inputs " + "FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us'" + ) + ).one() + variable = connection.execute( + text( + "SELECT is_input_variable, hidden_input, module_name, " + "adds_parameter_path, subtracts_parameter_path, enum_values " + "FROM variables WHERE name = 'employment_income' " + "AND tax_benefit_model_version_id = (" + "SELECT version.id FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us')" + ) + ).one() + parameter = connection.execute( + text( + "SELECT economy, household FROM parameters " + "WHERE name = 'gov.example.rate' " + "AND tax_benefit_model_version_id = (" + "SELECT version.id FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us')" + ) + ).one() + return { + "version": tuple(version), + "variable": tuple(variable), + "parameter": tuple(parameter), + } + + +def _clear_format_2_fields(engine: Engine) -> None: + """Make every stored version look as if published before catalog format 2.""" + with engine.begin() as connection: + connection.execute( + text( + "UPDATE tax_benefit_model_versions SET catalog_format = NULL, " + "entities = NULL, basic_inputs = NULL" + ) + ) + connection.execute( + text( + "UPDATE variables SET is_input_variable = NULL, hidden_input = NULL, " + "module_name = NULL, adds_parameter_path = NULL, " + "subtracts_parameter_path = NULL, enum_values = NULL" + ) + ) + connection.execute( + text("UPDATE parameters SET economy = NULL, household = NULL") + ) + + +def test_publication_stores_catalog_format_2_fields( + publication_engine: Engine, +) -> None: + publish_catalog(publication_engine, normalized_catalog()) + + assert _format_2_values(publication_engine) == { + "version": ( + 2, + [ + {"key": "person", "plural": "people", "is_person": True}, + {"key": "household", "plural": "households", "is_person": False}, + ], + ["employment_income"], + ), + "variable": ( + True, + False, + "household.income.employment_income", + None, + None, + None, + ), + "parameter": (True, False), + } + + +def test_version_stored_before_format_2_keeps_its_fields_empty_on_republish( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + _clear_format_2_fields(publication_engine) + before_ids = _identifiers(publication_engine) + + result = publish_catalog(publication_engine, catalog) + + assert result.as_dict()["outcome"] == "ok" + assert _identifiers(publication_engine) == before_ids + assert _format_2_values(publication_engine) == { + "version": (None, None, None), + "variable": (None, None, None, None, None, None), + "parameter": (None, None), + } + + +def test_version_stored_with_format_2_fails_when_a_format_2_field_changes( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_counts = _counts(publication_engine) + us = catalog.country("us") + changed_parameter = replace(us.parameters[0], household=True) + changed = replace( + catalog, + countries=( + replace(us, parameters=(changed_parameter,)), + catalog.country("uk"), + ), + ) + + with pytest.raises(CatalogPublicationError, match="differs"): + publish_catalog(publication_engine, changed) + + assert _counts(publication_engine) == before_counts + assert _format_2_values(publication_engine)["parameter"] == (True, False) diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index 2341003dc..afdc7f5a2 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -214,22 +214,42 @@ def test_publication_uses_mapped_tables_and_sqlalchemy_statements() -> None: assert all("VALUES" not in statement for statement in compiled_inserts) catalog = normalized_catalog() - comparisons = publication_reconciliation._comparison_pairs(catalog.country("us")) - assert [table_name for table_name, _staged, _stored in comparisons] == [ - "model version", - "variables", - "parameter nodes", - "parameters", - "parameter values", - "datasets", - ] - statements = [ - statement for _table_name, *pair in comparisons for statement in pair - ] + list(publication_reconciliation._region_comparison_pair(catalog.country("us"))) - assert all( - isinstance(statement, sa.sql.Select) and not isinstance(statement, TextClause) - for statement in statements + format_1_comparisons, format_2_comparisons = ( + publication_reconciliation._comparison_pairs( + catalog.country("us"), with_format_2=with_format_2 + ) + for with_format_2 in (False, True) ) + for comparisons in (format_1_comparisons, format_2_comparisons): + assert [table_name for table_name, _staged, _stored in comparisons] == [ + "model version", + "variables", + "parameter nodes", + "parameters", + "parameter values", + "datasets", + ] + statements = [ + statement for _table_name, *pair in comparisons for statement in pair + ] + list( + publication_reconciliation._region_comparison_pair(catalog.country("us")) + ) + assert all( + isinstance(statement, sa.sql.Select) + and not isinstance(statement, TextClause) + for statement in statements + ) + assert all( + len(staged.selected_columns) == len(stored.selected_columns) + for _table_name, staged, stored in comparisons + ) + # Format 2 compares model versions, variables and parameters on more fields. + assert [ + len(format_2[1].selected_columns) - len(format_1[1].selected_columns) + for format_1, format_2 in zip( + format_1_comparisons, format_2_comparisons, strict=True + ) + ] == [3, 6, 0, 2, 0, 0] class Result: def scalar_one(self): diff --git a/tests/unit/v2/test_catalog_staging_rows.py b/tests/unit/v2/test_catalog_staging_rows.py index f43f2772f..549743810 100644 --- a/tests/unit/v2/test_catalog_staging_rows.py +++ b/tests/unit/v2/test_catalog_staging_rows.py @@ -3,13 +3,16 @@ from __future__ import annotations import inspect +from dataclasses import replace from psycopg.types.json import Jsonb from policyengine_api.data.v2.catalog.publication_staging import ( + STAGE_CATALOG_VARIABLES, STAGING_TABLES, _catalog_rows, ) +from policyengine_api.data.v2.catalog.records import EnumValueRecord from tests.fixtures.v2_catalog import normalized_catalog @@ -49,6 +52,15 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: country.model_version.description, country.model_version.current_law_id, ("jsonb", country.model_version.metadata_time_periods), + 2, + ( + "jsonb", + [ + {"key": "person", "plural": "people", "is_person": True}, + {"key": "household", "plural": "households", "is_person": False}, + ], + ), + ("jsonb", ["employment_income"]), ) ] assert rows["stage_catalog_variables"] == [ @@ -66,6 +78,12 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: ("jsonb", record.default_value), None if record.adds is None else ("jsonb", record.adds), None if record.subtracts is None else ("jsonb", record.subtracts), + record.is_input_variable, + record.hidden_input, + record.module_name, + record.adds_parameter_path, + record.subtracts_parameter_path, + None, ) for record in country.variables ] @@ -82,6 +100,8 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: record.description, record.data_type, record.unit, + record.economy, + record.household, ) for record in country.parameters ] @@ -121,3 +141,23 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: ] assert len(rows["stage_catalog_parameter_values"]) == 2 assert len(rows["stage_catalog_regions"]) == 3 + + +def test_enum_values_are_staged_as_value_and_label_objects_in_order() -> None: + country = normalized_catalog().country("us") + variable = replace( + country.variables[0], + possible_values=["SINGLE", "JOINT"], + enum_values=( + EnumValueRecord(value="SINGLE", label="Single"), + EnumValueRecord(value="JOINT", label=2), + ), + ) + country = replace(country, variables=(variable,)) + + (row,) = (_plain(row) for row in _catalog_rows(country)[STAGE_CATALOG_VARIABLES]) + + assert row[-1] == ( + "jsonb", + [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], + ) From b8a75d12c139961f2ebcfa0153b39f19b98fdaf8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:31:18 +0400 Subject: [PATCH 046/118] Name the missing catalog format 2 fields when PolicyEngine.py lacks them Extraction now checks first that the installed PolicyEngine.py publishes catalog format 2's fields on the model version, every variable and every parameter, and fails naming each missing one. On 6.7.5 it used to stop at the first enum variable with "enum values do not match its possible values", which hid the cause. An enum label that is not JSON-compatible, such as NaN, now fails naming its variable and value. Entities or basic inputs that are present but not a sequence fail with "must be a sequence", and the Stage 9 document says that an empty basic-input list is stored as empty. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 12 +-- .../data/v2/catalog/extraction.py | 88 +++++++++++++++---- tests/unit/v2/test_catalog_metadata_fields.py | 88 ++++++++++++++++++- 3 files changed, 160 insertions(+), 28 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3eb0a1731..0d716beee 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -110,11 +110,13 @@ publishes entities, basic inputs and these flags: | Parameter | `economy`, `household` | Bracket schedules are parameter nodes with their stored label, or none when -the package stores none. Extraction stops if the public model publishes no -entities or basic inputs, has other than one person entity, names a basic -input or a parameter path that is not published, gives enum labels that do -not match the variable's possible values or are not JSON scalars, or gives a -flag that is not a boolean. An opt-in test +the package stores none. An empty basic-input list is stored as empty. +Extraction stops before reading any record if the installed PolicyEngine.py +does not publish these fields, as 6.7.5 does not, and names each missing +field. It also stops if the model has other than one person entity, names a +basic input or a parameter path that is not published, gives enum labels that +do not match the variable's possible values or are not JSON scalars, or gives +a flag that is not a boolean. An opt-in test (`tests/integration/test_v2_catalog_metadata_parity.py`, run by the v2 integration check) compares every format 2 field for the US and the UK with what `/{country}/metadata` builds. diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 7e795cd70..34ffcc803 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -393,7 +393,14 @@ def _normalize_enum_values( field_name=f"variable {name} enum value", maximum=255, ) - label = normalize_json_value(getattr(source_value, "label", None)) + try: + label = normalize_json_value(getattr(source_value, "label", None)) + except CatalogExtractionError as error: + msg = ( + f"variable {name} enum label for {value} is not JSON-compatible: " + f"{error}" + ) + raise CatalogExtractionError(msg) from error if isinstance(label, (dict, list)): msg = f"variable {name} enum label for {value} must be a JSON scalar" raise CatalogExtractionError(msg) @@ -619,17 +626,63 @@ def _normalize_parameter_nodes( return tuple(normalized_nodes) -def _normalize_entities(country_id: str, source: object) -> tuple[EntityRecord, ...]: - """Normalize the public model's entities, in its order, with one person entity.""" - source_entities = getattr(source, "entities", None) - if not isinstance(source_entities, Sequence) or isinstance( - source_entities, (str, bytes) - ): +# The public fields catalog format 2 reads, by the record that carries them. +FORMAT_2_FIELDS: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ("model version", "", ("entities", "basic_inputs")), + ( + "variable", + "variables_by_name", + ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + ), + ), + ("parameter", "parameters_by_name", ("economy", "household")), +) + + +def _require_format_2_fields(country_id: str, source: object) -> None: + """Fail first, naming each field, when PolicyEngine.py does not publish them. + + Older releases, such as 6.7.5, publish none of catalog format 2's fields. + """ + missing: list[str] = [] + for record_name, mapping_attribute, field_names in FORMAT_2_FIELDS: + if mapping_attribute: + records = getattr(source, mapping_attribute, None) + holders = list(records.values()) if isinstance(records, Mapping) else [] + else: + holders = [source] + absent = [ + field_name + for field_name in field_names + if any(not hasattr(holder, field_name) for holder in holders) + ] + if absent: + missing.append(f"{record_name} {', '.join(absent)}") + if missing: msg = ( - f"{country_id} public model publishes no entities; catalog format 2 " - "needs PolicyEngine.py's public entity records" + "the installed PolicyEngine.py does not publish catalog format 2's " + f"fields for {country_id}: {'; '.join(missing)}" ) raise CatalogExtractionError(msg) + + +def _source_sequence(country_id: str, source: object, attribute: str) -> Sequence: + values = getattr(source, attribute, None) + if not isinstance(values, Sequence) or isinstance(values, (str, bytes)): + msg = f"{country_id} public model {attribute} must be a sequence" + raise CatalogExtractionError(msg) + return values + + +def _normalize_entities(country_id: str, source: object) -> tuple[EntityRecord, ...]: + """Normalize the public model's entities, in its order, with one person entity.""" + source_entities = _source_sequence(country_id, source, "entities") entities: list[EntityRecord] = [] for source_entity in source_entities: key = _required_text( @@ -666,16 +719,12 @@ def _normalize_basic_inputs( *, variable_names: frozenset[str], ) -> tuple[str, ...]: - """Normalize the country's default household fields; each is a variable.""" - source_inputs = getattr(source, "basic_inputs", None) - if not isinstance(source_inputs, Sequence) or isinstance( - source_inputs, (str, bytes) - ): - msg = ( - f"{country_id} public model publishes no basic inputs; catalog format 2 " - "needs PolicyEngine.py's public basic inputs" - ) - raise CatalogExtractionError(msg) + """Normalize the country's default household fields; each is a variable. + + An empty list, which PolicyEngine.py gives a package that declares none, is + stored as empty. + """ + source_inputs = _source_sequence(country_id, source, "basic_inputs") basic_inputs = tuple( _required_text(value, field_name=f"{country_id} basic input", maximum=512) for value in source_inputs @@ -728,6 +777,7 @@ def normalize_model_metadata( Everything comes from PolicyEngine.py's public records. Separate from the certified data release, so the metadata can be checked on its own. """ + _require_format_2_fields(country_id, source) variables = tuple( _normalize_variable(variable, model_version_id=model_version_id) for variable in _public_named_records( diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py index cdab9464f..ab6f6414b 100644 --- a/tests/unit/v2/test_catalog_metadata_fields.py +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -217,6 +217,24 @@ def _remove_basic_inputs(models: dict[str, SimpleNamespace]) -> None: del models["us"].basic_inputs +def _make_entities_null(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities = None + + +def _make_basic_inputs_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].basic_inputs = "employment_income" + + +def _remove_variable_enum_values(models: dict[str, SimpleNamespace]) -> None: + del models["us"].variables[0].enum_values + + +def _make_enum_label_nan(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label=float("nan"))] + + def _name_unknown_basic_input(models: dict[str, SimpleNamespace]) -> None: models["us"].basic_inputs = ["employment_income", "unpublished_input"] @@ -259,6 +277,10 @@ def _make_economy_flag_missing(models: dict[str, SimpleNamespace]) -> None: del models["us"].parameters[0].economy +def _make_economy_flag_null(models: dict[str, SimpleNamespace]) -> None: + models["us"].parameters[0].economy = None + + def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: models["us"].parameters[0].household = "false" @@ -268,9 +290,10 @@ def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: [ ( _remove_entities, - "^us public model publishes no entities; catalog format 2 needs " - "PolicyEngine.py's public entity records$", + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: model version entities$", ), + (_make_entities_null, "^us public model entities must be a sequence$"), (_make_entity_flag_text, "^us entity person is_person must be a boolean$"), ( _remove_person_entity, @@ -279,8 +302,22 @@ def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: (_repeat_entity_key, "^us entity key 'household' is repeated$"), ( _remove_basic_inputs, - "^us public model publishes no basic inputs; catalog format 2 needs " - "PolicyEngine.py's public basic inputs$", + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: model version basic_inputs$", + ), + ( + _make_basic_inputs_text, + "^us public model basic_inputs must be a sequence$", + ), + ( + _remove_variable_enum_values, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: variable enum_values$", + ), + ( + _make_enum_label_nan, + "^variable employment_income enum label for SINGLE is not " + "JSON-compatible: NaN is not JSON-compatible$", ), ( _name_unknown_basic_input, @@ -318,6 +355,11 @@ def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: ), ( _make_economy_flag_missing, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: parameter economy$", + ), + ( + _make_economy_flag_null, "^parameter gov.example.rate economy must be a boolean$", ), ( @@ -335,3 +377,41 @@ def test_inconsistent_format_2_metadata_fails_with_a_specific_message( with pytest.raises(CatalogExtractionError, match=message): _extract(models) + + +def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> None: + """An older PolicyEngine.py fails before any record is checked, naming what lacks. + + PolicyEngine.py 6.7.5, for example, publishes none of these fields. + """ + models = source_models() + model = models["us"] + del model.entities, model.basic_inputs + for variable in model.variables: + for field_name in ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + ): + delattr(variable, field_name) + for parameter in model.parameters: + del parameter.economy, parameter.household + + with pytest.raises( + CatalogExtractionError, + match=r"^the installed PolicyEngine.py does not publish catalog format 2's " + r"fields for us: model version entities, basic_inputs; variable " + r"is_input_variable, hidden_input, module_name, adds_parameter_path, " + r"subtracts_parameter_path, enum_values; parameter economy, household$", + ): + _extract(models) + + +def test_an_empty_basic_input_list_is_stored_empty() -> None: + models = source_models() + models["us"].basic_inputs = [] + + assert _extract(models).country("us").model_version.basic_inputs == () From a94b2e2d4b6a25910b92c5b7f88cb3e11ed0ea97 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:35:19 +0400 Subject: [PATCH 047/118] Compare enum label types in the metadata parity test The parity test compared enum labels with ==, which treats 1, 1.0 and True as equal, so a label stored with the wrong type passed. It now compares each label's type name as well. Co-Authored-By: Claude Opus 5.5 --- .../test_v2_catalog_metadata_parity.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_v2_catalog_metadata_parity.py b/tests/integration/test_v2_catalog_metadata_parity.py index 9b9bd7790..f988d62bc 100644 --- a/tests/integration/test_v2_catalog_metadata_parity.py +++ b/tests/integration/test_v2_catalog_metadata_parity.py @@ -18,6 +18,8 @@ from policyengine_api.data.v2.catalog.extraction import normalize_model_metadata if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + from policyengine_api.data.v2.catalog.extraction import ModelMetadata pytestmark = pytest.mark.skipif( @@ -49,6 +51,20 @@ def _report(mismatches: list[str]) -> str: return f"{len(mismatches)} mismatches, first {min(len(mismatches), 20)}:\n{shown}" +def _typed_enum_values( + values: Iterable[Mapping[str, Any]], +) -> list[tuple[str, Any, str]]: + """Return each enum value with its label and the label's type name. + + Comparing the type name as well keeps 1, 1.0 and True apart, which ``==`` + treats as equal. + """ + return [ + (value["value"], value["label"], type(value["label"]).__name__) + for value in values + ] + + def test_entities_and_basic_inputs_match_v1( country_pair: tuple[ModelMetadata, Any], ) -> None: @@ -75,13 +91,18 @@ def test_variable_flags_modules_enum_labels_and_parts_match_v1( for variable in metadata.variables: v1_variable = v1_variables[variable.name] enum_values = ( - [ + _typed_enum_values( {"value": item.value, "label": item.label} for item in variable.enum_values - ] + ) if variable.enum_values is not None else None ) + v1_enum_values = ( + _typed_enum_values(v1_variable["possibleValues"]) + if "possibleValues" in v1_variable + else None + ) comparisons = { "is_input_variable": ( variable.is_input_variable, @@ -89,7 +110,7 @@ def test_variable_flags_modules_enum_labels_and_parts_match_v1( ), "hidden_input": (variable.hidden_input, v1_variable["hidden_input"]), "module_name": (variable.module_name, v1_variable["moduleName"]), - "enum_values": (enum_values, v1_variable.get("possibleValues")), + "enum_values": (enum_values, v1_enum_values), "adds_parameter_path": ( variable.adds_parameter_path, v1_variable["adds"] if isinstance(v1_variable["adds"], str) else None, From cca58e31ad34567a10efdae297eddbbd80ac0ff7 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:36:29 +0400 Subject: [PATCH 048/118] Test that enum labels and parameter paths round-trip through publication The publication tests stored only null enum values and parameter paths, so the JSON inserts and the comparison on republication never ran on stored values. A fixture now adds an enum variable with a string and an integer label, and a variable whose adds and subtracts are named by parameters. The tests check that they are stored exactly, the integer label as an integer, that republishing the same catalog changes nothing, and that a changed enum label or parameter path fails without changing the stored rows. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/v2_catalog.py | 69 +++++++++++ .../test_v2_catalog_publication.py | 111 +++++++++++++++++- 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 73f293237..6c0a47027 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -319,3 +319,72 @@ def normalized_catalog( installed_version=installed_version, region_references=region_references(), ) + + +def add_enum_and_parameter_part_records(model: SimpleNamespace) -> None: + """Add an enum variable and a variable whose parts are named by parameters. + + The enum's labels are a string and an integer. The two parameters hold the + lists ``countable_income`` adds and subtracts. + """ + for name in ("gov.example.countable_income", "gov.example.deductions"): + parameter = SimpleNamespace( + name=name, + label=None, + description=None, + data_type=list, + unit=None, + economy=True, + household=True, + parameter_values=[ + SimpleNamespace( + value=["employment_income"], + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ], + ) + model.parameters.append(parameter) + model.parameters_by_name[name] = parameter + variables = ( + SimpleNamespace( + name="filing_status", + label="Filing status", + entity="person", + description=None, + data_type="Enum", + possible_values=["SINGLE", "JOINT"], + default_value="SINGLE", + adds=None, + subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name="household.demographic.filing_status", + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=[ + SimpleNamespace(value="SINGLE", label="Single"), + SimpleNamespace(value="JOINT", label=2), + ], + ), + SimpleNamespace( + name="countable_income", + label="Countable income", + entity="person", + description=None, + data_type=float, + possible_values=None, + default_value=0.0, + adds=["employment_income"], + subtracts=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.countable_income", + adds_parameter_path="gov.example.countable_income", + subtracts_parameter_path="gov.example.deductions", + enum_values=None, + ), + ) + for variable in variables: + model.variables.append(variable) + model.variables_by_name[variable.name] = variable diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 0e4033379..18044b240 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -16,11 +16,13 @@ from sqlmodel import Session, select from policyengine_api.data.v2.catalog import publication as publication_module +from policyengine_api.data.v2.catalog.extraction import extract_catalog from policyengine_api.data.v2.catalog.publication import ( CatalogPublicationError, publish_catalog, unpublished_countries, ) +from policyengine_api.data.v2.catalog.records import EnumValueRecord from policyengine_api.data.v2.models import ( Dataset, DatasetVersion, @@ -32,7 +34,15 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from tests.fixtures.v2_catalog import normalized_catalog +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + add_enum_and_parameter_part_records, + bundle, + installed_version, + normalized_catalog, + region_references, + source_models, +) REPO = Path(__file__).parents[2] DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" @@ -678,3 +688,102 @@ def test_version_stored_with_format_2_fails_when_a_format_2_field_changes( assert _counts(publication_engine) == before_counts assert _format_2_values(publication_engine)["parameter"] == (True, False) + + +def _catalog_with_enum_and_parameter_parts(): + models = source_models() + add_enum_and_parameter_part_records(models["us"]) + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + region_references=region_references(), + ) + + +def _stored_us_variables(engine: Engine) -> dict[str, tuple[object, ...]]: + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT variable.name, variable.enum_values, " + "variable.adds_parameter_path, variable.subtracts_parameter_path " + "FROM variables AS variable " + "JOIN tax_benefit_model_versions AS version " + "ON version.id = variable.tax_benefit_model_version_id " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us'" + ) + ).all() + return {row[0]: tuple(row[1:]) for row in rows} + + +def _with_changed_us_variable(catalog, name: str, **changes: object): + us = catalog.country("us") + variables = tuple( + replace(variable, **changes) if variable.name == name else variable + for variable in us.variables + ) + return replace( + catalog, + countries=(replace(us, variables=variables), catalog.country("uk")), + ) + + +def test_enum_labels_and_parameter_paths_round_trip_exactly( + publication_engine: Engine, +) -> None: + catalog = _catalog_with_enum_and_parameter_parts() + + publish_catalog(publication_engine, catalog) + before_ids = _identifiers(publication_engine) + stored = _stored_us_variables(publication_engine) + republished = publish_catalog(publication_engine, catalog) + + assert stored["filing_status"] == ( + [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], + None, + None, + ) + assert type(stored["filing_status"][0][1]["label"]) is int + assert stored["countable_income"] == ( + None, + "gov.example.countable_income", + "gov.example.deductions", + ) + assert republished.as_dict()["outcome"] == "ok" + assert _identifiers(publication_engine) == before_ids + + +@pytest.mark.parametrize( + ("variable_name", "changes"), + [ + ( + "filing_status", + { + "enum_values": ( + EnumValueRecord(value="SINGLE", label="Single filer"), + EnumValueRecord(value="JOINT", label=2), + ) + }, + ), + ("countable_income", {"adds_parameter_path": "gov.example.deductions"}), + ("countable_income", {"subtracts_parameter_path": None}), + ], +) +def test_republication_detects_a_changed_enum_label_or_parameter_path( + publication_engine: Engine, + variable_name: str, + changes: dict[str, object], +) -> None: + catalog = _catalog_with_enum_and_parameter_parts() + publish_catalog(publication_engine, catalog) + stored = _stored_us_variables(publication_engine) + + with pytest.raises(CatalogPublicationError, match=r"in its variables$"): + publish_catalog( + publication_engine, + _with_changed_us_variable(catalog, variable_name, **changes), + ) + + assert _stored_us_variables(publication_engine) == stored From e46611fca4221d9fee6ec45172a78cd88244411e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:52:01 +0400 Subject: [PATCH 049/118] Type the catalog's entity and enum-value columns TaxBenefitModelVersion.entities and Variable.enum_values were typed as lists of untyped dictionaries. They are now lists of the TypedDicts CatalogEntity (key, plural, is_person) and CatalogEnumValue (value and declared label), which match EntityRecord and EnumValueRecord, and publication builds the stored JSON through them, so readers of these columns get the keys and value types without checking the shape again. The column types are unchanged, so there is no schema change. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/publication_staging.py | 16 ++++++---- policyengine_api/data/v2/models/__init__.py | 4 +++ policyengine_api/data/v2/models/metadata.py | 31 ++++++++++++++++--- tests/unit/v2/test_models.py | 27 ++++++++++++++++ 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index e0dfa3c31..3d341e488 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -19,6 +19,7 @@ VariableRecord, iter_batches, ) +from policyengine_api.data.v2.models import CatalogEntity, CatalogEnumValue if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator, Sequence @@ -178,11 +179,11 @@ def _model_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: country.model_version.catalog_format, Jsonb( [ - { - "key": entity.key, - "plural": entity.plural, - "is_person": entity.is_person, - } + CatalogEntity( + key=entity.key, + plural=entity.plural, + is_person=entity.is_person, + ) for entity in country.model_version.entities ] ), @@ -195,7 +196,10 @@ def _enum_values_json(record: VariableRecord) -> Jsonb | None: if record.enum_values is None: return None return Jsonb( - [{"value": item.value, "label": item.label} for item in record.enum_values] + [ + CatalogEnumValue(value=item.value, label=item.label) + for item in record.enum_values + ] ) diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index c63655bb6..1b1a73baf 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -40,6 +40,8 @@ HouseholdJobStatus, ) from policyengine_api.data.v2.models.metadata import ( # noqa: E402 + CatalogEntity, + CatalogEnumValue, Dataset, DatasetVersion, Parameter, @@ -99,6 +101,8 @@ "AggregateOutput", "AggregateType", "BudgetSummary", + "CatalogEntity", + "CatalogEnumValue", "ChangeAggregate", "CongressionalDistrictImpact", "ConstituencyImpact", diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index ca0153467..62a92b089 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -7,6 +7,7 @@ import sqlalchemy as sa from sqlmodel import Field, Relationship +from typing_extensions import TypedDict from policyengine_api.data.v2.models.base import ( IdentifiedModel, @@ -36,6 +37,28 @@ class RegionType(StrEnum): PLACE = "place" +class CatalogEntity(TypedDict): + """One entity of a catalog version's ``entities`` column. + + ``is_person`` marks the person entity; each version has exactly one. + """ + + key: str + plural: str + is_person: bool + + +class CatalogEnumValue(TypedDict): + """One value of a variable's ``enum_values`` column. + + ``value`` is the stored name and ``label`` the label the country package + declares, kept as declared (usually a string). + """ + + value: str + label: str | int | float | bool | None + + class TaxBenefitModel(TimestampedModel, table=True): """A country's tax-benefit model, unique by name. @@ -86,9 +109,9 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): current_law_id: int metadata_time_periods: list[int] = Field(sa_type=sa.JSON) # Catalog format 2 fields. Versions published before format 2 keep them - # null; entities are {key, plural, is_person} objects. + # null. catalog_format: int | None = None - entities: list[dict[str, Any]] | None = Field(default=None, sa_type=sa.JSON) + entities: list[CatalogEntity] | None = Field(default=None, sa_type=sa.JSON) basic_inputs: list[str] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") @@ -284,13 +307,13 @@ class Variable(IdentifiedModel, table=True): subtracts: list[str] | None = Field(default=None, sa_type=sa.JSON) # Catalog format 2 fields; null in versions published before it. A # parameter path names the parameter whose dated value lists the variables - # this one adds or subtracts. Enum values are {value, label} objects. + # this one adds or subtracts. is_input_variable: bool | None = None hidden_input: bool | None = None module_name: str | None = Field(default=None, max_length=512) adds_parameter_path: str | None = Field(default=None, max_length=512) subtracts_parameter_path: str | None = Field(default=None, max_length=512) - enum_values: list[dict[str, Any]] | None = Field(default=None, sa_type=sa.JSON) + enum_values: list[CatalogEnumValue] | None = Field(default=None, sa_type=sa.JSON) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index cdca61496..8cff45e68 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -721,3 +721,30 @@ def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: "subtracts_parameter_path", ): assert variables.c[column_name].type.length == 512 + + +def test_catalog_format_2_json_columns_have_typed_shapes() -> None: + """The entity and enum-value columns name typed shapes the catalog records match. + + Readers of these columns get the keys and value types without re-checking. + """ + from typing import get_type_hints + + from policyengine_api.data.v2.catalog.records import EntityRecord, EnumValueRecord + from policyengine_api.data.v2.models import ( + CatalogEntity, + CatalogEnumValue, + Variable, + ) + + assert TaxBenefitModelVersion.model_fields["entities"].annotation == ( + list[CatalogEntity] | None + ) + assert Variable.model_fields["enum_values"].annotation == ( + list[CatalogEnumValue] | None + ) + for typed_shape, record in ( + (CatalogEntity, EntityRecord), + (CatalogEnumValue, EnumValueRecord), + ): + assert get_type_hints(typed_shape) == get_type_hints(record) From 494f43465d8b513924d3bf62fa4b479fa9505b47 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:50:42 +0400 Subject: [PATCH 050/118] Take each country's default dataset from the PolicyEngine.py release The catalog fixed its own default datasets (populace_us_2024 and microcosm_uk_2024_25) and refused a release that named another. Each country's default dataset now comes from the installed PolicyEngine.py release's manifest. Publication still stops if the manifest names no default, if the release does not certify it, or if the release routes national regions to another dataset; those messages now name the datasets instead of a reviewed selection. Region scoping's argument is renamed to default_dataset, and the Stage 9 document and the changelog fragment describe the release's default. Co-Authored-By: Claude Opus 5.5 --- ...an-1f-115-catalog-metadata-fields.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 18 +++--- .../data/v2/catalog/extraction.py | 24 ++++---- .../data/v2/catalog/region_scoping.py | 25 ++++---- tests/unit/v2/test_catalog_extraction.py | 60 ++++++++++++++++++- .../v2/test_catalog_extraction_failures.py | 7 ++- 6 files changed, 100 insertions(+), 36 deletions(-) diff --git a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md index 7394dc074..fe578f85e 100644 --- a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md +++ b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md @@ -1 +1 @@ -Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, and its enum values' labels; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. +Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, and its enum values' labels; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. Each country's default dataset now comes from the installed PolicyEngine.py release's manifest; the Public API no longer fixes its own (populace_us_2024 and microcosm_uk_2024_25), and publication stops if the release names no default or does not certify it. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0d716beee..db5e02d61 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -139,15 +139,19 @@ Regions come from the Public API's checked-in reference files, one per country (`policyengine_api/data/v2/catalog/region_reference_files/`), not from PolicyEngine.py's region registry. Each row lists a region's code, label, type, parent, US state, the value that selects its households, and the v1 identifier -it replaces. A scoping rule per country and region type gives each region its -dataset from the certified data release and its filter: +it replaces. Each country's default dataset is the one the installed +PolicyEngine.py release names in its manifest (`default_dataset`); the Public +API sets none of its own. Publication stops if the manifest names no default, +if the release does not certify it, or if the release routes national regions +to another dataset. A scoping rule per country and region type gives each +region its dataset from the certified data release and its filter: | Regions | Dataset | Filter | | --- | --- | --- | -| US national | the release's national dataset (`populace_us_2024`) | none | +| US national | the release's default dataset (`populace_us_2024` in 6.10.0) | none | | US states and congressional districts | the release's state and district dataset | `row_filter` on `state_fips` or `congressional_district_geoid` | -| UK national | the release's national dataset (`microcosm_uk_2024_25`) | none | -| UK nations | the release's national dataset | `row_filter` on `region` | +| UK national | the release's default dataset (`microcosm_uk_2024_25` in 6.10.0) | none | +| UK nations | the release's default dataset | `row_filter` on `region` | | UK constituencies and local authorities | the release's local-area dataset (`microcosm_uk_2024_25_dense`), which carries their codes | `row_filter` on `constituency_code_oa` or `la_code_oa`, value the ONS code | If the release names no dataset for a US state or district, those regions use @@ -165,8 +169,8 @@ household columns. It does not check that a column holds rows for every region's value. Review the fallback and omitted-region summaries on every new PolicyEngine.py -release. A changed count may identify an upstream catalog change that needs a -reviewed dataset selection even when publication otherwise succeeds. +release. A changed count may identify an upstream change to the certified data +release even when publication otherwise succeeds. A catalog version is identified by its country and PolicyEngine.py version and never changes once published. When the command republishes a stored version diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 34ffcc803..51c7bd5b2 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -49,10 +49,6 @@ "policyengine-us", "policyengine-uk", ) -REVIEWED_DEFAULT_DATASETS = { - "us": "populace_us_2024", - "uk": "microcosm_uk_2024_25", -} CURRENT_LAW_IDS = {"us": 2, "uk": 1} METADATA_TIME_PERIODS = { "us": tuple(range(2035, 2021, -1)), @@ -813,14 +809,17 @@ def normalize_model_metadata( ) -def _verified_default_dataset(country_id: str, release_manifest: object) -> str: - """Return the reviewed default dataset once the manifest names it as its default.""" - reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] - manifest_default = getattr(release_manifest, "default_dataset", None) - if manifest_default != reviewed_default: - msg = f"{country_id} default dataset does not match the reviewed selection" +def _release_default_dataset(country_id: str, release_manifest: object) -> str: + """Return the default dataset the installed PolicyEngine.py release names. + + The Public API sets no default of its own; region scoping then requires the + release to certify it. + """ + default_dataset = getattr(release_manifest, "default_dataset", None) + if not isinstance(default_dataset, str) or not default_dataset: + msg = f"{country_id} release manifest names no default dataset" raise CatalogExtractionError(msg) - return reviewed_default + return default_dataset def _normalize_region( @@ -872,12 +871,11 @@ def _normalize_country( country_id, source, model_version_id=model_version_id ) - reviewed_default = _verified_default_dataset(country_id, release_manifest) scoping = scope_regions( country_id, region_references, release_manifest, - reviewed_default=reviewed_default, + default_dataset=_release_default_dataset(country_id, release_manifest), ) datasets = tuple( diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py index 3432dcb4c..aa11d8cc7 100644 --- a/policyengine_api/data/v2/catalog/region_scoping.py +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -237,13 +237,14 @@ def scope_regions( references: tuple[RegionReference, ...], release_manifest: object, *, - reviewed_default: str, + default_dataset: str, ) -> RegionScoping: """Give every runnable reference region its certified dataset and filter. - The national region must use the reviewed default dataset. A region whose - type has no scoping rule is left out, and a runnable region may not sit - under a region that is left out. + ``default_dataset`` is the release's own default. The release must certify + it, and the national region must use it. A region whose type has no + scoping rule is left out, and a runnable region may not sit under a region + that is left out. """ if not isinstance(release_manifest, ReleaseManifest): msg = ( @@ -252,12 +253,16 @@ def scope_regions( ) raise CatalogExtractionError(msg) artifacts = certified_artifacts(country_id, release_manifest) - if reviewed_default not in artifacts: - msg = f"{country_id} release does not certify {reviewed_default}" + if default_dataset not in artifacts: + msg = f"{country_id} release does not certify {default_dataset}" raise CatalogExtractionError(msg) role_datasets = _role_datasets(country_id, release_manifest, artifacts) - if role_datasets.get(NATIONAL_DATASET_ROLE, reviewed_default) != reviewed_default: - msg = f"{country_id} national region does not use the reviewed dataset" + national_dataset = role_datasets.get(NATIONAL_DATASET_ROLE, default_dataset) + if national_dataset != default_dataset: + msg = ( + f"{country_id} release routes national regions to {national_dataset}, " + f"not to its default dataset {default_dataset}" + ) raise CatalogExtractionError(msg) omitted = { @@ -287,7 +292,7 @@ def scope_regions( "cannot select its households" ) raise CatalogExtractionError(msg) - dataset_name = reviewed_default + dataset_name = default_dataset if rule.dataset_role != NATIONAL_DATASET_ROLE: fallback_counts[reference.region_type] += 1 regions.append( @@ -301,7 +306,7 @@ def scope_regions( return RegionScoping( regions=tuple(regions), dataset_names=frozenset( - {reviewed_default, *(region.dataset_name for region in regions)} + {default_dataset, *(region.dataset_name for region in regions)} ), certified_artifacts=artifacts, fallback_counts=dict(sorted(fallback_counts.items())), diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index bc6d45eca..66ace2ced 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -367,9 +367,65 @@ def test_rejects_incomplete_public_model_catalogs() -> None: installed_version=installed_version, ) + +def _use_another_certified_default(model: SimpleNamespace, dataset_name: str) -> None: + """Make the release default to another dataset it certifies, for every region.""" + manifest = model.release_manifest + manifest.default_dataset = dataset_name + manifest.datasets[dataset_name] = SimpleNamespace( + path=f"{dataset_name}.h5", + revision="fixture", + sha256="0" * 64, + repo_id=None, + repo_type=None, + ) + for template in manifest.region_datasets.values(): + template.path_template = f"{dataset_name}.h5" + + +def test_each_country_uses_the_default_dataset_its_release_names() -> None: + """The release's own default is used; the Public API fixes none of its own.""" + models = source_models() + _use_another_certified_default(models["us"], "populace_us_2025") + _use_another_certified_default(models["uk"], "microcosm_uk_2025_26") + + catalog = extract_catalog( + region_references=region_references(), + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + for country_id, default_dataset in ( + ("us", "populace_us_2025"), + ("uk", "microcosm_uk_2025_26"), + ): + country = catalog.country(country_id) + dataset_names = {dataset.id: dataset.name for dataset in country.datasets} + national = next( + region for region in country.regions if region.region_type == "national" + ) + assert dataset_names[national.default_dataset_id] == default_dataset + assert set(dataset_names.values()) == {default_dataset} + + +@pytest.mark.parametrize( + ("default_dataset", "message"), + [ + (None, "^us release manifest names no default dataset$"), + ("", "^us release manifest names no default dataset$"), + ("populace_us_2025", "^us release does not certify populace_us_2025$"), + ], +) +def test_a_missing_or_uncertified_release_default_fails( + default_dataset: str | None, + message: str, +) -> None: models = source_models() - models["us"].release_manifest.default_dataset = "unreviewed_us_2025" - with pytest.raises(CatalogExtractionError, match="reviewed selection"): + models["us"].release_manifest.default_dataset = default_dataset + + with pytest.raises(CatalogExtractionError, match=message): extract_catalog( region_references=region_references(), bundle=bundle(), diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index 70f5f954c..b09bff9fd 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -160,7 +160,7 @@ def _remove_the_dataset_map( models["us"].release_manifest.datasets = None -def _remove_the_reviewed_default_dataset( +def _remove_the_release_default_dataset( models: dict[str, SimpleNamespace], _references: References ) -> None: del models["us"].release_manifest.datasets["populace_us_2024"] @@ -214,7 +214,7 @@ def _remove_every_artifact_repository( "^us release manifest lacks datasets, region_datasets or data_package$", ), ( - _remove_the_reviewed_default_dataset, + _remove_the_release_default_dataset, "^us release does not certify populace_us_2024$", ), ( @@ -241,7 +241,8 @@ def _remove_every_artifact_repository( ), ( _point_national_region_at_another_dataset, - "^uk national region does not use the reviewed dataset$", + "^uk release routes national regions to frs_2024_25, not to its default " + "dataset microcosm_uk_2024_25$", ), ( _name_an_uncertified_region_dataset, From 1f69e91440248f3d5c317ccd02cf39f8b18de5b3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:42:26 +0400 Subject: [PATCH 051/118] Say that the locked PolicyEngine.py 6.10.0 also lacks catalog format 2's fields The Stage 9 document, the format 2 field check's docstring and its test named only 6.7.5 as a release without the fields; the locked 6.10.0 lacks them too. The Stage 9 regions table also names the UK local-area dataset as the one 6.10.0 uses, like the rows above it. Documentation only. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 6 +++--- policyengine_api/data/v2/catalog/extraction.py | 3 ++- tests/unit/v2/test_catalog_metadata_fields.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index db5e02d61..b8f99b3e4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -112,8 +112,8 @@ publishes entities, basic inputs and these flags: Bracket schedules are parameter nodes with their stored label, or none when the package stores none. An empty basic-input list is stored as empty. Extraction stops before reading any record if the installed PolicyEngine.py -does not publish these fields, as 6.7.5 does not, and names each missing -field. It also stops if the model has other than one person entity, names a +does not publish these fields, as neither 6.7.5 nor the locked 6.10.0 does, +and names each missing field. It also stops if the model has other than one person entity, names a basic input or a parameter path that is not published, gives enum labels that do not match the variable's possible values or are not JSON scalars, or gives a flag that is not a boolean. An opt-in test @@ -152,7 +152,7 @@ region its dataset from the certified data release and its filter: | US states and congressional districts | the release's state and district dataset | `row_filter` on `state_fips` or `congressional_district_geoid` | | UK national | the release's default dataset (`microcosm_uk_2024_25` in 6.10.0) | none | | UK nations | the release's default dataset | `row_filter` on `region` | -| UK constituencies and local authorities | the release's local-area dataset (`microcosm_uk_2024_25_dense`), which carries their codes | `row_filter` on `constituency_code_oa` or `la_code_oa`, value the ONS code | +| UK constituencies and local authorities | the release's local-area dataset (`microcosm_uk_2024_25_dense` in 6.10.0), which carries their codes | `row_filter` on `constituency_code_oa` or `la_code_oa`, value the ONS code | If the release names no dataset for a US state or district, those regions use the default dataset and the command emits one summary warning with the affected diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 51c7bd5b2..1faad4e4d 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -644,7 +644,8 @@ def _normalize_parameter_nodes( def _require_format_2_fields(country_id: str, source: object) -> None: """Fail first, naming each field, when PolicyEngine.py does not publish them. - Older releases, such as 6.7.5, publish none of catalog format 2's fields. + Releases without them, such as 6.7.5 and the locked 6.10.0, publish none of + catalog format 2's fields. """ missing: list[str] = [] for record_name, mapping_attribute, field_names in FORMAT_2_FIELDS: diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py index ab6f6414b..8ef59e60e 100644 --- a/tests/unit/v2/test_catalog_metadata_fields.py +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -380,9 +380,9 @@ def test_inconsistent_format_2_metadata_fails_with_a_specific_message( def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> None: - """An older PolicyEngine.py fails before any record is checked, naming what lacks. + """A release without the fields fails before any record is checked, naming each. - PolicyEngine.py 6.7.5, for example, publishes none of these fields. + PolicyEngine.py 6.7.5 and the locked 6.10.0, for example, publish none of them. """ models = source_models() model = models["us"] From 5c565e61f98e08b7809a5f75017818107a3b2e85 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:17:30 +0400 Subject: [PATCH 052/118] Add each variable's unit and definition period to the catalog format 2 columns The app's v3 household builder and breakdown need each variable's unit (currency formatting) and definition period ("per month" or "per year" labels, household validation), and PolicyEngine.py's public Variable record carries both. The v2 catalog stored neither. Revision f5c1d9b4685b is not merged yet, so it was regenerated with autogenerate against the v2 configuration, keeping its ID through --rev-id and its parent fa7fa1035444. It now adds 13 nullable columns with no defaults, including variables.unit (128 characters) and variables.definition_period (32 characters), and writes no data. Co-Authored-By: Claude Opus 5.5 --- ...c1d9b4685b_store_catalog_format_2_metadata.py | 16 +++++++++++++++- policyengine_api/data/v2/models/metadata.py | 4 ++++ tests/integration/test_alembic_v2_lifecycle.py | 4 +++- tests/unit/v2/test_alembic_v2.py | 2 ++ tests/unit/v2/test_models.py | 4 ++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py index b0dc56d28..fbaefb08f 100644 --- a/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py +++ b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py @@ -2,7 +2,7 @@ Revision ID: f5c1d9b4685b Revises: fa7fa1035444 -Create Date: 2026-10-22 18:59:48.186978 +Create Date: 2026-10-26 20:13:40.733924 Generation: uv run alembic -c alembic-v2.ini revision --autogenerate """ @@ -61,11 +61,25 @@ def upgrade() -> None: ), ) op.add_column("variables", sa.Column("enum_values", sa.JSON(), nullable=True)) + op.add_column( + "variables", + sa.Column("unit", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + ) + op.add_column( + "variables", + sa.Column( + "definition_period", + sqlmodel.sql.sqltypes.AutoString(length=32), + nullable=True, + ), + ) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("variables", "definition_period") + op.drop_column("variables", "unit") op.drop_column("variables", "enum_values") op.drop_column("variables", "subtracts_parameter_path") op.drop_column("variables", "adds_parameter_path") diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 62a92b089..5431a6abc 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -314,6 +314,10 @@ class Variable(IdentifiedModel, table=True): adds_parameter_path: str | None = Field(default=None, max_length=512) subtracts_parameter_path: str | None = Field(default=None, max_length=512) enum_values: list[CatalogEnumValue] | None = Field(default=None, sa_type=sa.JSON) + # The unit (for example ``currency-USD``) and the period a value covers + # (``year``, ``month`` or ``eternity``), as the country package declares. + unit: str | None = Field(default=None, max_length=128) + definition_period: str | None = Field(default=None, max_length=32) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 61ff7a72b..89837594a 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -1426,6 +1426,8 @@ def test_previous_release_still_finds_user_policies() -> None: "adds_parameter_path", "subtracts_parameter_path", "enum_values", + "unit", + "definition_period", ), "parameters": ("economy", "household"), } @@ -1517,7 +1519,7 @@ def test_catalog_format_2_columns_leave_earlier_versions_empty() -> None: with engine.connect() as connection: assert _format_2_values(connection, version_id) == [ (None, None, None), - (None, None, None, None, None, None), + (None, None, None, None, None, None, None, None), (None, None), ] diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 5a3d20de6..d6eb45ae0 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -844,6 +844,8 @@ def test_catalog_format_2_revision_adds_only_nullable_columns() -> None: "adds_parameter_path", "subtracts_parameter_path", "enum_values", + "unit", + "definition_period", ), "parameters": ("economy", "household"), } diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 8cff45e68..c93d214b0 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -699,6 +699,8 @@ def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: "adds_parameter_path": AutoString, "subtracts_parameter_path": AutoString, "enum_values": sa.JSON, + "unit": AutoString, + "definition_period": AutoString, }, "parameters": { "economy": sa.Boolean, @@ -721,6 +723,8 @@ def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: "subtracts_parameter_path", ): assert variables.c[column_name].type.length == 512 + assert variables.c["unit"].type.length == 128 + assert variables.c["definition_period"].type.length == 32 def test_catalog_format_2_json_columns_have_typed_shapes() -> None: From 376846411efd8bd95bd696166b1a967a544523c2 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:23:52 +0400 Subject: [PATCH 053/118] Extract each variable's unit and definition period from PolicyEngine.py's public records Extraction now reads each variable's unit and definition period from PolicyEngine.py's public Variable record (both optional strings, at most 128 and 32 characters), and the format 2 field check names them when a release does not publish them. 6.7.5 and the locked 6.10.0 already carry these two, so on 6.10.0 the check still names only the other fields. The parity test compares both with the unit and definitionPeriod that /{country}/metadata builds for every US and UK variable. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/extraction.py | 16 +++++++- policyengine_api/data/v2/catalog/records.py | 2 + tests/fixtures/v2_catalog.py | 6 +++ .../test_v2_catalog_metadata_parity.py | 5 +++ tests/unit/v2/test_catalog_metadata_fields.py | 37 ++++++++++++++++++- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 1faad4e4d..84606329f 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -360,6 +360,16 @@ def optional_names(attribute: str) -> list[str] | None: maximum=512, ), enum_values=enum_values, + unit=_optional_text( + getattr(source, "unit", None), + field_name=f"variable {name} unit", + maximum=128, + ), + definition_period=_optional_text( + getattr(source, "definition_period", None), + field_name=f"variable {name} definition_period", + maximum=32, + ), ) @@ -635,6 +645,8 @@ def _normalize_parameter_nodes( "adds_parameter_path", "subtracts_parameter_path", "enum_values", + "unit", + "definition_period", ), ), ("parameter", "parameters_by_name", ("economy", "household")), @@ -644,8 +656,8 @@ def _normalize_parameter_nodes( def _require_format_2_fields(country_id: str, source: object) -> None: """Fail first, naming each field, when PolicyEngine.py does not publish them. - Releases without them, such as 6.7.5 and the locked 6.10.0, publish none of - catalog format 2's fields. + Older releases, such as 6.7.5 and the locked 6.10.0, publish only the + variables' unit and definition period of catalog format 2's fields. """ missing: list[str] = [] for record_name, mapping_attribute, field_names in FORMAT_2_FIELDS: diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index 56eaafc95..622a35cb6 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -97,6 +97,8 @@ class VariableRecord: adds_parameter_path: str | None subtracts_parameter_path: str | None enum_values: tuple[EnumValueRecord, ...] | None + unit: str | None + definition_period: str | None @dataclass(frozen=True, slots=True) diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 6c0a47027..d24c6ff44 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -227,6 +227,8 @@ def _model_source( adds_parameter_path=None, subtracts_parameter_path=None, enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter_node = SimpleNamespace( name="gov.example", @@ -366,6 +368,8 @@ def add_enum_and_parameter_part_records(model: SimpleNamespace) -> None: SimpleNamespace(value="SINGLE", label="Single"), SimpleNamespace(value="JOINT", label=2), ], + unit=None, + definition_period="year", ), SimpleNamespace( name="countable_income", @@ -383,6 +387,8 @@ def add_enum_and_parameter_part_records(model: SimpleNamespace) -> None: adds_parameter_path="gov.example.countable_income", subtracts_parameter_path="gov.example.deductions", enum_values=None, + unit="currency-USD", + definition_period="month", ), ) for variable in variables: diff --git a/tests/integration/test_v2_catalog_metadata_parity.py b/tests/integration/test_v2_catalog_metadata_parity.py index f988d62bc..3a426ad0f 100644 --- a/tests/integration/test_v2_catalog_metadata_parity.py +++ b/tests/integration/test_v2_catalog_metadata_parity.py @@ -111,6 +111,11 @@ def test_variable_flags_modules_enum_labels_and_parts_match_v1( "hidden_input": (variable.hidden_input, v1_variable["hidden_input"]), "module_name": (variable.module_name, v1_variable["moduleName"]), "enum_values": (enum_values, v1_enum_values), + "unit": (variable.unit, v1_variable["unit"]), + "definition_period": ( + variable.definition_period, + v1_variable["definitionPeriod"], + ), "adds_parameter_path": ( variable.adds_parameter_path, v1_variable["adds"] if isinstance(v1_variable["adds"], str) else None, diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py index 8ef59e60e..48c85587f 100644 --- a/tests/unit/v2/test_catalog_metadata_fields.py +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -57,6 +57,8 @@ def _add_variable(model: SimpleNamespace, **fields: object) -> None: adds_parameter_path=None, subtracts_parameter_path=None, enum_values=None, + unit=None, + definition_period="year", ) for name, value in fields.items(): setattr(variable, name, value) @@ -114,6 +116,8 @@ def test_variables_carry_input_flags_module_and_unset_parts() -> None: assert variable.adds_parameter_path is None assert variable.subtracts_parameter_path is None assert variable.enum_values is None + assert variable.unit == "currency-USD" + assert variable.definition_period == "year" def test_enum_values_keep_their_declared_labels_in_order() -> None: @@ -239,6 +243,18 @@ def _name_unknown_basic_input(models: dict[str, SimpleNamespace]) -> None: models["us"].basic_inputs = ["employment_income", "unpublished_input"] +def _remove_variable_unit(models: dict[str, SimpleNamespace]) -> None: + del models["us"].variables[0].unit + + +def _make_unit_a_number(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].unit = 1 + + +def _make_definition_period_a_list(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].definition_period = ["year"] + + def _make_input_flag_text(models: dict[str, SimpleNamespace]) -> None: models["us"].variables[0].is_input_variable = "true" @@ -323,6 +339,19 @@ def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: _name_unknown_basic_input, "^us basic input 'unpublished_input' is not a published variable$", ), + ( + _remove_variable_unit, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: variable unit$", + ), + ( + _make_unit_a_number, + "^variable employment_income unit must be a string or null$", + ), + ( + _make_definition_period_a_list, + "^variable employment_income definition_period must be a string or null$", + ), ( _make_input_flag_text, "^variable employment_income is_input_variable must be a boolean or null$", @@ -382,7 +411,8 @@ def test_inconsistent_format_2_metadata_fails_with_a_specific_message( def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> None: """A release without the fields fails before any record is checked, naming each. - PolicyEngine.py 6.7.5 and the locked 6.10.0, for example, publish none of them. + PolicyEngine.py 6.7.5 and the locked 6.10.0, for example, publish only the + variables' unit and definition period of these fields. """ models = source_models() model = models["us"] @@ -395,6 +425,8 @@ def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> "adds_parameter_path", "subtracts_parameter_path", "enum_values", + "unit", + "definition_period", ): delattr(variable, field_name) for parameter in model.parameters: @@ -405,7 +437,8 @@ def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> match=r"^the installed PolicyEngine.py does not publish catalog format 2's " r"fields for us: model version entities, basic_inputs; variable " r"is_input_variable, hidden_input, module_name, adds_parameter_path, " - r"subtracts_parameter_path, enum_values; parameter economy, household$", + r"subtracts_parameter_path, enum_values, unit, definition_period; parameter " + r"economy, household$", ): _extract(models) From 7c0eac46f41179edc6b8c25447a86181fa7a3618 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:27:56 +0400 Subject: [PATCH 054/118] Store each variable's unit and definition period when a catalog version is published Publication now stages and inserts each variable's unit and definition period, and a stored format 2 version must match on both when the same PolicyEngine.py version is published again; a version stored before format 2 keeps them empty. The Stage 9 document and the changelog fragment list both fields and say that 6.7.5 and the locked 6.10.0 already publish these two. Co-Authored-By: Claude Opus 5.5 --- ...an-1f-115-catalog-metadata-fields.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 24 +++++++++---------- .../v2/catalog/publication_reconciliation.py | 8 +++++++ .../data/v2/catalog/publication_staging.py | 4 ++++ .../test_v2_catalog_publication.py | 19 +++++++++++---- tests/unit/v2/test_catalog_publication.py | 2 +- tests/unit/v2/test_catalog_staging_rows.py | 5 +++- 7 files changed, 45 insertions(+), 19 deletions(-) diff --git a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md index fe578f85e..03f5b55f3 100644 --- a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md +++ b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md @@ -1 +1 @@ -Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, and its enum values' labels; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. Each country's default dataset now comes from the installed PolicyEngine.py release's manifest; the Public API no longer fixes its own (populace_us_2024 and microcosm_uk_2024_25), and publication stops if the release names no default or does not certify it. +Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, its enum values' labels, its unit and its definition period; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. Each country's default dataset now comes from the installed PolicyEngine.py release's manifest; the Public API no longer fixes its own (populace_us_2024 and microcosm_uk_2024_25), and publication stops if the release names no default or does not certify it. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index b8f99b3e4..0ad3eb455 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -106,20 +106,20 @@ publishes entities, basic inputs and these flags: | Record | Fields | | --- | --- | | Model version | `catalog_format` (2), `entities` (`key`, `plural`, `is_person`, in the package's order), `basic_inputs` (the household builder's default fields) | -| Variable | `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path` and `subtracts_parameter_path` (the parameter whose dated value lists the variables summed; `adds` and `subtracts` keep the list PolicyEngine.py resolved), `enum_values` (`value` and declared `label`, in order) | +| Variable | `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path` and `subtracts_parameter_path` (the parameter whose dated value lists the variables summed; `adds` and `subtracts` keep the list PolicyEngine.py resolved), `enum_values` (`value` and declared `label`, in order), `unit` (for example `currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which the household builder and breakdown read | | Parameter | `economy`, `household` | -Bracket schedules are parameter nodes with their stored label, or none when -the package stores none. An empty basic-input list is stored as empty. -Extraction stops before reading any record if the installed PolicyEngine.py -does not publish these fields, as neither 6.7.5 nor the locked 6.10.0 does, -and names each missing field. It also stops if the model has other than one person entity, names a -basic input or a parameter path that is not published, gives enum labels that -do not match the variable's possible values or are not JSON scalars, or gives -a flag that is not a boolean. An opt-in test -(`tests/integration/test_v2_catalog_metadata_parity.py`, run by the v2 -integration check) compares every format 2 field for the US and the UK with -what `/{country}/metadata` builds. +Bracket schedules are parameter nodes with their stored label, or none when the +package stores none. An empty basic-input list is stored as empty. Extraction +stops before reading any record if the installed PolicyEngine.py does not +publish these fields, and names each missing field: 6.7.5 and the locked 6.10.0 +publish only the variables' unit and definition period. It also stops if the +model has other than one person entity, names a basic input or a parameter path +that is not published, gives enum labels that do not match the variable's +possible values or are not JSON scalars, or gives a flag that is not a boolean. +An opt-in test (`tests/integration/test_v2_catalog_metadata_parity.py`, run by +the v2 integration check) compares every format 2 field for the US and the UK +with what `/{country}/metadata` builds. Publication never rewrites a stored version. A version published before format 2 has a null `catalog_format` and null format 2 fields, and later diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 9ddcad20d..14390b6ee 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -168,6 +168,8 @@ def format_2(*columns: sa.ColumnElement) -> tuple[sa.ColumnElement, ...]: STAGE_CATALOG_VARIABLES.c.adds_parameter_path, STAGE_CATALOG_VARIABLES.c.subtracts_parameter_path, STAGE_CATALOG_VARIABLES.c.enum_values, + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.definition_period, ), ).where(STAGE_CATALOG_VARIABLES.c.country_id == country.country_id) actual_variables = ( @@ -188,6 +190,8 @@ def format_2(*columns: sa.ColumnElement) -> tuple[sa.ColumnElement, ...]: VARIABLES.c.adds_parameter_path, VARIABLES.c.subtracts_parameter_path, sa.cast(VARIABLES.c.enum_values, JSONB), + VARIABLES.c.unit, + VARIABLES.c.definition_period, ), ) .select_from(_versioned_table_from(VARIABLES)) @@ -525,6 +529,8 @@ def count_region_differences( "adds_parameter_path", "subtracts_parameter_path", "enum_values", + "unit", + "definition_period", "tax_benefit_model_version_id", ], sa.select( @@ -557,6 +563,8 @@ def count_region_differences( STAGE_CATALOG_VARIABLES.c.enum_values, VARIABLES.c.enum_values.type, ), + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.definition_period, MODEL_VERSIONS.c.id, ) .select_from( diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index 3d341e488..f4243fa64 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -68,6 +68,8 @@ sa.Column("adds_parameter_path", sa.Text), sa.Column("subtracts_parameter_path", sa.Text), sa.Column("enum_values", JSONB), + sa.Column("unit", sa.Text), + sa.Column("definition_period", sa.Text), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -225,6 +227,8 @@ def _variable_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: record.adds_parameter_path, record.subtracts_parameter_path, _enum_values_json(record), + record.unit, + record.definition_period, ) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 18044b240..92adc02d9 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -576,7 +576,8 @@ def _format_2_values(engine: Engine) -> dict[str, object]: variable = connection.execute( text( "SELECT is_input_variable, hidden_input, module_name, " - "adds_parameter_path, subtracts_parameter_path, enum_values " + "adds_parameter_path, subtracts_parameter_path, enum_values, " + "unit, definition_period " "FROM variables WHERE name = 'employment_income' " "AND tax_benefit_model_version_id = (" "SELECT version.id FROM tax_benefit_model_versions AS version " @@ -614,7 +615,8 @@ def _clear_format_2_fields(engine: Engine) -> None: text( "UPDATE variables SET is_input_variable = NULL, hidden_input = NULL, " "module_name = NULL, adds_parameter_path = NULL, " - "subtracts_parameter_path = NULL, enum_values = NULL" + "subtracts_parameter_path = NULL, enum_values = NULL, " + "unit = NULL, definition_period = NULL" ) ) connection.execute( @@ -643,6 +645,8 @@ def test_publication_stores_catalog_format_2_fields( None, None, None, + "currency-USD", + "year", ), "parameter": (True, False), } @@ -662,7 +666,7 @@ def test_version_stored_before_format_2_keeps_its_fields_empty_on_republish( assert _identifiers(publication_engine) == before_ids assert _format_2_values(publication_engine) == { "version": (None, None, None), - "variable": (None, None, None, None, None, None), + "variable": (None, None, None, None, None, None, None, None), "parameter": (None, None), } @@ -707,7 +711,8 @@ def _stored_us_variables(engine: Engine) -> dict[str, tuple[object, ...]]: rows = connection.execute( text( "SELECT variable.name, variable.enum_values, " - "variable.adds_parameter_path, variable.subtracts_parameter_path " + "variable.adds_parameter_path, variable.subtracts_parameter_path, " + "variable.unit, variable.definition_period " "FROM variables AS variable " "JOIN tax_benefit_model_versions AS version " "ON version.id = variable.tax_benefit_model_version_id " @@ -744,12 +749,16 @@ def test_enum_labels_and_parameter_paths_round_trip_exactly( [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], None, None, + None, + "year", ) assert type(stored["filing_status"][0][1]["label"]) is int assert stored["countable_income"] == ( None, "gov.example.countable_income", "gov.example.deductions", + "currency-USD", + "month", ) assert republished.as_dict()["outcome"] == "ok" assert _identifiers(publication_engine) == before_ids @@ -769,6 +778,8 @@ def test_enum_labels_and_parameter_paths_round_trip_exactly( ), ("countable_income", {"adds_parameter_path": "gov.example.deductions"}), ("countable_income", {"subtracts_parameter_path": None}), + ("filing_status", {"unit": "currency-USD"}), + ("countable_income", {"definition_period": "year"}), ], ) def test_republication_detects_a_changed_enum_label_or_parameter_path( diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index afdc7f5a2..47cb262ba 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -249,7 +249,7 @@ def test_publication_uses_mapped_tables_and_sqlalchemy_statements() -> None: for format_1, format_2 in zip( format_1_comparisons, format_2_comparisons, strict=True ) - ] == [3, 6, 0, 2, 0, 0] + ] == [3, 8, 0, 2, 0, 0] class Result: def scalar_one(self): diff --git a/tests/unit/v2/test_catalog_staging_rows.py b/tests/unit/v2/test_catalog_staging_rows.py index 549743810..e5c86c3d6 100644 --- a/tests/unit/v2/test_catalog_staging_rows.py +++ b/tests/unit/v2/test_catalog_staging_rows.py @@ -84,6 +84,8 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: record.adds_parameter_path, record.subtracts_parameter_path, None, + record.unit, + record.definition_period, ) for record in country.variables ] @@ -157,7 +159,8 @@ def test_enum_values_are_staged_as_value_and_label_objects_in_order() -> None: (row,) = (_plain(row) for row in _catalog_rows(country)[STAGE_CATALOG_VARIABLES]) - assert row[-1] == ( + enum_values_column = [column.name for column in STAGE_CATALOG_VARIABLES.columns] + assert row[enum_values_column.index("enum_values")] == ( "jsonb", [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], ) From 7e0fb951845418ced7f4171dfa5412662c6b0e09 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:32:24 +0400 Subject: [PATCH 055/118] Name the unit and definition period in the format 2 test and fixture names After the unit and definition period were added, the round-trip, republication, parity and extraction tests and the fixture helper still named only the earlier fields, so a failure did not say which field it covers. They now name the unit and period (the fixture helper becomes add_format_2_variable_records, with a docstring listing what each variable fills), and the staging-row test's list of every staged column name is called staged_column_names. Tests only; behavior is unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/fixtures/v2_catalog.py | 10 ++++++---- .../integration/test_v2_catalog_metadata_parity.py | 2 +- tests/integration/test_v2_catalog_publication.py | 14 +++++++------- tests/unit/v2/test_catalog_metadata_fields.py | 2 +- tests/unit/v2/test_catalog_staging_rows.py | 4 ++-- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index d24c6ff44..05bbf00e1 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -323,11 +323,13 @@ def normalized_catalog( ) -def add_enum_and_parameter_part_records(model: SimpleNamespace) -> None: - """Add an enum variable and a variable whose parts are named by parameters. +def add_format_2_variable_records(model: SimpleNamespace) -> None: + """Add two variables that between them fill every format 2 variable field. - The enum's labels are a string and an integer. The two parameters hold the - lists ``countable_income`` adds and subtracts. + ``filing_status`` is an enum whose labels are a string and an integer, with + no unit and a yearly period. ``countable_income`` names the parameters that + hold the lists it adds and subtracts, with a currency unit and a monthly + period. """ for name in ("gov.example.countable_income", "gov.example.deductions"): parameter = SimpleNamespace( diff --git a/tests/integration/test_v2_catalog_metadata_parity.py b/tests/integration/test_v2_catalog_metadata_parity.py index 3a426ad0f..079e31610 100644 --- a/tests/integration/test_v2_catalog_metadata_parity.py +++ b/tests/integration/test_v2_catalog_metadata_parity.py @@ -80,7 +80,7 @@ def test_entities_and_basic_inputs_match_v1( assert list(metadata.basic_inputs) == (v1_country.metadata["basicInputs"]) -def test_variable_flags_modules_enum_labels_and_parts_match_v1( +def test_variable_flags_modules_enums_parts_units_and_periods_match_v1( country_pair: tuple[ModelMetadata, Any], ) -> None: metadata, v1_country = country_pair diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 92adc02d9..a5ba64d1e 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -36,7 +36,7 @@ from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from tests.fixtures.v2_catalog import ( POLICYENGINE_VERSION, - add_enum_and_parameter_part_records, + add_format_2_variable_records, bundle, installed_version, normalized_catalog, @@ -694,9 +694,9 @@ def test_version_stored_with_format_2_fails_when_a_format_2_field_changes( assert _format_2_values(publication_engine)["parameter"] == (True, False) -def _catalog_with_enum_and_parameter_parts(): +def _catalog_with_format_2_variable_records(): models = source_models() - add_enum_and_parameter_part_records(models["us"]) + add_format_2_variable_records(models["us"]) return extract_catalog( bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, @@ -735,10 +735,10 @@ def _with_changed_us_variable(catalog, name: str, **changes: object): ) -def test_enum_labels_and_parameter_paths_round_trip_exactly( +def test_enum_labels_parameter_paths_units_and_periods_round_trip_exactly( publication_engine: Engine, ) -> None: - catalog = _catalog_with_enum_and_parameter_parts() + catalog = _catalog_with_format_2_variable_records() publish_catalog(publication_engine, catalog) before_ids = _identifiers(publication_engine) @@ -782,12 +782,12 @@ def test_enum_labels_and_parameter_paths_round_trip_exactly( ("countable_income", {"definition_period": "year"}), ], ) -def test_republication_detects_a_changed_enum_label_or_parameter_path( +def test_republication_detects_a_changed_enum_label_parameter_path_unit_or_period( publication_engine: Engine, variable_name: str, changes: dict[str, object], ) -> None: - catalog = _catalog_with_enum_and_parameter_parts() + catalog = _catalog_with_format_2_variable_records() publish_catalog(publication_engine, catalog) stored = _stored_us_variables(publication_engine) diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py index 48c85587f..36a1095e0 100644 --- a/tests/unit/v2/test_catalog_metadata_fields.py +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -106,7 +106,7 @@ def test_catalog_format_2_publishes_entities_and_basic_inputs() -> None: assert model_version.basic_inputs == ("employment_income",) -def test_variables_carry_input_flags_module_and_unset_parts() -> None: +def test_variables_carry_flags_module_unit_period_and_unset_parts() -> None: variable = normalized_catalog().country("us").variables[0] assert variable.name == "employment_income" diff --git a/tests/unit/v2/test_catalog_staging_rows.py b/tests/unit/v2/test_catalog_staging_rows.py index e5c86c3d6..e8b6f0bd4 100644 --- a/tests/unit/v2/test_catalog_staging_rows.py +++ b/tests/unit/v2/test_catalog_staging_rows.py @@ -159,8 +159,8 @@ def test_enum_values_are_staged_as_value_and_label_objects_in_order() -> None: (row,) = (_plain(row) for row in _catalog_rows(country)[STAGE_CATALOG_VARIABLES]) - enum_values_column = [column.name for column in STAGE_CATALOG_VARIABLES.columns] - assert row[enum_values_column.index("enum_values")] == ( + staged_column_names = [column.name for column in STAGE_CATALOG_VARIABLES.columns] + assert row[staged_column_names.index("enum_values")] == ( "jsonb", [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], ) From f803106ee3cf68983c64b1efec72fc6d0c761161 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:35:53 +0400 Subject: [PATCH 056/118] Check format 2's variable text fields against their column lengths Extraction wrote the length limits of the module name, the two parameter paths, the unit and the definition period a second time, beside the Variable model's max_length. It now reads each limit from the column that stores the field (VARIABLE_TEXT_LIMITS), so the two cannot drift apart, and a test feeds each field one character too many and checks that extraction stops with the field's name and limit. Co-Authored-By: Claude Opus 5.5 --- .../data/v2/catalog/extraction.py | 34 +++++++++++++--- tests/unit/v2/test_catalog_metadata_fields.py | 40 +++++++++++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 84606329f..4400c004a 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -39,6 +39,7 @@ filtered_artifacts, scope_regions, ) +from policyengine_api.data.v2.models import Variable if TYPE_CHECKING: from policyengine_api.json_types import JSONValue @@ -49,6 +50,29 @@ "policyengine-us", "policyengine-uk", ) + + +def _variable_column_length(column_name: str) -> int: + """Return the length limit of one text column of the v2 ``variables`` table.""" + length = getattr(Variable.__table__.c[column_name].type, "length", None) + if not isinstance(length, int): + msg = f"variables.{column_name} has no length limit" + raise TypeError(msg) + return length + + +# Catalog format 2's variable text fields, checked against the columns that +# store them, so the two limits cannot drift apart. +VARIABLE_TEXT_LIMITS: Mapping[str, int] = { + column_name: _variable_column_length(column_name) + for column_name in ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "unit", + "definition_period", + ) +} CURRENT_LAW_IDS = {"us": 2, "uk": 1} METADATA_TIME_PERIODS = { "us": tuple(range(2035, 2021, -1)), @@ -347,28 +371,28 @@ def optional_names(attribute: str) -> list[str] | None: module_name=_optional_text( getattr(source, "module_name", None), field_name=f"variable {name} module_name", - maximum=512, + maximum=VARIABLE_TEXT_LIMITS["module_name"], ), adds_parameter_path=_optional_text( getattr(source, "adds_parameter_path", None), field_name=f"variable {name} adds_parameter_path", - maximum=512, + maximum=VARIABLE_TEXT_LIMITS["adds_parameter_path"], ), subtracts_parameter_path=_optional_text( getattr(source, "subtracts_parameter_path", None), field_name=f"variable {name} subtracts_parameter_path", - maximum=512, + maximum=VARIABLE_TEXT_LIMITS["subtracts_parameter_path"], ), enum_values=enum_values, unit=_optional_text( getattr(source, "unit", None), field_name=f"variable {name} unit", - maximum=128, + maximum=VARIABLE_TEXT_LIMITS["unit"], ), definition_period=_optional_text( getattr(source, "definition_period", None), field_name=f"variable {name} definition_period", - maximum=32, + maximum=VARIABLE_TEXT_LIMITS["definition_period"], ), ) diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py index 36a1095e0..6b8cc3ddc 100644 --- a/tests/unit/v2/test_catalog_metadata_fields.py +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -448,3 +448,43 @@ def test_an_empty_basic_input_list_is_stored_empty() -> None: models["us"].basic_inputs = [] assert _extract(models).country("us").model_version.basic_inputs == () + + +FORMAT_2_VARIABLE_TEXT_FIELDS = ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "unit", + "definition_period", +) + + +def _variable_column_length(field_name: str) -> int: + from policyengine_api.data.v2.models import Variable + + return Variable.__table__.c[field_name].type.length + + +def test_variable_text_limits_are_the_variable_column_lengths() -> None: + """Extraction checks each format 2 text field against its column's length.""" + from policyengine_api.data.v2.catalog.extraction import VARIABLE_TEXT_LIMITS + + column_lengths = { + field_name: _variable_column_length(field_name) + for field_name in FORMAT_2_VARIABLE_TEXT_FIELDS + } + + assert column_lengths == VARIABLE_TEXT_LIMITS + + +@pytest.mark.parametrize("field_name", FORMAT_2_VARIABLE_TEXT_FIELDS) +def test_an_over_long_format_2_text_field_stops_extraction(field_name: str) -> None: + length = _variable_column_length(field_name) + models = source_models() + setattr(models["us"].variables[0], field_name, "x" * (length + 1)) + + with pytest.raises( + CatalogExtractionError, + match=rf"^variable employment_income {field_name} exceeds {length} characters$", + ): + _extract(models) From 816e00519953c5186636d44f1ef8cf69aa3c1d84 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:25:32 +0400 Subject: [PATCH 057/118] Serve v2 metadata responses with an ETag and a cache lifetime The metadata routers now use CacheableMetadataRoute, which gives every successful response a strong ETag (a hash of its exact body, so any content or format change gives a new one) and Cache-Control public: a day when the request names its catalog version, which never changes, and five minutes for the running version, which changes on deploy. A matching If-None-Match (strong or weak, alone or in a list) gets 304 with no body. Failures carry neither header, and the bodies are unchanged. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/migration/stage-9-v2-metadata.md | 12 +++ .../fastapi_routes/v2/metadata/common.py | 88 +++++++++++++++++- .../v2/metadata/geography_routes.py | 7 +- .../v2/metadata/model_routes.py | 7 +- .../v2/metadata/parameter_routes.py | 7 +- tests/unit/v2/test_metadata_routes.py | 92 +++++++++++++++++++ 7 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md new file mode 100644 index 000000000..e6b747a86 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -0,0 +1 @@ +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0ad3eb455..fc3e1fc5f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,18 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Response caching + +Every successful metadata response carries a strong `ETag`, a hash of its +exact body, and `Cache-Control: public`. A request that names its catalog with +`policyengine_version` may be reused for a day (`max-age=86400`): a published +catalog version never changes, so only a change to the route's response format +can change such a response, and the `ETag` lets a client revalidate after the +day. A request without it selects the running version, which changes on +deploy, and may be reused for five minutes (`max-age=300`). A request whose +`If-None-Match` names the current `ETag` (strong or weak form, alone or in a +list) gets HTTP 304 with no body. Error responses carry neither header. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index fd4eaab5d..9cd6ff688 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,13 @@ from __future__ import annotations import contextlib +import hashlib from typing import TYPE_CHECKING, Any, TypeVar +from fastapi.routing import APIRoute from pydantic import BaseModel +from starlette.responses import Response +from starlette.status import HTTP_200_OK from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, @@ -24,8 +28,9 @@ ) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Coroutine + from fastapi import Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.dependencies import ( @@ -62,6 +67,71 @@ ResponseT = TypeVar("ResponseT", bound=BaseModel) +ReadT = TypeVar("ReadT") + +EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 +"""How long a response for a named catalog version may be reused (a day). + +A published catalog version never changes, so such a response stays valid +until its code's response format changes; the ETag lets a client revalidate +it cheaply after that. +""" +RUNNING_VERSION_MAX_AGE_SECONDS = 300 +"""How long a response for the running catalog version may be reused. + +Without ``policyengine_version`` the running PolicyEngine.py version answers, +and it changes whenever the API deploys a new one. +""" + + +class CacheableMetadataRoute(APIRoute): + """A metadata route whose successful responses carry caching headers. + + See ``with_cache_headers``. Failures are served unchanged, without them. + """ + + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + """Wrap FastAPI's handler so a 200 response gains its caching headers.""" + handle = super().get_route_handler() + + async def handle_with_cache_headers(request: Request) -> Response: + response = await handle(request) + if response.status_code != HTTP_200_OK: + return response + return with_cache_headers(request, response) + + return handle_with_cache_headers + + +def with_cache_headers(request: Request, response: Response) -> Response: + """Give a JSON response a strong ETag and a public cache lifetime. + + The ETag is a hash of the exact body, so it changes with any change to the + content, including the catalog format. A request naming its catalog with + ``policyengine_version`` may be reused for a day, one that takes the + running version for five minutes. A request whose ``If-None-Match`` names + the ETag gets a 304 with no body. + """ + entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' + max_age = ( + EXPLICIT_VERSION_MAX_AGE_SECONDS + if "policyengine_version" in request.query_params + else RUNNING_VERSION_MAX_AGE_SECONDS + ) + headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} + if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): + return Response(status_code=304, headers=headers) + response.headers.update(headers) + return response + + +def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: + if if_none_match is None: + return False + candidates = {candidate.strip() for candidate in if_none_match.split(",")} + return "*" in candidates or bool( + {entity_tag, f"W/{entity_tag}"}.intersection(candidates) + ) def read_resource( @@ -75,6 +145,20 @@ def read_resource( (400, 404 or 503, and 500 for anything unexpected), and the reader is closed whatever happens. """ + return run_metadata_read( + dependencies, lambda reader: response_type(result=operation(reader)) + ) + + +def run_metadata_read( + dependencies: NativeRouteDependencies, + operation: Callable[[V2MetadataResourceReader], ReadT], +) -> ReadT | JSONResponse: + """Run one metadata read and return its result unchanged. + + Failures become the typed v2 error responses described by read_resource, + and the reader is closed whatever happens. + """ reader = None try: factory = dependencies.v2_metadata_reader_factory @@ -85,7 +169,7 @@ def read_resource( factory = _default_v2_metadata_reader_factory reader = factory() - return response_type(result=operation(reader)) + return operation(reader) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 17ff62e27..f5add9640 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -36,7 +37,11 @@ def build_v2_metadata_geography_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/datasets", diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 927b52536..cbb3a8ef5 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -37,7 +38,11 @@ def build_v2_metadata_model_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/tax-benefit-models", diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 0f4714e5c..5895c8564 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -11,6 +11,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( @@ -49,7 +50,11 @@ def build_v2_metadata_parameter_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/parameters", diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..8f2084547 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -583,3 +583,95 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "application/json" ]["schema"] assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" + + +def _every_resource_path(ids: dict[str, object]) -> list[str]: + return [ + "/v2/tax-benefit-models?country_id=us", + "/v2/tax-benefit-models/by-country/us", + f"/v2/tax-benefit-models/{ids['model_id']}?country_id=us", + "/v2/tax-benefit-model-versions?country_id=us", + f"/v2/tax-benefit-model-versions/{ids['model_version_id']}?country_id=us", + "/v2/variables?country_id=us", + f"/v2/variables/{ids['variable_id']}?country_id=us", + "/v2/parameters?country_id=us", + "/v2/parameters/children?country_id=us&parent_path=gov.example", + f"/v2/parameters/{ids['parameter_id']}?country_id=us", + "/v2/parameter-values?country_id=us", + f"/v2/parameter-values/{ids['parameter_value_id']}?country_id=us", + "/v2/datasets?country_id=us", + f"/v2/datasets/{ids['dataset_id']}?country_id=us", + "/v2/regions?country_id=us", + "/v2/regions/by-code/us?country_id=us", + f"/v2/regions/{ids['region_id']}?country_id=us", + "/v2/economy-options?country_id=us", + ] + + +def test_every_resource_route_serves_an_entity_tag_and_cache_lifetime() -> None: + results, ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + + for path in _every_resource_path(ids): + response = client.get(path) + assert response.status_code == 200, path + assert response.headers["cache-control"] == "public, max-age=300", path + assert response.headers["etag"].startswith('"'), path + assert response.headers["etag"].endswith('"'), path + + +def test_a_named_catalog_version_is_reusable_for_a_day() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + query = {"country_id": "us", "policyengine_version": "4.20.3"} + + first = client.get("/v2/variables", params=query) + second = client.get("/v2/variables", params=query) + + assert first.headers["cache-control"] == "public, max-age=86400" + assert first.headers["etag"] == second.headers["etag"] + + +def test_a_matching_entity_tag_gets_a_not_modified_response_without_a_body() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + path = "/v2/variables?country_id=us&policyengine_version=4.20.3" + entity_tag = client.get(path).headers["etag"] + + revalidated = client.get(path, headers={"If-None-Match": entity_tag}) + weakly_revalidated = client.get(path, headers={"If-None-Match": f"W/{entity_tag}"}) + listed = client.get(path, headers={"If-None-Match": f'"other", {entity_tag}'}) + stale = client.get(path, headers={"If-None-Match": '"other"'}) + + for not_modified in (revalidated, weakly_revalidated, listed): + assert not_modified.status_code == 304 + assert not_modified.content == b"" + assert not_modified.headers["etag"] == entity_tag + assert not_modified.headers["cache-control"] == "public, max-age=86400" + assert stale.status_code == 200 + assert stale.json()["status"] == "ok" + + +def test_different_content_gets_a_different_entity_tag() -> None: + results, _ids = _resource_results() + changed_results, _changed_ids = _resource_results() + + first = _client(lambda: ResourceReader(results)).get("/v2/variables?country_id=us") + changed = _client(lambda: ResourceReader(changed_results)).get( + "/v2/variables?country_id=us" + ) + + assert first.headers["etag"] != changed.headers["etag"] + + +def test_failed_reads_carry_no_cache_headers() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get( + "/v2/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert response.status_code == 404 + assert "etag" not in response.headers + assert "cache-control" not in response.headers From 7af658237d5027ad8beee1727172ec42fc3c0966 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:33:08 +0400 Subject: [PATCH 058/118] Serve the small parts of a catalog version at /v2/model-catalog GET /v2/model-catalog returns one catalog version's model and model version, entities, basic inputs, every region and logical input dataset, and the economy choices, in three queries and without reading variables, parameters or their values. A catalog without regions, such as a household-only country's, gets empty region, dataset and choice lists rather than an error; a version published before catalog format 2 has a null catalog_format and empty entities and basic inputs. The result names the selected PolicyEngine.py version so a client can ask for the other parts by that exact version. The query uses the shared CatalogQuery model, the route joins the metadata route group and the app's metadata workflow contract, and the migration contract documents are regenerated. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 5 +- docs/generated/migration_contracts.json | 21 ++- docs/migration/stage-9-v2-metadata.md | 15 ++ .../fastapi_routes/dependencies.py | 15 ++ .../v2/metadata/catalog_routes.py | 67 +++++++++ .../v2/metadata/response_models.py | 7 + policyengine_api/fastapi_routes/v2/routes.py | 4 + policyengine_api/migration_registry.py | 1 + .../database_connectors/reads_datasets.py | 14 ++ .../services/v2/metadata/services.py | 50 +++++++ .../services/v2/metadata/transformations.py | 52 +++++++ .../services/v2/metadata/types.py | 45 ++++++ tests/contract/registry.py | 17 +++ tests/integration/test_v2_metadata_routes.py | 50 +++++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 104 +++++++++++++ tests/unit/v2/test_metadata_service.py | 140 +++++++++++++++++- 18 files changed, 605 insertions(+), 5 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index e6b747a86..e1d3d7b8b 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1 +1,2 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. +GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..c9fcbd3a3 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 89 | +| request count | 90 | | db entity count | 6 | | sim flow count | 3 | @@ -18,7 +18,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | @@ -215,6 +215,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/regions/by-code/state/ca?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.item` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..d1550bdc5 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 90, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -33,6 +33,7 @@ "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", @@ -1614,6 +1615,24 @@ "result.time_period", "result.datasets" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index fc3e1fc5f..cc2ae7b25 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,21 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Loading a catalog version on demand + +`GET /v2/model-catalog?country_id=[&policyengine_version=]` +returns only the small parts of one catalog version: the model and model +version (with the current-law policy id and the years offered), the entities +(`key`, `plural`, `is_person`), the basic inputs, every region and logical +input dataset, and the economy choices. A catalog without regions, such as a +household-only country's, has empty region, dataset and economy-choice lists +rather than an error. A version published before catalog format 2 has a null +`catalog_format` and empty entities and basic inputs. The result names the +selected PolicyEngine.py version, so a client that starts without a version +learns the running one and requests every other part for that exact version. +The query uses the shared `CatalogQuery` model: unknown and repeated +parameters are rejected. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..59745fec8 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -38,6 +38,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -311,6 +312,20 @@ def get_economy_options( """ ... + def get_model_catalog( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + These are the model and model version, entities, basic inputs, regions, + logical input datasets and economy choices, which are empty for a + catalog without regions. Raise ``MetadataCatalogUnavailableError`` when + a catalog with regions lacks what ``get_economy_options`` needs. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py new file mode 100644 index 000000000..2c73c44d1 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -0,0 +1,67 @@ +"""API v2 routes that serve a catalog version in parts loaded on demand.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import APIRouter, Depends +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.metadata.common import ( + ERROR_RESPONSES, + CacheableMetadataRoute, + read_resource, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataModelCatalogResponse, +) +from policyengine_api.query_parameters import CatalogQuery + +if TYPE_CHECKING: + from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] + + +def build_v2_model_catalog_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the routes that serve one catalog version in parts. + + Each request opens and closes its own metadata reader, so building the + router touches no database. + """ + router = APIRouter( + prefix="/v2", + responses=ERROR_RESPONSES, + route_class=CacheableMetadataRoute, + ) + + @router.get( + "/model-catalog", + response_model=MetadataModelCatalogResponse, + summary="Get the small parts of one PolicyEngine.py catalog version", + description=( + "Returns the selected catalog version's model, model version, " + "entities, basic inputs, regions, logical input datasets and " + "economy choices, which are empty lists for a catalog without " + "regions. The result names the selected PolicyEngine.py version, " + "so a client can request the version's other parts by that exact " + "version; a response for a named version may be reused for a day." + ), + ) + def get_model_catalog( + query: CatalogQueryDependency, + ) -> MetadataModelCatalogResponse | JSONResponse: + return read_resource( + dependencies, + MetadataModelCatalogResponse, + lambda reader: reader.get_model_catalog( + query.country_id, query.policyengine_version + ), + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index d53660834..be095bb8a 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -10,6 +10,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -139,3 +140,9 @@ class MetadataEconomyOptionsResponse( MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] ): """Success response holding the options for configuring an economy comparison.""" + + +class MetadataModelCatalogResponse( + MetadataResourceSuccessResponse[MetadataModelCatalogResult] +): + """Success response holding the small parts of one catalog version.""" diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..b55b57436 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -12,6 +12,9 @@ build_v2_household_router, ) from policyengine_api.fastapi_routes.v2.me.routes import build_v2_me_router +from policyengine_api.fastapi_routes.v2.metadata.catalog_routes import ( + build_v2_model_catalog_router, +) from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) @@ -56,6 +59,7 @@ def build_v2_router( router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) + router.include_router(build_v2_model_catalog_router(dependencies)) @router.get( "/v2/openapi.json", diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..25fd541f6 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -39,6 +39,7 @@ class RouteGroupConfig: "metadata", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py index cd5dbc276..b6d79b6fa 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -48,3 +48,17 @@ def read_dataset( ), ) return rows[0] if rows else None + + +def read_input_datasets(session: Session, *, model_version_id: UUID) -> list[Dataset]: + """Return every logical input dataset of a model version in name order.""" + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)), + ) diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 938eb660e..199b12683 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -14,6 +14,7 @@ from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( read_dataset, read_datasets, + read_input_datasets, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, @@ -37,6 +38,7 @@ metadata_dataset, metadata_economy_options, metadata_model, + metadata_model_catalog, metadata_model_selection, metadata_model_version, metadata_parameter, @@ -53,6 +55,7 @@ MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -541,3 +544,50 @@ def get_economy_options( return metadata_economy_options( selected, regions=regions, national_dataset=dataset ) + + def get_model_catalog( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + They are the model and model version, entities, basic inputs, every + region and logical input dataset, and the economy choices. A catalog + without regions offers no economy calculation, so its choices are + empty; one with regions raises MetadataCatalogUnavailableError when it + lacks the national region, its default dataset or the model-version + options, as get_economy_options does. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + datasets = read_input_datasets( + session, model_version_id=selected.model_version.id + ) + national_dataset = None + if regions: + national_region = next( + (region for region in regions if region.code == country_id), None + ) + national_dataset = validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + next( + ( + dataset + for dataset in datasets + if dataset.id == national_region.default_dataset_id + ), + None, + ) + if national_region is not None + else None + ), + ) + return metadata_model_catalog( + selected, + regions=regions, + datasets=datasets, + national_dataset=national_dataset, + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 68ebe21fe..ce8eb7bd2 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -10,8 +10,11 @@ MetadataCanonicalParameterValue, MetadataDataset, MetadataDatasetOption, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -221,9 +224,22 @@ def metadata_economy_options( The national dataset is the only dataset choice, and it is the default. """ + options = _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) return MetadataEconomyOptionsResult( policyengine_version=selected.policyengine_version, current_law_id=selected.model_version.current_law_id, + region=options.region, + time_period=options.time_period, + datasets=options.datasets, + ) + + +def _economy_option_lists( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptions: + return MetadataEconomyOptions( region=[ MetadataRegionOption( name=region.code, @@ -243,3 +259,39 @@ def metadata_economy_options( ) ], ) + + +def metadata_model_catalog( + selected: SelectedCatalog, + *, + regions: list[Region], + datasets: list[Dataset], + national_dataset: Dataset | None, +) -> MetadataModelCatalogResult: + """Build the small parts of the selected catalog version. + + Without a national dataset (a catalog without regions) the economy choices + are empty. Entities and basic inputs a version published before catalog + format 2 lacks are empty lists. + """ + model_version = selected.model_version + return MetadataModelCatalogResult( + policyengine_version=selected.policyengine_version, + catalog_format=model_version.catalog_format, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + entities=[ + MetadataEntity.model_validate(entity) + for entity in model_version.entities or [] + ], + basic_inputs=list(model_version.basic_inputs or []), + regions=[metadata_region(region) for region in regions], + datasets=[metadata_dataset(dataset) for dataset in datasets], + economy_options=( + _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) + if national_dataset is not None + else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) + ), + ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index a7336d7e9..e810a663b 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -239,3 +239,48 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] + + +class MetadataEntity(StrictResponseModel): + """A kind of member a household holds, such as a person or a tax unit. + + plural is the key the household's members of this kind are listed under, + and is_person marks the entity every household input about a person uses. + """ + + key: str + plural: str + is_person: bool + + +class MetadataEconomyOptions(StrictResponseModel): + """The region, year and dataset choices for an economy calculation. + + Every list is empty for a catalog without regions, such as a + household-only country's, which offers no economy calculation. + """ + + region: list[MetadataRegionOption] + time_period: list[MetadataTimePeriodOption] + datasets: list[MetadataDatasetOption] + + +class MetadataModelCatalogResult(StrictResponseModel): + """The small parts of one catalog version, read when a client starts. + + policyengine_version names the selected catalog version, so a client can + request its other parts for that exact version. catalog_format is null, + and entities and basic_inputs are empty, for a version published before + catalog format 2. Variables, parameters and their values are served by + their own routes. + """ + + policyengine_version: str + catalog_format: int | None + model: MetadataModel + model_version: MetadataModelVersionDetail + entities: list[MetadataEntity] + basic_inputs: list[str] + regions: list[MetadataRegion] + datasets: list[MetadataDataset] + economy_options: MetadataEconomyOptions diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..c1fbe02e8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -671,6 +671,23 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 326eb78d9..efeb702b5 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -360,3 +360,53 @@ def test_postgres_parameter_collection_remains_available_without_values( assert parameters.status_code == values.status_code == 200 assert parameters.json()["result"]["items"] assert values.json()["result"]["items"] == [] + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_postgres_model_catalog_serves_the_small_parts_of_a_version( + published_engine: Engine, + country_id: str, +) -> None: + client = _client(published_engine) + country = normalized_catalog().country(country_id) + + running = client.get("/v2/model-catalog", params={"country_id": country_id}) + named = client.get( + "/v2/model-catalog", + params={"country_id": country_id, "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert running.status_code == 200 + assert running.headers["cache-control"] == "public, max-age=300" + assert named.headers["cache-control"] == "public, max-age=86400" + assert running.headers["etag"] == named.headers["etag"] + result = running.json()["result"] + assert result["policyengine_version"] == POLICYENGINE_VERSION + assert result["catalog_format"] == country.model_version.catalog_format + assert result["entities"] == [ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ] + assert result["basic_inputs"] == list(country.model_version.basic_inputs) + assert [region["code"] for region in result["regions"]] == sorted( + region.code for region in country.regions + ) + assert [dataset["name"] for dataset in result["datasets"]] == sorted( + dataset.name for dataset in country.datasets + ) + assert [option["name"] for option in result["economy_options"]["region"]] == [ + region["code"] for region in result["regions"] + ] + assert "variables" not in result + assert "parameters" not in result + + revalidated = client.get( + "/v2/model-catalog", + params={"country_id": country_id}, + headers={"If-None-Match": running.headers["etag"]}, + ) + assert revalidated.status_code == 304 diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..4125c6dcb 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 89, + "request_count": 90, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 8f2084547..81ff8669e 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -29,8 +29,11 @@ MetadataDataset, MetadataDatasetOption, MetadataDetailResult, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -193,6 +196,31 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_model_catalog": MetadataModelCatalogResult( + policyengine_version=version, + catalog_format=2, + model=model, + model_version=model_version, + entities=[ + MetadataEntity(key="person", plural="people", is_person=True) + ], + basic_inputs=["employment_income"], + regions=[region], + datasets=[dataset], + economy_options=MetadataEconomyOptions( + region=[ + MetadataRegionOption( + name="us", label="United States", type="national" + ) + ], + time_period=[MetadataTimePeriodOption(name=2026, label="2026")], + datasets=[ + MetadataDatasetOption( + name="populace_us_2024", label="Microcosm" + ) + ], + ), + ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, current_law_id=2, @@ -521,6 +549,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", + "/v2/model-catalog", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -605,6 +634,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: "/v2/regions/by-code/us?country_id=us", f"/v2/regions/{ids['region_id']}?country_id=us", "/v2/economy-options?country_id=us", + "/v2/model-catalog?country_id=us", ] @@ -675,3 +705,77 @@ def test_failed_reads_carry_no_cache_headers() -> None: assert response.status_code == 404 assert "etag" not in response.headers assert "cache-control" not in response.headers + + +def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", params={"country_id": "US"} + ) + + assert response.status_code == 200 + result = response.json()["result"] + assert result["policyengine_version"] == "4.20.3" + assert result["catalog_format"] == 2 + assert result["entities"] == [ + {"key": "person", "plural": "people", "is_person": True} + ] + assert result["basic_inputs"] == ["employment_income"] + assert [region["code"] for region in result["regions"]] == ["us"] + assert result["economy_options"]["datasets"][0]["default"] is True + assert reader.calls == [("get_model_catalog", ("us", None), {})] + assert reader.closed + + +def test_model_catalog_route_names_an_explicit_version_for_a_day() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", + params={"country_id": "uk", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + assert reader.calls == [("get_model_catalog", ("uk", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + [ + "", + "country_id=ca", + "country_id=us&country_id=uk", + "country_id=us&limit=10", + "country_id=us&policyengine_version=latest", + ], +) +def test_model_catalog_route_rejects_invalid_queries(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog?{query}") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert calls == [] + + +def test_model_catalog_route_declares_its_query_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version"} + assert parameters["country_id"]["required"] is True + assert parameters["policyengine_version"]["required"] is False + success = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 1f80d1b3f..996ba1ef5 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -34,7 +34,10 @@ MetadataDatabaseSession, ) from policyengine_api.services.v2.metadata.services import V2MetadataService -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.types import ( + MetadataEntity, + ParameterValueFilter, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -74,6 +77,16 @@ def _insert_country( if time_periods is None else time_periods ), + catalog_format=country.model_version.catalog_format, + entities=[ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ], + basic_inputs=list(country.model_version.basic_inputs), ) ) session.add_all( @@ -89,6 +102,19 @@ def _insert_country( default_value=record.default_value, adds=record.adds, subtracts=record.subtracts, + is_input_variable=record.is_input_variable, + hidden_input=record.hidden_input, + module_name=record.module_name, + adds_parameter_path=record.adds_parameter_path, + subtracts_parameter_path=record.subtracts_parameter_path, + enum_values=( + None + if record.enum_values is None + else [ + {"value": enum_value.value, "label": enum_value.label} + for enum_value in record.enum_values + ] + ), ) for record in country.variables ) @@ -111,6 +137,8 @@ def _insert_country( description=record.description, data_type=record.data_type, unit=record.unit, + economy=record.economy, + household=record.household, ) for record in country.parameters ) @@ -560,6 +588,115 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") +def _recorded_statements(session: Session, read): + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement.lower()) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record_statement) + try: + result = read() + finally: + event.remove(bind, "before_cursor_execute", record_statement) + return result, statements + + +def test_model_catalog_holds_only_the_small_parts_of_a_catalog_version( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, lambda: _service(catalog_session).get_model_catalog("us") + ) + us_catalog = normalized_catalog().country("us") + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert result.model.name == "policyengine-us" + assert ( + result.model_version.current_law_id == us_catalog.model_version.current_law_id + ) + assert result.entities == [ + MetadataEntity(key="person", plural="people", is_person=True), + MetadataEntity(key="household", plural="households", is_person=False), + ] + assert result.basic_inputs == ["employment_income"] + assert [region.code for region in result.regions] == sorted( + region.code for region in us_catalog.regions + ) + assert [dataset.name for dataset in result.datasets] == sorted( + dataset.name for dataset in us_catalog.datasets + ) + assert [option.name for option in result.economy_options.region] == [ + region.code for region in result.regions + ] + assert [option.name for option in result.economy_options.time_period] == list( + us_catalog.model_version.metadata_time_periods + ) + assert [option.label for option in result.economy_options.datasets] == ["Microcosm"] + assert len(statements) == 3 + for table in ("variables", "parameters", "parameter_values", "parameter_nodes"): + assert all(f"from {table}" not in statement for statement in statements) + + +def test_model_catalog_without_regions_offers_no_economy_calculation( + catalog_session: Session, +) -> None: + # A household-only country publishes no regions or datasets; its catalog + # still serves entities and basic inputs, with empty economy choices. + model_version = _us_model_version(catalog_session) + for region in catalog_session.exec( + select(Region).where(Region.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(region) + catalog_session.flush() + for dataset in catalog_session.exec( + select(Dataset).where(Dataset.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(dataset) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.regions == [] + assert result.datasets == [] + assert result.economy_options.region == [] + assert result.economy_options.time_period == [] + assert result.economy_options.datasets == [] + assert result.basic_inputs == ["employment_income"] + + +def test_model_catalog_published_before_format_2_has_empty_new_fields( + catalog_session: Session, +) -> None: + model_version = _us_model_version(catalog_session) + model_version.catalog_format = None + model_version.entities = None + model_version.basic_inputs = None + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.catalog_format is None + assert result.entities == [] + assert result.basic_inputs == [] + + +def test_model_catalog_with_regions_requires_the_national_region( + catalog_session: Session, +) -> None: + national_region = catalog_session.exec( + select(Region).where(Region.code == "us") + ).one() + catalog_session.delete(national_region) + catalog_session.commit() + + with pytest.raises(MetadataCatalogUnavailableError, match="national v2 region"): + _service(catalog_session).get_model_catalog("us") + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -624,6 +761,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region", "get_region_by_code", "get_economy_options", + "get_model_catalog", } for method_name in method_names: From 290b042df403cf0c2395b8392f1f7de7cea6014e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:40:25 +0400 Subject: [PATCH 059/118] Serve every variable of a catalog version with its part parameters' values GET /v2/model-catalog/variables returns all of a catalog version's variables in name order, in one response, and adds the catalog format 2 fields (input and hidden-input flags, module, adds and subtracts parameter paths, and enum values with their labels) to every variable response, including /v2/variables pages. part_parameter_values maps each parameter path a variable's adds or subtracts names to that parameter's current-law values in start-date order, read in one query (none when no variable names one), so a client picks the list for the year it shows; a path the version lacks maps to an empty list. Times are served in UTC whatever the database session's time zone, so a response's bytes and ETag do not depend on it. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 3 +- docs/generated/migration_contracts.json | 17 +- docs/migration/stage-9-v2-metadata.md | 12 ++ .../fastapi_routes/dependencies.py | 13 ++ .../v2/metadata/catalog_routes.py | 24 +++ .../v2/metadata/response_models.py | 7 + .../database_connectors/reads_parameters.py | 33 +++- .../database_connectors/reads_variables.py | 10 ++ .../services/v2/metadata/services.py | 27 ++++ .../services/v2/metadata/transformations.py | 66 ++++++++ .../services/v2/metadata/types.py | 50 +++++- tests/contract/registry.py | 14 ++ tests/integration/test_v2_metadata_routes.py | 98 ++++++++++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 62 ++++++++ tests/unit/v2/test_metadata_service.py | 147 ++++++++++++++++++ 17 files changed, 581 insertions(+), 5 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index e1d3d7b8b..5e897c7ed 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,2 +1,3 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index c9fcbd3a3..33212c71d 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 90 | +| request count | 91 | | db entity count | 6 | | sim flow count | 3 | @@ -216,6 +216,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | | `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | +| `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index d1550bdc5..74592e7fc 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 90, + "request_count": 91, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -1633,6 +1633,21 @@ "result.datasets", "result.economy_options" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/variables?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index cc2ae7b25..ed8456fc4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -42,6 +42,18 @@ learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated parameters are rejected. +`GET /v2/model-catalog/variables` (same query) returns every variable of the +version in name order, in one response, with the catalog format 2 fields: +`is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, +`subtracts_parameter_path` and `enum_values` (each value's name and its label, +in the type the country package declares). Where a variable's `adds` or +`subtracts` list comes from a parameter, `part_parameter_values` maps that +parameter's path to its current-law values in start-date order +(`start_date`, `end_date`, `value`), so a client picks the list for the year +it shows instead of the list resolved on 2025-01-01; a path whose parameter +the version lacks maps to an empty list. `GET /v2/variables` pages carry the +same variable fields. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 59745fec8..7de4f62e0 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -34,6 +34,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -326,6 +327,18 @@ def get_model_catalog( """ ... + def get_catalog_variables( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also carries the current-law values of each parameter a variable's + adds or subtracts names. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 2c73c44d1..a648ece21 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -14,6 +14,7 @@ read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, ) from policyengine_api.query_parameters import CatalogQuery @@ -64,4 +65,27 @@ def get_model_catalog( ), ) + @router.get( + "/model-catalog/variables", + response_model=MetadataCatalogVariablesResponse, + summary="Get every variable of one PolicyEngine.py catalog version", + description=( + "Returns all of the selected catalog version's variables in name " + "order, in one response. Where a variable's adds or subtracts list " + "comes from a parameter, part_parameter_values carries that " + "parameter's current-law values by date, so a client can pick the " + "list for any year." + ), + ) + def get_catalog_variables( + query: CatalogQueryDependency, + ) -> MetadataCatalogVariablesResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogVariablesResponse, + lambda reader: reader.get_catalog_variables( + query.country_id, query.policyengine_version + ), + ) + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index be095bb8a..56791d2bb 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,6 +6,7 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -146,3 +147,9 @@ class MetadataModelCatalogResponse( MetadataResourceSuccessResponse[MetadataModelCatalogResult] ): """Success response holding the small parts of one catalog version.""" + + +class MetadataCatalogVariablesResponse( + MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] +): + """Success response holding every variable of one catalog version.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 02062798a..739b6b0cb 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import sqlalchemy as sa from sqlmodel import Session, col, select @@ -11,6 +11,7 @@ from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows if TYPE_CHECKING: + from collections.abc import Collection from uuid import UUID from policyengine_api.services.v2.metadata.types import ParameterValueSelection @@ -131,3 +132,33 @@ def read_parameter_value( ), ) return rows[0] if rows else None + + +def read_current_law_values_by_parameter_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Any]: + """Return the current-law values of the named parameters of a model version. + + Each row has the parameter's name, the value's value_json, start_date and + end_date. Values tied to a policy or dynamic are excluded. Rows are ordered + by parameter name, then start date. + """ + if not parameter_names: + return [] + return read_rows( + session, + select( + col(Parameter.name), + col(ParameterValue.value_json), + col(ParameterValue.start_date), + col(ParameterValue.end_date), + ) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + .order_by(col(Parameter.name), col(ParameterValue.start_date)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py index 4d84be396..1378f6528 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -62,3 +62,13 @@ def read_variable( ), ) return rows[0] if rows else None + + +def read_all_variables(session: Session, *, model_version_id: UUID) -> list[Variable]: + """Return every variable of a model version in name order.""" + return read_rows( + session, + select(Variable) + .where(col(Variable.tax_benefit_model_version_id) == model_version_id) + .order_by(col(Variable.name)), + ) diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 199b12683..e6d4506c7 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -20,6 +20,7 @@ read_parameter_children, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_current_law_values_by_parameter_name, read_parameter, read_parameter_value, read_parameter_values, @@ -31,10 +32,12 @@ read_regions, ) from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_all_variables, read_variable, read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + metadata_catalog_variables, metadata_dataset, metadata_economy_options, metadata_model, @@ -48,9 +51,11 @@ metadata_variable, page_result, parameter_value_selection, + part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, @@ -591,3 +596,25 @@ def get_model_catalog( datasets=datasets, national_dataset=national_dataset, ) + + def get_catalog_variables( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also returns the current-law values of each parameter a variable's + adds or subtracts names, so a client can pick the list for any year. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + variables = read_all_variables( + session, model_version_id=selected.model_version.id + ) + part_values = read_current_law_values_by_parameter_name( + session, + model_version_id=selected.model_version.id, + parameter_names=part_parameter_paths(variables), + ) + return metadata_catalog_variables( + selected, variables=variables, part_values=part_values + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index ce8eb7bd2..651f26414 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,11 +8,14 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataEconomyOptions, MetadataEconomyOptionsResult, MetadataEntity, + MetadataEnumValue, MetadataModel, MetadataModelCatalogResult, MetadataModelSelectionResult, @@ -106,6 +109,57 @@ def metadata_variable(variable: Variable) -> MetadataVariable: default_value=variable.default_value, adds=variable.adds, subtracts=variable.subtracts, + is_input_variable=variable.is_input_variable, + hidden_input=variable.hidden_input, + module_name=variable.module_name, + adds_parameter_path=variable.adds_parameter_path, + subtracts_parameter_path=variable.subtracts_parameter_path, + enum_values=( + None + if variable.enum_values is None + else [ + MetadataEnumValue.model_validate(enum_value) + for enum_value in variable.enum_values + ] + ), + ) + + +def part_parameter_paths(variables: list[Variable]) -> set[str]: + """Return every parameter path the variables' adds or subtracts names.""" + return { + path + for variable in variables + for path in (variable.adds_parameter_path, variable.subtracts_parameter_path) + if path is not None + } + + +def metadata_catalog_variables( + selected: SelectedCatalog, *, variables: list[Variable], part_values: list[Any] +) -> MetadataCatalogVariablesResult: + """Build every variable of the selected version and its part parameters' values. + + part_values holds rows of read_current_law_values_by_parameter_name, in + parameter-name and start-date order. Each path a variable names gets an + entry, empty when no row carries its name. + """ + values_by_path: dict[str, list[MetadataDatedValue]] = { + path: [] for path in sorted(part_parameter_paths(variables)) + } + for row in part_values: + values_by_path.setdefault(row.name, []).append( + MetadataDatedValue( + start_date=as_utc(row.start_date), + end_date=None if row.end_date is None else as_utc(row.end_date), + value=row.value_json, + ) + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, ) @@ -132,6 +186,18 @@ def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParamete ) +def as_utc(moment: datetime) -> datetime: + """Return moment in UTC, treating a naive time as UTC. + + The database may return times in its session's time zone (or, in SQLite, + without one); serving them in UTC keeps a response's bytes, and so its + ETag, the same whatever that setting is. + """ + if moment.tzinfo is None: + return moment.replace(tzinfo=UTC) + return moment.astimezone(UTC) + + def utc_day_start(selected_time: datetime) -> datetime: """Return midnight UTC on the day of selected_time, treating a naive time as UTC.""" if selected_time.tzinfo is None: diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index e810a663b..d6633eeaf 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -71,11 +71,26 @@ class MetadataModelVersion(StrictResponseModel): description: str | None +class MetadataEnumValue(StrictResponseModel): + """One value of an enum variable: its stored name and its display label. + + The label keeps the type the country package declares it with. + """ + + value: str + label: str | int | float | bool | None + + class MetadataVariable(StrictResponseModel): """A variable the tax-benefit model computes or accepts as input. adds and subtracts, when present, list the variables summed into or - subtracted from this one. + subtracted from this one. When the model takes that list from a + parameter, adds_parameter_path or subtracts_parameter_path names the + parameter, whose dated values give the list for each year (adds and + subtracts hold its value on 2025-01-01). The catalog format 2 fields + (is_input_variable, hidden_input, module_name, the two parameter paths + and enum_values) are null in versions published before that format. """ id: UUID @@ -88,6 +103,12 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None = None + hidden_input: bool | None = None + module_name: str | None = None + adds_parameter_path: str | None = None + subtracts_parameter_path: str | None = None + enum_values: list[MetadataEnumValue] | None = None class MetadataParameterSummary(StrictResponseModel): @@ -284,3 +305,30 @@ class MetadataModelCatalogResult(StrictResponseModel): regions: list[MetadataRegion] datasets: list[MetadataDataset] economy_options: MetadataEconomyOptions + + +class MetadataDatedValue(StrictResponseModel): + """A current-law parameter value from start_date until end_date. + + A null end_date means the value applies until a later value starts, or + indefinitely. + """ + + start_date: datetime + end_date: datetime | None + value: JsonValue + + +class MetadataCatalogVariablesResult(StrictResponseModel): + """Every variable of one catalog version, in name order. + + part_parameter_values has an entry for each parameter path a variable's + adds_parameter_path or subtracts_parameter_path names: that parameter's + current-law values in start-date order, or an empty list when the + catalog has no such parameter. + """ + + policyengine_version: str + catalog_format: int | None + variables: list[MetadataVariable] + part_parameter_values: dict[str, list[MetadataDatedValue]] diff --git a/tests/contract/registry.py b/tests/contract/registry.py index c1fbe02e8..b126a838e 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -688,6 +688,20 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog/variables?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index efeb702b5..68a44ebda 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 @@ -20,8 +21,11 @@ from policyengine_api.data.v2.catalog.publication import publish_catalog from policyengine_api.data.v2.models import ( Dataset, + Parameter, + ParameterValue, TaxBenefitModel, TaxBenefitModelVersion, + Variable, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -410,3 +414,97 @@ def test_postgres_model_catalog_serves_the_small_parts_of_a_version( headers={"If-None-Match": running.headers["etag"]}, ) assert revalidated.status_code == 304 + + +def _us_model_version_id(engine: Engine): + with Session(engine) as session: + return session.exec( + select(TaxBenefitModelVersion.id) + .join( + TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id + ) + .where(TaxBenefitModel.name == "policyengine-us") + ).one() + + +def test_postgres_catalog_variables_carry_part_parameter_values_by_date( + published_engine: Engine, +) -> None: + model_version_id = _us_model_version_id(published_engine) + parts_id = uuid4() + with Session(published_engine) as session: + session.add( + Parameter( + id=parts_id, + tax_benefit_model_version_id=model_version_id, + name="gov.example.parts", + label="Income parts", + economy=False, + household=True, + ) + ) + session.flush() + session.add_all( + [ + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + ), + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + ), + Variable( + tax_benefit_model_version_id=model_version_id, + name="total_income", + entity="person", + data_type="float", + default_value=0, + adds=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + ), + ] + ) + session.commit() + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income", + "total_income", + ] + assert result["variables"][0]["module_name"] == ( + "household.income.employment_income" + ) + assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2024-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + }, + { + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income", "pension_income"], + }, + ] + } + uk = client.get("/v2/model-catalog/variables", params={"country_id": "uk"}) + assert [variable["name"] for variable in uk.json()["result"]["variables"]] == [ + "employment_income" + ] + assert uk.json()["result"]["part_parameter_values"] == {} diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 4125c6dcb..a52730e6e 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 90, + "request_count": 91, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 81ff8669e..e522a114b 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -26,8 +26,10 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataDetailResult, MetadataEconomyOptions, MetadataEconomyOptionsResult, @@ -196,6 +198,20 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_variables": MetadataCatalogVariablesResult( + policyengine_version=version, + catalog_format=2, + variables=[variable], + part_parameter_values={ + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income"], + ) + ] + }, + ), "get_model_catalog": MetadataModelCatalogResult( policyengine_version=version, catalog_format=2, @@ -550,6 +566,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets/{dataset_id}", "/v2/economy-options", "/v2/model-catalog", + "/v2/model-catalog/variables", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -635,6 +652,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: f"/v2/regions/{ids['region_id']}?country_id=us", "/v2/economy-options?country_id=us", "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", ] @@ -779,3 +797,47 @@ def test_model_catalog_route_declares_its_query_in_openapi() -> None: assert parameters["policyengine_version"]["required"] is False success = operation["responses"]["200"]["content"]["application/json"]["schema"] assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} + + +def test_catalog_variables_route_serves_every_variable_and_part_values() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["catalog_format"] == 2 + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income" + ] + assert result["variables"][0]["module_name"] is None + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + } + ] + } + assert reader.calls == [("get_catalog_variables", ("us", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + ["", "country_id=us&offset=0", "country_id=us&search=income"], +) +def test_catalog_variables_route_rejects_paging_and_unknown_queries( + query: str, +) -> None: + results, _ids = _resource_results() + response = _client(lambda: ResourceReader(results)).get( + f"/v2/model-catalog/variables?{query}" + ) + + assert response.status_code == 422 diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 996ba1ef5..d30ba98b8 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -35,7 +35,9 @@ ) from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.types import ( + MetadataDatedValue, MetadataEntity, + MetadataEnumValue, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -697,6 +699,150 @@ def test_model_catalog_with_regions_requires_the_national_region( _service(catalog_session).get_model_catalog("us") +def _add_variable_with_parameter_named_parts(session: Session) -> None: + model_version = _us_model_version(session) + parts = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.parts", + label="Income parts", + description=None, + data_type="list", + unit=None, + economy=False, + household=True, + ) + session.add(parts) + session.add_all( + [ + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["reform_only_income"], + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ), + ] + ) + session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="total_income", + label="Total income", + entity="person", + description=None, + data_type="float", + possible_values=["LOW", "HIGH"], + default_value=0, + adds=["employment_income"], + subtracts=None, + is_input_variable=False, + hidden_input=True, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + subtracts_parameter_path="gov.example.unpublished_parts", + enum_values=[ + {"value": "LOW", "label": "Low"}, + {"value": "HIGH", "label": 2}, + ], + ) + ) + session.commit() + + +def test_catalog_variables_hold_every_variable_with_its_format_2_fields( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("us"), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [variable.name for variable in result.variables] == [ + "employment_income", + "total_income", + ] + employment_income, total_income = result.variables + assert employment_income.is_input_variable is True + assert employment_income.hidden_input is False + assert employment_income.module_name == "household.income.employment_income" + assert employment_income.adds_parameter_path is None + assert total_income.hidden_input is True + assert total_income.adds_parameter_path == "gov.example.parts" + assert total_income.subtracts_parameter_path == "gov.example.unpublished_parts" + assert total_income.enum_values == [ + MetadataEnumValue(value="LOW", label="Low"), + MetadataEnumValue(value="HIGH", label=2), + ] + assert len(statements) == 3 + + +def test_catalog_variables_carry_the_dated_values_of_parameter_named_parts( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result = _service(catalog_session).get_catalog_variables("us") + + assert result.part_parameter_values == { + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + value=["employment_income"], + ), + MetadataDatedValue( + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income", "pension_income"], + ), + ], + "gov.example.unpublished_parts": [], + } + + +def test_catalog_variables_without_parameter_named_parts_read_no_values( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("uk"), + ) + + assert [variable.name for variable in result.variables] == ["employment_income"] + assert result.part_parameter_values == {} + assert len(statements) == 2 + assert all("parameter_values" not in statement for statement in statements) + + +def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> None: + variable = _service(catalog_session).list_variables("us").items[0] + + assert variable.is_input_variable is True + assert variable.module_name == "household.income.employment_income" + assert variable.enum_values is None + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -762,6 +908,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region_by_code", "get_economy_options", "get_model_catalog", + "get_catalog_variables", } for method_name in method_names: From adff8a2560ae106cb4c68e97c1f0ad320db16624 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:53:39 +0400 Subject: [PATCH 060/118] Fetch several parameters by name with their values and breadcrumb labels GET /v2/model-catalog/parameters takes up to 100 repeated name keys (the shared CatalogParameterQuery) and returns those parameters in name order, each with its economy and household flags and its current-law values by date, the stored labels of the parameter nodes above them (a bracket index such as rates[0] counting as a child of its schedule rates), and the requested names the version lacks, in four queries on the existing (version, name) unique indexes. A client showing or editing a reform loads all of its parameters in this one request. The economy and household flags also join every parameter summary, so /v2/parameters pages and details carry them. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/engineering/migration-contracts.md | 3 +- docs/generated/migration_contracts.json | 18 ++- docs/migration/stage-9-v2-metadata.md | 12 ++ .../fastapi_routes/dependencies.py | 17 ++- .../v2/metadata/catalog_routes.py | 30 +++- .../v2/metadata/response_models.py | 7 + policyengine_api/query_parameters.py | 17 +++ .../reads_parameter_nodes.py | 34 +++++ .../database_connectors/reads_parameters.py | 20 +++ .../services/v2/metadata/services.py | 47 ++++++ .../services/v2/metadata/transformations.py | 84 +++++++++-- .../services/v2/metadata/types.py | 29 +++- tests/contract/registry.py | 18 +++ tests/integration/test_v2_metadata_routes.py | 35 +++++ .../unit/test_migration_contract_artifacts.py | 2 +- tests/unit/v2/test_metadata_routes.py | 87 +++++++++++ tests/unit/v2/test_metadata_service.py | 139 ++++++++++++++++++ 18 files changed, 584 insertions(+), 16 deletions(-) create mode 100644 policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 5e897c7ed..1e4e961ec 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,3 +1,4 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 33212c71d..411656a62 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -8,7 +8,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | ---: | | route group count | 9 | | workflow count | 16 | -| request count | 91 | +| request count | 92 | | db entity count | 6 | | sim flow count | 3 | @@ -217,6 +217,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | | `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | | `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | +| `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | ### `simulation_submit_poll` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 74592e7fc..9232097e8 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,7 +1,7 @@ { "metadata": { "db_entity_count": 6, - "request_count": 91, + "request_count": 92, "route_group_count": 9, "sim_flow_count": 3, "workflow_count": 16 @@ -1648,6 +1648,22 @@ "result.variables", "result.part_parameter_values" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names" + ] } ] }, diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index ed8456fc4..a5aaaba4c 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -54,6 +54,18 @@ it shows instead of the list resolved on 2025-01-01; a path whose parameter the version lacks maps to an empty list. `GET /v2/variables` pages carry the same variable fields. +`GET /v2/model-catalog/parameters` takes the same query plus one or more +`name` keys (at most 100, each an exact parameter path) and returns those +parameters in name order, each with its `economy` and `household` flags and its +current-law values in start-date order, in one response: a client showing or +editing a reform loads all of its parameters at once. `node_labels` maps each +ancestor of a returned parameter that the version stores as a parameter node +to its stored label (null when it has none), for breadcrumbs; a bracket index +such as `rates[0]` counts as a child of its schedule `rates`. `missing_names` +lists requested names the version has no parameter for, so a reform saved +against an older version still opens. `GET /v2/parameters` pages and details +carry the same `economy` and `household` flags. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 7de4f62e0..6631c8630 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -15,7 +15,7 @@ from policyengine_api.readiness import is_ready if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from uuid import UUID from policyengine_api.json_types import JSONObject @@ -34,6 +34,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -339,6 +340,20 @@ def get_catalog_variables( """ ... + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values; the result also carries the + stored labels of their ancestor nodes and the names the version lacks. + """ + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index a648ece21..8d91e8630 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -14,10 +14,11 @@ read_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, ) -from policyengine_api.query_parameters import CatalogQuery +from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -25,6 +26,9 @@ CatalogQueryDependency = Annotated[ CatalogQuery, Depends(query_dependency(CatalogQuery)) ] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] def build_v2_model_catalog_router( @@ -88,4 +92,28 @@ def get_catalog_variables( ), ) + @router.get( + "/model-catalog/parameters", + response_model=MetadataCatalogParametersResponse, + summary="Get several parameters of one catalog version by name", + description=( + "Returns the named parameters (repeat name for each, at most 100) " + "in name order, each with its economy and household flags and its " + "current-law values by date, plus the stored labels of their " + "ancestor nodes for breadcrumbs and the requested names the " + "version has no parameter for. A client showing or editing a " + "reform loads all of its parameters in this one request." + ), + ) + def get_catalog_parameters( + query: CatalogParameterQueryDependency, + ) -> MetadataCatalogParametersResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogParametersResponse, + lambda reader: reader.get_catalog_parameters( + query.country_id, query.policyengine_version, names=query.name + ), + ) + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index 56791d2bb..697ab6f76 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,6 +6,7 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -153,3 +154,9 @@ class MetadataCatalogVariablesResponse( MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] ): """Success response holding every variable of one catalog version.""" + + +class MetadataCatalogParametersResponse( + MetadataResourceSuccessResponse[MetadataCatalogParametersResult] +): + """Success response holding parameters fetched by name, with their values.""" diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index e06e9cdcb..a9f0816ae 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -232,6 +232,23 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +MAXIMUM_PARAMETER_NAMES = 100 +ParameterName = Annotated[ + str, + Field(min_length=1, max_length=512, description="Exact parameter path"), +] + + +class CatalogParameterQuery(CatalogQuery): + """Query contract for fetching several parameters of a catalog by name.""" + + name: list[ParameterName] = Field( + min_length=1, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Exact parameter paths; repeat the key for each one", + ) + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py new file mode 100644 index 000000000..154a1a6eb --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -0,0 +1,34 @@ +"""Database selections for parameter-node labels.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows + +if TYPE_CHECKING: + from collections.abc import Collection + from uuid import UUID + + +def read_parameter_node_labels( + session: Session, *, model_version_id: UUID, node_names: Collection[str] +) -> list[Any]: + """Return the name and stored label of each named node of a model version. + + Names the version has no node for are absent. Rows are in name order. + """ + if not node_names: + return [] + return read_rows( + session, + select(col(ParameterNode.name), col(ParameterNode.label)) + .where( + col(ParameterNode.tax_benefit_model_version_id) == model_version_id, + col(ParameterNode.name).in_(sorted(node_names)), + ) + .order_by(col(ParameterNode.name)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 739b6b0cb..e16bca767 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -53,6 +53,26 @@ def read_parameters( ) +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Parameter]: + """Return the named parameters of a model version in name order. + + Names the version has no parameter for are absent. + """ + if not parameter_names: + return [] + return read_rows( + session, + select(Parameter) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + ) + .order_by(col(Parameter.name)), + ) + + def read_parameter( session: Session, *, model_version_id: UUID, parameter_id: UUID ) -> Parameter | None: diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index e6d4506c7..33f9ef07f 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -16,6 +16,9 @@ read_datasets, read_input_datasets, ) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 # a dotted module path cannot be wrapped + read_parameter_node_labels, +) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, ) @@ -25,6 +28,7 @@ read_parameter_value, read_parameter_values, read_parameters, + read_parameters_by_name, ) from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( read_input_dataset, @@ -37,6 +41,8 @@ read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + ancestor_paths, + metadata_catalog_parameters, metadata_catalog_variables, metadata_dataset, metadata_economy_options, @@ -55,6 +61,7 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, @@ -77,6 +84,7 @@ ) if TYPE_CHECKING: + from collections.abc import Sequence from uuid import UUID from policyengine_api.services.v2.metadata.database_session import ( @@ -618,3 +626,42 @@ def get_catalog_variables( return metadata_catalog_variables( selected, variables=variables, part_values=part_values ) + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values, and the result carries the stored + labels of their ancestor nodes and the requested names the version + lacks. Repeated names count once. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + model_version_id = selected.model_version.id + requested_names = set(names) + parameters = read_parameters_by_name( + session, model_version_id=model_version_id, parameter_names=requested_names + ) + found_names = {parameter.name for parameter in parameters} + value_rows = read_current_law_values_by_parameter_name( + session, model_version_id=model_version_id, parameter_names=found_names + ) + node_rows = read_parameter_node_labels( + session, + model_version_id=model_version_id, + node_names={ + ancestor for name in found_names for ancestor in ancestor_paths(name) + }, + ) + return metadata_catalog_parameters( + selected, + requested_names=requested_names, + parameters=parameters, + value_rows=value_rows, + node_rows=node_rows, + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 651f26414..cedd21ab2 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,6 +8,8 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, @@ -144,23 +146,37 @@ def metadata_catalog_variables( parameter-name and start-date order. Each path a variable names gets an entry, empty when no row carries its name. """ - values_by_path: dict[str, list[MetadataDatedValue]] = { - path: [] for path in sorted(part_parameter_paths(variables)) + values_by_path = _dated_values_by_parameter_name( + part_values, parameter_names=part_parameter_paths(variables) + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, + ) + + +def _dated_values_by_parameter_name( + value_rows: list[Any], *, parameter_names: set[str] +) -> dict[str, list[MetadataDatedValue]]: + """Group value rows by parameter name, with an empty list for each other name. + + value_rows holds rows of read_current_law_values_by_parameter_name, in + parameter-name and start-date order. + """ + values_by_name: dict[str, list[MetadataDatedValue]] = { + name: [] for name in sorted(parameter_names) } - for row in part_values: - values_by_path.setdefault(row.name, []).append( + for row in value_rows: + values_by_name.setdefault(row.name, []).append( MetadataDatedValue( start_date=as_utc(row.start_date), end_date=None if row.end_date is None else as_utc(row.end_date), value=row.value_json, ) ) - return MetadataCatalogVariablesResult( - policyengine_version=selected.policyengine_version, - catalog_format=selected.model_version.catalog_format, - variables=[metadata_variable(variable) for variable in variables], - part_parameter_values=values_by_path, - ) + return values_by_name def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: @@ -172,6 +188,54 @@ def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: description=parameter.description, data_type=parameter.data_type, unit=parameter.unit, + economy=parameter.economy, + household=parameter.household, + ) + + +def ancestor_paths(parameter_name: str) -> list[str]: + """Return the paths of every node above parameter_name, from the root down. + + A dot separates a child, and so does the ``[`` of a bracket index: + ``gov.tax.rates[0].rate`` has the ancestors ``gov``, ``gov.tax``, + ``gov.tax.rates`` and ``gov.tax.rates[0]``. + """ + return [ + parameter_name[:position] + for position, character in enumerate(parameter_name) + if character in ".[" and position > 0 + ] + + +def metadata_catalog_parameters( + selected: SelectedCatalog, + *, + requested_names: set[str], + parameters: list[Parameter], + value_rows: list[Any], + node_rows: list[Any], +) -> MetadataCatalogParametersResult: + """Build the requested parameters with their values and ancestor labels. + + value_rows holds rows of read_current_law_values_by_parameter_name and + node_rows rows of read_parameter_node_labels. + """ + values_by_name = _dated_values_by_parameter_name( + value_rows, parameter_names={parameter.name for parameter in parameters} + ) + found_names = {parameter.name for parameter in parameters} + return MetadataCatalogParametersResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + parameters=[ + MetadataCatalogParameter( + **metadata_parameter(parameter).model_dump(), + values=values_by_name[parameter.name], + ) + for parameter in parameters + ], + node_labels={row.name: row.label for row in node_rows}, + missing_names=sorted(requested_names - found_names), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index d6633eeaf..472d270e2 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -114,7 +114,9 @@ class MetadataVariable(StrictResponseModel): class MetadataParameterSummary(StrictResponseModel): """A policy parameter, described without its values. - Its current-law values are listed by GET /v2/parameter-values. + Its current-law values are listed by GET /v2/parameter-values. economy and + household say whether a reform of an economy or of a household may change + it; they are null in versions published before catalog format 2. """ id: UUID @@ -123,6 +125,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None + economy: bool | None = None + household: bool | None = None class MetadataCanonicalParameterValue(StrictResponseModel): @@ -332,3 +336,26 @@ class MetadataCatalogVariablesResult(StrictResponseModel): catalog_format: int | None variables: list[MetadataVariable] part_parameter_values: dict[str, list[MetadataDatedValue]] + + +class MetadataCatalogParameter(MetadataParameterSummary): + """A parameter with its current-law values in start-date order.""" + + values: list[MetadataDatedValue] + + +class MetadataCatalogParametersResult(StrictResponseModel): + """Requested parameters of one catalog version, in name order. + + node_labels maps each ancestor of a requested parameter that the version + stores as a parameter node to the node's stored label (null when it has + none), for breadcrumbs; a bracket index such as ``rates[0]`` counts as a + child of its schedule ``rates``. missing_names lists requested names the + version has no parameter for. + """ + + policyengine_version: str + catalog_format: int | None + parameters: list[MetadataCatalogParameter] + node_labels: dict[str, str | None] + missing_names: list[str] diff --git a/tests/contract/registry.py b/tests/contract/registry.py index b126a838e..b84980a1c 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -702,6 +702,24 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path=( + "/v2/model-catalog/parameters?country_id=us" + "&name=gov.irs.income.bracket.rates.1" + ), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 68a44ebda..5f5e6af26 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -508,3 +508,38 @@ def test_postgres_catalog_variables_carry_part_parameter_values_by_date( "employment_income" ] assert uk.json()["result"]["part_parameter_values"] == {} + + +def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( + published_engine: Engine, +) -> None: + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/parameters", + params=[ + ("country_id", "us"), + ("policyengine_version", POLICYENGINE_VERSION), + ("name", "gov.example.rate"), + ("name", "gov.removed.amount"), + ], + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [parameter["name"] for parameter in result["parameters"]] == [ + "gov.example.rate" + ] + parameter = result["parameters"][0] + assert (parameter["economy"], parameter["household"]) == (True, False) + assert parameter["values"] == [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": "2025-12-31T00:00:00Z", + "value": 0.1, + }, + {"start_date": "2026-01-01T00:00:00Z", "end_date": None, "value": 0.2}, + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed.amount"] diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index a52730e6e..e35785e12 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -14,7 +14,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["metadata"] == { "route_group_count": 9, "workflow_count": 16, - "request_count": 91, + "request_count": 92, "db_entity_count": 6, "sim_flow_count": 3, } diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e522a114b..6b1b252e9 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -26,6 +26,8 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, @@ -198,6 +200,24 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_parameters": MetadataCatalogParametersResult( + policyengine_version=version, + catalog_format=2, + parameters=[ + MetadataCatalogParameter( + **parameter.model_dump(), + values=[ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=0.1, + ) + ], + ) + ], + node_labels={"gov.example": "Example policy"}, + missing_names=["gov.removed"], + ), "get_catalog_variables": MetadataCatalogVariablesResult( policyengine_version=version, catalog_format=2, @@ -567,6 +587,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/economy-options", "/v2/model-catalog", "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -653,6 +674,7 @@ def _every_resource_path(ids: dict[str, object]) -> list[str]: "/v2/economy-options?country_id=us", "/v2/model-catalog?country_id=us", "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/parameters?country_id=us&name=gov.example.rate", ] @@ -841,3 +863,68 @@ def test_catalog_variables_route_rejects_paging_and_unknown_queries( ) assert response.status_code == 422 + + +def test_catalog_parameters_route_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/parameters" + "?country_id=us&policyengine_version=4.20.3" + "&name=gov.example.rate&name=gov.removed" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["parameters"][0]["name"] == "gov.example.rate" + assert result["parameters"][0]["values"] == [ + {"start_date": "2025-01-01T00:00:00Z", "end_date": None, "value": 0.1} + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed"] + assert reader.calls == [ + ( + "get_catalog_parameters", + ("us", "4.20.3"), + {"names": ["gov.example.rate", "gov.removed"]}, + ) + ] + + +@pytest.mark.parametrize( + "query", + [ + "country_id=us", + "country_id=us&name=", + "country_id=us&name=gov.example.rate&offset=0", + "country_id=us&" + "&".join(f"name=gov.p{index}" for index in range(101)), + ], +) +def test_catalog_parameters_route_rejects_invalid_name_lists(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog/parameters?{query}") + + assert response.status_code == 422 + assert calls == [] + + +def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog/parameters"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version", "name"} + assert parameters["name"]["required"] is True + name_schema = parameters["name"]["schema"] + assert name_schema["type"] == "array" + assert name_schema["minItems"] == 1 + assert name_schema["maxItems"] == 100 diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d30ba98b8..d8813b960 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -34,6 +34,7 @@ MetadataDatabaseSession, ) from policyengine_api.services.v2.metadata.services import V2MetadataService +from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( MetadataDatedValue, MetadataEntity, @@ -843,6 +844,142 @@ def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> N assert variable.enum_values is None +def _add_bracket_schedule(session: Session) -> None: + model_version = _us_model_version(session) + session.add_all( + [ + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates", + label="Example rates", + description=None, + ), + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0]", + label=None, + description=None, + ), + ] + ) + bracket_rate = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0].rate", + label="Rate", + description=None, + data_type="float", + unit="/1", + economy=True, + household=True, + ) + session.add(bracket_rate) + session.add( + ParameterValue( + id=uuid4(), + parameter_id=bracket_rate.id, + value_json=0.2, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ) + session.commit() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("gov", []), + ("gov.example.rate", ["gov", "gov.example"]), + ( + "gov.example.rates[0].rate", + ["gov", "gov.example", "gov.example.rates", "gov.example.rates[0]"], + ), + ("gov.example.rates[10]", ["gov", "gov.example", "gov.example.rates"]), + ], +) +def test_ancestor_paths_treat_a_bracket_index_as_a_child( + name: str, expected: list[str] +) -> None: + assert ancestor_paths(name) == expected + + +def test_named_parameters_load_with_values_and_ancestor_labels_in_one_read( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_parameters( + "us", + names=["gov.example.rates[0].rate", "gov.example.rate", "gov.example.rate"], + ), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [parameter.name for parameter in result.parameters] == [ + "gov.example.rate", + "gov.example.rates[0].rate", + ] + example_rate, bracket_rate = result.parameters + assert (example_rate.economy, example_rate.household) == (True, False) + assert [value.value for value in example_rate.values] == [0.1, 0.2] + assert example_rate.values[0].start_date == datetime(2025, 1, 1, tzinfo=UTC) + assert example_rate.values[0].end_date == datetime(2025, 12, 31, tzinfo=UTC) + assert [value.value for value in bracket_rate.values] == [0.2] + assert result.node_labels == { + "gov.example": "Example policy", + "gov.example.rates": "Example rates", + "gov.example.rates[0]": None, + } + assert result.missing_names == [] + assert len(statements) == 4 + + +def test_named_parameters_list_names_the_version_lacks( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_catalog_parameters( + "uk", names=["gov.example.rate", "gov.removed.amount"] + ) + + assert [parameter.name for parameter in result.parameters] == ["gov.example.rate"] + assert result.missing_names == ["gov.removed.amount"] + + +def test_named_parameters_exclude_policy_and_dynamic_values( + catalog_session: Session, +) -> None: + model_version = _us_model_version(catalog_session) + parameter = catalog_session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version.id, + Parameter.name == "gov.example.rate", + ) + ).one() + catalog_session.add( + ParameterValue( + id=uuid4(), + parameter_id=parameter.id, + value_json=0.9, + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ) + ) + catalog_session.commit() + + result = _service(catalog_session).get_catalog_parameters( + "us", names=["gov.example.rate"] + ) + + assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -852,6 +989,7 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package / "data" / "v2" / "catalog" / "catalog_selection.py", connector_directory / "reads.py", connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_nodes.py", connector_directory / "reads_parameter_tree.py", connector_directory / "reads_parameters.py", connector_directory / "reads_regions.py", @@ -909,6 +1047,7 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_economy_options", "get_model_catalog", "get_catalog_variables", + "get_catalog_parameters", } for method_name in method_names: From 830ee5b8992f29ebedfb374f9f635755e0dddf57 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:59:16 +0400 Subject: [PATCH 061/118] Filter parameter pages and values by exact parameter names GET /v2/parameters accepts repeated name keys and GET /v2/parameter-values repeated parameter_name keys (at most 100 each), so a client can page only those parameters or their values. Because their query contracts change, both routes now read their query through the shared ParameterCollectionQuery and ParameterValueCollectionQuery models: unknown and repeated scalar parameters, an unsupported country_id and a malformed policyengine_version are rejected with 422 before any read (the last two returned 400 from the read before, and unknown parameters were ignored). The service and connector take the search and the names as one ParameterFilter, like ParameterValueFilter. Co-Authored-By: Claude Opus 5.5 --- ...1f-116-catalog-routes-on-demand.changed.md | 1 + docs/migration/stage-9-v2-metadata.md | 9 ++ .../fastapi_routes/dependencies.py | 13 +- .../v2/metadata/parameter_routes.py | 54 ++++---- policyengine_api/query_parameters.py | 29 +++++ .../database_connectors/reads_parameters.py | 28 +++- .../services/v2/metadata/services.py | 17 ++- .../services/v2/metadata/transformations.py | 1 + .../services/v2/metadata/types.py | 23 +++- tests/unit/v2/test_metadata_routes.py | 121 ++++++++++++++++++ tests/unit/v2/test_metadata_service.py | 31 ++++- 11 files changed, 281 insertions(+), 46 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md new file mode 100644 index 000000000..62dc783e3 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -0,0 +1 @@ +/v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. Both read their query through the shared query models, so unknown or repeated parameters, an unsupported country_id and a malformed policyengine_version now return 422. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index a5aaaba4c..028a6713e 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -66,6 +66,15 @@ lists requested names the version has no parameter for, so a reform saved against an older version still opens. `GET /v2/parameters` pages and details carry the same `economy` and `household` flags. +`GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page +only those exact parameters, and `GET /v2/parameter-values` repeated +`parameter_name` keys to page only those parameters' values. Both routes now +read their query through the shared query models (`ParameterCollectionQuery` +and `ParameterValueCollectionQuery`): unknown and repeated scalar parameters, +an unsupported `country_id` and a malformed `policyengine_version` are +rejected with HTTP 422 before any read; before, the last two returned HTTP 400 +from the read and unknown parameters were ignored. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 6631c8630..1dfcdaaa5 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -48,6 +48,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.policies.types import ( @@ -193,11 +194,12 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: - """Return parameters in name order, optionally filtered by ``search``. + """Return parameters in name order, optionally filtered. - ``search`` is a case-insensitive substring of the name, label or description. + The filter's ``search`` is a case-insensitive substring of the name, + label or description; its ``names`` keep only those exact names. """ ... @@ -234,9 +236,10 @@ def list_parameter_values( offset: int = 0, limit: int = 100, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - """Return canonical values, optionally for one parameter. + """Return canonical values, optionally for one or several parameters. - With the filter's ``current``, only values in effect on the UTC day of its + The filter's ``parameter_id`` or ``parameter_names`` select parameters. + With its ``current``, only values in effect on the UTC day of its ``now`` (default: the present) are returned. """ ... diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 5895c8564..252682bb0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,13 +2,13 @@ from __future__ import annotations -from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated from uuid import UUID from fastapi import APIRouter, Depends, Query from starlette.responses import JSONResponse +from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, @@ -21,7 +21,14 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.query_parameters import ( + ParameterCollectionQuery, + ParameterValueCollectionQuery, +) +from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueFilter, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -30,16 +37,13 @@ Limit = Annotated[int, Query(ge=1, le=500)] -@dataclass(frozen=True) -class ParameterValueListingQuery: - """The query parameters of the canonical parameter-value listing route.""" - - country_id: str - policyengine_version: str | None = None - parameter_id: UUID | None = None - current: bool = False - offset: Offset = 0 - limit: Limit = 100 +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] def build_v2_metadata_parameter_router( @@ -62,21 +66,20 @@ def build_v2_metadata_parameter_router( summary="List parameters from a selected PolicyEngine.py catalog", ) def list_parameters( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: ParameterCollectionQueryDependency, ) -> MetadataParameterPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterPageResponse, lambda reader: reader.list_parameters( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + parameter_filter=ParameterFilter( + search=query.search, + names=None if query.name is None else tuple(query.name), + ), ), ) @@ -130,7 +133,7 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( - query: Annotated[ParameterValueListingQuery, Depends()], + query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: return read_resource( dependencies, @@ -141,6 +144,11 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + parameter_names=( + None + if query.parameter_name is None + else tuple(query.parameter_name) + ), ), offset=query.offset, limit=query.limit, diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index a9f0816ae..ec614f109 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -249,6 +249,35 @@ class CatalogParameterQuery(CatalogQuery): ) +class ParameterCollectionQuery(CatalogQuery, PaginationQuery): + """Query contract for a page of a catalog's parameters.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths; repeat the key", + ) + + +class ParameterValueCollectionQuery(CatalogQuery, PaginationQuery): + """Query contract for a page of a catalog's current-law parameter values.""" + + parameter_id: ResourceId | None = None + parameter_name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths' values; repeat the key", + ) + current: bool = Field( + default=False, + description="Keep only the values in effect today (UTC)", + ) + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index e16bca767..39ab71bd5 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -14,7 +14,10 @@ from collections.abc import Collection from uuid import UUID - from policyengine_api.services.v2.metadata.types import ParameterValueSelection + from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueSelection, + ) def _escape_like(value: str) -> str: @@ -27,17 +30,23 @@ def read_parameters( model_version_id: UUID, offset: int, limit: int, - search: str | None, + parameter_filter: ParameterFilter, ) -> list[Parameter]: """Return a page of a model version's parameters in name order. - search matches name, label or description as a literal, case-insensitive - substring. Fetch limit + 1 rows so the caller can tell whether another page - exists. + The filter's search matches name, label or description as a literal, + case-insensitive substring, and its names keep only parameters with those + exact names. Fetch limit + 1 rows so the caller can tell whether another + page exists. """ statement = select(Parameter).where( col(Parameter.tax_benefit_model_version_id) == model_version_id ) + if parameter_filter.names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(parameter_filter.names)) + ) + search = parameter_filter.search if search: pattern = f"%{_escape_like(search)}%" statement = statement.where( @@ -98,8 +107,9 @@ def read_parameter_values( """Return a page of a model version's current-law parameter values. Values tied to a policy or dynamic are excluded. The selection's - parameter_id keeps one parameter's values and its selected_day keeps those - in effect on that day. Rows are ordered by parameter name, then newest + parameter_id keeps one parameter's values, its parameter_names the values + of the parameters with those names, and its selected_day those in effect + on that day. Rows are ordered by parameter name, then newest first, and limit + 1 are fetched so the caller can tell whether another page exists. """ @@ -116,6 +126,10 @@ def read_parameter_values( ) if parameter_id is not None: statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selection.parameter_names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(selection.parameter_names)) + ) if selected_day is not None: statement = statement.where( col(ParameterValue.start_date) <= selected_day, diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 33f9ef07f..cdae4758e 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -75,6 +75,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -271,12 +272,13 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: """Return a page of the catalog's parameters in name order. - search keeps parameters whose name, label or description contains it, - ignoring case. + The filter's search keeps parameters whose name, label or description + contains it, ignoring case, and its names keep only parameters with + those exact names. No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -286,7 +288,9 @@ def list_parameters( model_version_id=selected.model_version.id, offset=offset, limit=limit, - search=search, + parameter_filter=( + parameter_filter if parameter_filter is not None else ParameterFilter() + ), ) return page_result( selected, @@ -359,8 +363,9 @@ def list_parameter_values( """Return a page of current-law parameter values, newest first per parameter. Values belonging to a reform or dynamic are never listed. The filter's - parameter_id keeps one parameter's values, and current=True keeps those in - effect on the UTC day of now (today by default). No filter lists them all. + parameter_id keeps one parameter's values, parameter_names the values of + the parameters with those exact names, and current=True those in effect + on the UTC day of now (today by default). No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index cedd21ab2..4b9167845 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -286,6 +286,7 @@ def parameter_value_selection( return ParameterValueSelection( parameter_id=value_filter.parameter_id, selected_day=selected_day, + parameter_names=value_filter.parameter_names, ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 472d270e2..36dee6c43 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -11,28 +11,43 @@ from pydantic import BaseModel, ConfigDict, JsonValue +@dataclass(frozen=True) +class ParameterFilter: + """Which parameters a listing keeps. + + ``search`` keeps parameters whose name, label or description contains it, + ignoring case, and ``names`` the parameters with those exact names. + """ + + search: str | None = None + names: tuple[str, ...] | None = None + + @dataclass(frozen=True) class ParameterValueFilter: """Which current-law parameter values a listing keeps. - ``parameter_id`` keeps one parameter's values, and ``current`` keeps the - values in effect on the UTC day of ``now`` (today when ``now`` is None). + ``parameter_id`` keeps one parameter's values, ``parameter_names`` the + values of the parameters with those exact names, and ``current`` the values + in effect on the UTC day of ``now`` (today when ``now`` is None). """ parameter_id: UUID | None = None current: bool = False now: datetime | None = None + parameter_names: tuple[str, ...] | None = None @dataclass(frozen=True) class ParameterValueSelection: - """The parameter and effective day a parameter-value read is narrowed to. + """The parameters and effective day a parameter-value read is narrowed to. - None for either leaves the read unnarrowed by it. + None for any of them leaves the read unnarrowed by it. """ parameter_id: UUID | None selected_day: datetime | None + parameter_names: tuple[str, ...] | None = None class StrictResponseModel(BaseModel): diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 6b1b252e9..366647f29 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -47,6 +47,7 @@ MetadataRegionOption, MetadataTimePeriodOption, MetadataVariable, + ParameterFilter, ) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, @@ -928,3 +929,123 @@ def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: assert name_schema["type"] == "array" assert name_schema["minItems"] == 1 assert name_schema["maxItems"] == 100 + + +def test_parameter_collection_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + client = _client(lambda: reader) + + named = client.get( + "/v2/parameters?country_id=US&name=gov.example.rate&name=gov.other.rate" + ) + paged = client.get("/v2/parameters?country_id=us&offset=5&limit=50&search=rate") + + assert named.status_code == 200 + assert paged.status_code == 200 + assert reader.calls == [ + ( + "list_parameters", + ("us", None), + { + "offset": 0, + "limit": 100, + "parameter_filter": ParameterFilter( + search=None, names=("gov.example.rate", "gov.other.rate") + ), + }, + ), + ( + "list_parameters", + ("us", None), + { + "offset": 5, + "limit": 50, + "parameter_filter": ParameterFilter(search="rate", names=None), + }, + ), + ] + + +def test_parameter_values_fetch_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3" + "¶meter_name=gov.example.rate¶meter_name=gov.other.rate¤t=true" + ) + + assert response.status_code == 200 + name, args, kwargs = reader.calls[0] + assert (name, args) == ("list_parameter_values", ("us", "4.20.3")) + assert kwargs["value_filter"].parameter_names == ( + "gov.example.rate", + "gov.other.rate", + ) + assert kwargs["value_filter"].parameter_id is None + assert kwargs["value_filter"].current is True + assert (kwargs["offset"], kwargs["limit"]) == (0, 100) + + +@pytest.mark.parametrize( + "path", + [ + "/v2/parameters?country_id=us¶meter_id=1", + "/v2/parameters?country_id=us&limit=5&limit=6", + "/v2/parameters?country_id=us&name=", + "/v2/parameters?country_id=ca", + "/v2/parameters?country_id=us&" + + "&".join(f"name=gov.p{index}" for index in range(101)), + "/v2/parameter-values?country_id=us&name=gov.example.rate", + "/v2/parameter-values?country_id=us¤t=true¤t=false", + "/v2/parameter-values?country_id=us¶meter_id=not-a-uuid", + ], +) +def test_parameter_collections_reject_unknown_repeated_and_invalid_queries( + path: str, +) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(path) + + assert response.status_code == 422 + assert calls == [] + + +def test_parameter_collections_declare_their_name_filters_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + def query_parameters(path: str) -> dict[str, dict]: + return { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + } + + parameters = query_parameters("/v2/parameters") + values = query_parameters("/v2/parameter-values") + assert set(parameters) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "search", + "name", + } + assert parameters["name"]["required"] is False + assert set(values) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "parameter_id", + "parameter_name", + "current", + } + assert values["parameter_name"]["required"] is False diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d8813b960..fc0827b06 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -39,6 +39,7 @@ MetadataDatedValue, MetadataEntity, MetadataEnumValue, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -452,7 +453,9 @@ def test_resource_filters_and_details_remain_inside_selected_catalog( ) -> None: service = _service(catalog_session) variables = service.list_variables("us", search="employment") - parameters = service.list_parameters("us", search="example rate") + parameters = service.list_parameters( + "us", parameter_filter=ParameterFilter(search="example rate") + ) states = service.list_regions("us", region_type="state") region = service.get_region_by_code("us", "state/ca") @@ -980,6 +983,32 @@ def test_named_parameters_exclude_policy_and_dynamic_values( assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] +def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + service = _service(catalog_session) + + parameters = service.list_parameters( + "us", + parameter_filter=ParameterFilter( + names=("gov.example.rates[0].rate", "gov.removed.amount") + ), + ) + values = service.list_parameter_values( + "us", + value_filter=ParameterValueFilter( + parameter_names=("gov.example.rate", "gov.example.rates[0].rate") + ), + ) + + assert [parameter.name for parameter in parameters.items] == [ + "gov.example.rates[0].rate" + ] + assert parameters.items[0].economy is True + assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 17300e7bc7b4d15781019eb58c5a089214428039 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:07:34 +0400 Subject: [PATCH 062/118] Keep whole catalog-version responses once built in each process Measured against the full 6.10.0 catalog in a local database, every new query uses an existing (version, name) or (version, code) unique index and takes under 4 ms; the cost of /v2/model-catalog/variables (about 160 ms, 3.9 MB, 0.58 MB compressed) is reading 6,167 rows and building their models. A published catalog version never changes, so each process now keeps the serialized bodies of /v2/model-catalog and /v2/model-catalog/variables once built, in a bounded store (eight route, country and requested-version combinations, least recently used dropped first) shared by every request of the default dependencies. Concurrent first requests wait for one read, and failed reads are not kept. A kept variables response takes about 25 ms, most of it compression. Tests pass their own store or none. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 1 + docs/migration/stage-9-v2-metadata.md | 10 ++ .../fastapi_routes/dependencies.py | 13 ++ .../v2/metadata/catalog_routes.py | 13 +- .../fastapi_routes/v2/metadata/common.py | 32 +++++ .../v2/metadata/response_cache.py | 73 ++++++++++++ tests/unit/v2/test_metadata_response_cache.py | 111 ++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 59 +++++++++- 8 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/response_cache.py create mode 100644 tests/unit/v2/test_metadata_response_cache.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 1e4e961ec..4f665f71d 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -2,3 +2,4 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 028a6713e..3cbe58fab 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -87,6 +87,16 @@ deploy, and may be reused for five minutes (`max-age=300`). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. +Each API process also keeps the serialized bodies of `/v2/model-catalog` and +`/v2/model-catalog/variables` once built, for up to eight route, country and +requested-version combinations (least recently used dropped first; a request +without `policyengine_version` is keyed to the process's running version, +which never changes while it runs). Only the first successful request per +process reads the database; concurrent first requests wait for one read, and +a failed read is not kept. Measured locally against the full 6.10.0 US +catalog, the variables route takes about 160 ms when read and about 25 ms +when kept, most of it compression. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 1dfcdaaa5..43f6fbf4f 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -10,6 +10,9 @@ from fastapi import Depends from fastapi.security import APIKeyHeader, HTTPAuthorizationCredentials, HTTPBearer +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready @@ -773,6 +776,12 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_metadata_response_cache() -> MetadataResponseCache: + """Return the whole catalog-version responses this process keeps.""" + return MetadataResponseCache() + + def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -978,6 +987,9 @@ class NativeRouteDependencies: None ) v2_account_service_factory: Callable[[], V2AccountResourceService] | None = None + # Whole catalog-version responses (/v2/model-catalog and its variables) are + # kept here once built; None reads them on every request. + v2_metadata_response_cache: MetadataResponseCache | None = None # User-owned routes attach this at router level. The default validates the # Auth0 bearer token, or else accepts the unauthenticated-user header, and # fails closed (401/403/503); tests replace it with a callable returning a @@ -993,6 +1005,7 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_metadata_response_cache=_process_metadata_response_cache(), v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 8d91e8630..bef2ba326 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -5,13 +5,14 @@ from typing import TYPE_CHECKING, Annotated from fastapi import APIRouter, Depends -from starlette.responses import JSONResponse +from starlette.responses import JSONResponse, Response from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, read_resource, + read_whole_version_resource, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataCatalogParametersResponse, @@ -60,9 +61,10 @@ def build_v2_model_catalog_router( ) def get_model_catalog( query: CatalogQueryDependency, - ) -> MetadataModelCatalogResponse | JSONResponse: - return read_resource( + ) -> MetadataModelCatalogResponse | Response: + return read_whole_version_resource( dependencies, + ("model-catalog", query.country_id, query.policyengine_version), MetadataModelCatalogResponse, lambda reader: reader.get_model_catalog( query.country_id, query.policyengine_version @@ -83,9 +85,10 @@ def get_model_catalog( ) def get_catalog_variables( query: CatalogQueryDependency, - ) -> MetadataCatalogVariablesResponse | JSONResponse: - return read_resource( + ) -> MetadataCatalogVariablesResponse | Response: + return read_whole_version_resource( dependencies, + ("model-catalog/variables", query.country_id, query.policyengine_version), MetadataCatalogVariablesResponse, lambda reader: reader.get_catalog_variables( query.country_id, query.policyengine_version diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 9cd6ff688..d44c66f8a 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -37,6 +37,9 @@ NativeRouteDependencies, V2MetadataResourceReader, ) + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + ResponseKey, + ) ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { @@ -150,6 +153,35 @@ def read_resource( ) +def read_whole_version_resource( + dependencies: NativeRouteDependencies, + key: ResponseKey, + response_type: type[ResponseT], + operation: Callable[[V2MetadataResourceReader], object], +) -> ResponseT | Response: + """Serve a whole catalog-version response, built once per process when kept. + + With the dependencies' response cache, the serialized body for ``key`` is + read and built only on the first successful request; failures are served + as ``read_resource`` serves them and never kept. Without a cache this is + ``read_resource``. + """ + cache = dependencies.v2_metadata_response_cache + if cache is None: + return read_resource(dependencies, response_type, operation) + + def build() -> bytes | JSONResponse: + result = read_resource(dependencies, response_type, operation) + if isinstance(result, BaseModel): + return result.model_dump_json().encode() + return result + + built = cache.get_or_build(key, build) + if isinstance(built, bytes): + return Response(content=built, media_type="application/json") + return built + + def run_metadata_read( dependencies: NativeRouteDependencies, operation: Callable[[V2MetadataResourceReader], ReadT], diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py new file mode 100644 index 000000000..ae688ae4c --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -0,0 +1,73 @@ +"""A bounded, process-wide store of whole catalog-version response bodies.""" + +from __future__ import annotations + +import threading +from collections import OrderedDict +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + from starlette.responses import JSONResponse + +ResponseKey = tuple[str, str, str | None] +"""A route name, a country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + +DEFAULT_RESPONSE_CAPACITY = 8 + + +class MetadataResponseCache: + """Keep the serialized bodies of whole-version responses, built once each. + + A published catalog version never changes, so a body built for a route, + country and version stays valid for the life of the process. The most + recently used ``capacity`` bodies are kept. Requests for a body being built + wait for that build instead of starting another, and a failed read (a v2 + error response) is returned but not kept, so the next request tries again. + """ + + def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: + """Keep at most ``capacity`` bodies (at least one).""" + if capacity < 1: + msg = "capacity must be positive" + raise ValueError(msg) + self._capacity = capacity + self._bodies: OrderedDict[ResponseKey, bytes] = OrderedDict() + self._lock = threading.Lock() + self._build_locks: dict[ResponseKey, threading.Lock] = {} + + def _stored(self, key: ResponseKey) -> bytes | None: + with self._lock: + body = self._bodies.get(key) + if body is not None: + self._bodies.move_to_end(key) + return body + + def get_or_build( + self, + key: ResponseKey, + build: Callable[[], bytes | JSONResponse], + ) -> bytes | JSONResponse: + """Return the body kept for ``key``, calling ``build`` only when none is.""" + body = self._stored(key) + if body is not None: + return body + with self._lock: + build_lock = self._build_locks.setdefault(key, threading.Lock()) + with build_lock: + body = self._stored(key) + if body is not None: + return body + built = build() + with self._lock: + self._build_locks.pop(key, None) + if isinstance(built, bytes): + self._bodies[key] = built + self._bodies.move_to_end(key) + while len(self._bodies) > self._capacity: + self._bodies.popitem(last=False) + return built diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py new file mode 100644 index 000000000..a96d7b005 --- /dev/null +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -0,0 +1,111 @@ +"""The process-wide store of whole catalog-version response bodies.""" + +from __future__ import annotations + +import threading +import time + +import pytest +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) + + +def _counting_build(body: bytes, builds: list[str]): + def build() -> bytes: + builds.append("built") + return body + + return build + + +def test_a_body_is_built_once_per_key_and_reused() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + key = ("variables", "us", "6.10.0") + + first = cache.get_or_build(key, _counting_build(b'{"a":1}', builds)) + second = cache.get_or_build(key, _counting_build(b'{"b":2}', builds)) + + assert first == b'{"a":1}' + assert second == b'{"a":1}' + assert builds == ["built"] + + +def test_different_routes_countries_and_versions_are_kept_apart() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + for key in ( + ("variables", "us", "6.10.0"), + ("variables", "uk", "6.10.0"), + ("variables", "us", None), + ("model-catalog", "us", "6.10.0"), + ): + cache.get_or_build(key, _counting_build(repr(key).encode(), builds)) + + assert len(builds) == 4 + + +def test_a_failed_read_is_returned_but_not_kept() -> None: + cache = MetadataResponseCache() + failure = JSONResponse({"status": "error"}, status_code=503) + builds: list[str] = [] + + def failing_build() -> JSONResponse: + builds.append("failed") + return failure + + first = cache.get_or_build(("variables", "us", None), failing_build) + second = cache.get_or_build( + ("variables", "us", None), _counting_build(b"{}", builds) + ) + + assert first is failure + assert second == b"{}" + assert builds == ["failed", "built"] + + +def test_the_least_recently_used_body_is_dropped_beyond_capacity() -> None: + cache = MetadataResponseCache(capacity=2) + builds: list[str] = [] + oldest, middle, newest = (("variables", country, None) for country in "abc") + + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(middle, _counting_build(b"2", builds)) + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(newest, _counting_build(b"3", builds)) + cache.get_or_build(oldest, _counting_build(b"1", builds)) + cache.get_or_build(middle, _counting_build(b"2", builds)) + + assert builds == ["built", "built", "built", "built"] + + +def test_concurrent_requests_for_one_key_build_it_once() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + started = threading.Barrier(4) + + def slow_build() -> bytes: + builds.append("built") + time.sleep(0.05) + return b"{}" + + def request() -> None: + started.wait() + cache.get_or_build(("variables", "us", None), slow_build) + + threads = [threading.Thread(target=request) for _ in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert builds == ["built"] + + +def test_capacity_must_be_positive() -> None: + with pytest.raises(ValueError, match="capacity"): + MetadataResponseCache(capacity=0) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 366647f29..c3b50b6e5 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -20,6 +20,9 @@ from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, @@ -286,7 +289,7 @@ def detail(item: object) -> MetadataDetailResult: ) -def _client(factory) -> TestClient: +def _client(factory, *, response_cache=None) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -305,6 +308,7 @@ def v1_metadata(country_id: str): metadata_reader_factory=lambda: None, specification_provider=dict, v2_metadata_reader_factory=factory, + v2_metadata_response_cache=response_cache, ) settings = RouteImplementationSettings( health=RouteImplementation.FLASK_FALLBACK, @@ -1049,3 +1053,56 @@ def query_parameters(path: str) -> dict[str, dict]: "current", } assert values["parameter_name"]["required"] is False + + +def test_whole_version_responses_are_read_once_per_process_and_version() -> None: + results, _ids = _resource_results() + readers: list[ResourceReader] = [] + + def factory() -> ResourceReader: + readers.append(ResourceReader(results)) + return readers[-1] + + client = _client(factory, response_cache=MetadataResponseCache()) + paths = [ + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3", + "/v2/model-catalog/variables?country_id=uk", + ] + + first_responses = [client.get(path) for path in paths] + repeated_responses = [client.get(path) for path in paths] + + assert len(readers) == len(paths) + for first, repeated in zip(first_responses, repeated_responses, strict=True): + assert repeated.status_code == 200 + assert repeated.content == first.content + assert repeated.headers["etag"] == first.headers["etag"] + assert repeated.headers["cache-control"] == first.headers["cache-control"] + assert all(reader.closed for reader in readers) + + +def test_failed_whole_version_reads_are_retried() -> None: + results, _ids = _resource_results() + readers = [ + ResourceReader(results, error=MetadataCatalogUnavailableError("down")), + ResourceReader(results), + ] + client = _client(lambda: readers.pop(0), response_cache=MetadataResponseCache()) + + failed = client.get("/v2/model-catalog/variables?country_id=us") + recovered = client.get("/v2/model-catalog/variables?country_id=us") + + assert failed.status_code == 503 + assert "etag" not in failed.headers + assert recovered.status_code == 200 + assert readers == [] + + +def test_default_dependencies_share_one_response_cache() -> None: + first = route_dependencies.NativeRouteDependencies.defaults() + second = route_dependencies.NativeRouteDependencies.defaults() + + assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) + assert first.v2_metadata_response_cache is second.v2_metadata_response_cache From 08d990d3c5f9a083a38d433442d5a7cf3d26bb7f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:09:30 +0400 Subject: [PATCH 063/118] Check the on-demand catalog routes on each release and record their times The release workflow's live metadata test, run against each staging and production Cloud Run candidate, now loads each country's catalog in parts: /v2/model-catalog for the running version, its variables for that exact version twice and once with If-None-Match (which must get 304), and one parameter by name. It appends each request's time and body size to the GitHub job summary (GITHUB_STEP_SUMMARY, set by the runner in every step), so every release records the routes' times against its own catalog. The new paths join its OpenAPI check. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 8 ++ tests/integration/test_live_v2_metadata.py | 109 ++++++++++++++++++++- 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3cbe58fab..7d0f0a36f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -340,6 +340,14 @@ service error. None of these outcomes reads or repairs v1. Unsupported countries and methods return typed client errors. +The same test loads each country's catalog in parts as a client starting +cold would: `GET /v2/model-catalog` for the running version, then +`/v2/model-catalog/variables` for that exact version twice (the second served +from the process's kept body) and once more with `If-None-Match`, which must +return 304, then `/v2/model-catalog/parameters` for one parameter. It appends +each request's time and body size to the GitHub job summary, so every staging +and production release records these route times against its own catalog. + Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for success and error responses. diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 235304156..803e43b05 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -1,8 +1,11 @@ """Read-only checks for deployed Cloud Run v2 metadata preview routes.""" +import os +import time import tomllib from pathlib import Path +import httpx import pytest REPO = Path(__file__).parents[2] @@ -44,7 +47,15 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: assert response.status_code == 200, response.text[:500] document = response.json() expected_paths = {f"/v2/{resource}" for resource in PAGED_RESOURCES} - expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) + expected_paths.update( + { + "/v2/parameters/children", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", + } + ) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) assert "V2ErrorResponse" in document["components"]["schemas"] @@ -104,3 +115,99 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: assert payload["status"] == "error" assert payload["message"] assert "result" not in payload or payload["result"] is None + + +def _timed_get( + api_client: httpx.Client, path: str, **request_options: object +) -> tuple[httpx.Response, float]: + started = time.perf_counter() + response = api_client.get(path, **request_options) + return response, time.perf_counter() - started + + +def _record_route_times( + country_id: str, route_times: list[tuple[str, float, httpx.Response]] +) -> None: + """Append each route's time and size to the GitHub job summary, when there is one. + + The release workflow runs this test against each Cloud Run candidate, so + the summary keeps every release's staging and production route times. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + lines = [ + f"\n#### v2 catalog route times ({country_id})\n", + "| Request | Time (ms) | Body (bytes, uncompressed) |", + "| --- | ---: | ---: |", + ] + lines.extend( + f"| {label} | {seconds * 1000:.0f} | {len(response.content)} |" + for label, seconds, response in route_times + ) + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write("\n".join(lines) + "\n") + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> None: + expected_version = _installed_policyengine_version() + route_times: list[tuple[str, float, httpx.Response]] = [] + + catalog_response, seconds = _timed_get( + api_client, "/v2/model-catalog", params={"country_id": country_id} + ) + route_times.append( + ("/v2/model-catalog (running version)", seconds, catalog_response) + ) + catalog = _assert_ok(catalog_response, expected_version) + assert catalog["regions"] + assert catalog["economy_options"]["region"] + assert "variables" not in catalog + pinned = {"country_id": country_id, "policyengine_version": expected_version} + + variables_response, seconds = _timed_get( + api_client, "/v2/model-catalog/variables", params=pinned + ) + route_times.append(("/v2/model-catalog/variables", seconds, variables_response)) + repeated_variables, seconds = _timed_get( + api_client, "/v2/model-catalog/variables", params=pinned + ) + route_times.append( + ("/v2/model-catalog/variables (repeated)", seconds, repeated_variables) + ) + variables = _assert_ok(variables_response, expected_version) + assert variables["variables"] + assert variables_response.headers["cache-control"] == "public, max-age=86400" + assert repeated_variables.headers["etag"] == variables_response.headers["etag"] + revalidated, seconds = _timed_get( + api_client, + "/v2/model-catalog/variables", + params=pinned, + headers={"If-None-Match": variables_response.headers["etag"]}, + ) + route_times.append( + ("/v2/model-catalog/variables (revalidated)", seconds, revalidated) + ) + assert revalidated.status_code == 304 + + parameter_name = _assert_ok( + api_client.get("/v2/parameters", params={**pinned, "limit": 1}), + expected_version, + )["items"][0]["name"] + parameters_response, seconds = _timed_get( + api_client, + "/v2/model-catalog/parameters", + params={**pinned, "name": parameter_name}, + ) + route_times.append( + ("/v2/model-catalog/parameters (1 name)", seconds, parameters_response) + ) + parameters = _assert_ok(parameters_response, expected_version) + assert [parameter["name"] for parameter in parameters["parameters"]] == [ + parameter_name + ] + assert parameters["parameters"][0]["values"] + assert parameters["missing_names"] == [] + + _record_route_times(country_id, route_times) From 4c64114867d7268cdca0eccf26b7b9baef9be11c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 22:36:17 +0400 Subject: [PATCH 064/118] Reuse current parameter values only until the next UTC midnight GET /v2/parameter-values?current=true returns the values in effect on the request's UTC day, but its response got the same lifetime as any other: a day when it named a version. A cache that fetched it late on 31 December would have served last year's values for up to a day. The route now marks such a request in request.state, and the caching route caps its lifetime at the seconds left until the next UTC midnight, whichever lifetime would otherwise apply. No other metadata route depends on the time of the request. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 16 +++--- .../fastapi_routes/v2/metadata/common.py | 29 ++++++++++- .../v2/metadata/parameter_routes.py | 6 ++- tests/unit/v2/test_metadata_routes.py | 49 +++++++++++++++++++ 5 files changed, 92 insertions(+), 10 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 4f665f71d..a80f72ae6 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,4 +1,4 @@ -Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version. A request whose If-None-Match names the ETag gets 304 with no body. +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the next UTC midnight for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 7d0f0a36f..0b9873082 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -80,12 +80,16 @@ from the read and unknown parameters were ignored. Every successful metadata response carries a strong `ETag`, a hash of its exact body, and `Cache-Control: public`. A request that names its catalog with `policyengine_version` may be reused for a day (`max-age=86400`): a published -catalog version never changes, so only a change to the route's response format -can change such a response, and the `ETag` lets a client revalidate after the -day. A request without it selects the running version, which changes on -deploy, and may be reused for five minutes (`max-age=300`). A request whose -`If-None-Match` names the current `ETag` (strong or weak form, alone or in a -list) gets HTTP 304 with no body. Error responses carry neither header. +catalog version never changes, so such a response changes only with the +route's response format or with the day it is read on, and the `ETag` lets a +client revalidate after the day. A request without it selects the running +version, which changes on deploy, and may be reused for five minutes +(`max-age=300`). A response that depends on the request's UTC day, which is +`GET /v2/parameter-values?current=true` (the values in effect today), may be +reused only until the next UTC midnight, whichever lifetime would otherwise +apply. A request whose `If-None-Match` names the current `ETag` (strong or +weak form, alone or in a list) gets HTTP 304 with no body. Error responses +carry neither header. Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, for up to eight route, country and diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d44c66f8a..ce5314c0d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -4,6 +4,7 @@ import contextlib import hashlib +from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Any, TypeVar from fastapi.routing import APIRoute @@ -87,6 +88,26 @@ """ +DEPENDS_ON_UTC_DAY = "depends_on_utc_day" +"""The ``request.state`` attribute a route sets when its result depends on the day. + +Such a response, for example current values on the request's UTC day, may be +reused only until the next UTC midnight, whatever version it names. +""" + + +def utc_now() -> datetime: + """Return the current time in UTC; tests replace it to fix the clock.""" + return datetime.now(UTC) + + +def _seconds_until_next_utc_day(moment: datetime) -> int: + next_day = (moment + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + return max(int((next_day - moment).total_seconds()), 0) + + class CacheableMetadataRoute(APIRoute): """A metadata route whose successful responses carry caching headers. @@ -112,8 +133,10 @@ def with_cache_headers(request: Request, response: Response) -> Response: The ETag is a hash of the exact body, so it changes with any change to the content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the - running version for five minutes. A request whose ``If-None-Match`` names - the ETag gets a 304 with no body. + running version for five minutes, and one whose route marked it as + depending on the day (``DEPENDS_ON_UTC_DAY``) no later than the next UTC + midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with + no body. """ entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' max_age = ( @@ -121,6 +144,8 @@ def with_cache_headers(request: Request, response: Response) -> Response: if "policyengine_version" in request.query_params else RUNNING_VERSION_MAX_AGE_SECONDS ) + if getattr(request.state, DEPENDS_ON_UTC_DAY, False): + max_age = min(max_age, _seconds_until_next_utc_day(utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): return Response(status_code=304, headers=headers) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 252682bb0..ceb22e985 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -5,11 +5,12 @@ from typing import TYPE_CHECKING, Annotated from uuid import UUID -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, Query, Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( + DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, @@ -133,8 +134,11 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( + request: Request, query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: + # Current values are those in effect on the request's UTC day. + setattr(request.state, DEPENDS_ON_UTC_DAY, query.current) return read_resource( dependencies, MetadataParameterValuePageResponse, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index c3b50b6e5..d77571581 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1106,3 +1106,52 @@ def test_default_dependencies_share_one_response_cache() -> None: assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) assert first.v2_metadata_response_cache is second.v2_metadata_response_cache + + +@pytest.mark.parametrize( + ("now", "query", "expected_cache_control"), + [ + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=true", + "public, max-age=3600", + ), + ( + datetime(2026, 12, 31, 23, 58, tzinfo=UTC), + "¤t=true", + "public, max-age=120", + ), + ( + datetime(2026, 6, 1, 12, 0, tzinfo=UTC), + "¤t=true", + "public, max-age=300", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=false", + "public, max-age=86400", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3", + "public, max-age=86400", + ), + ], +) +def test_current_parameter_values_are_reused_only_until_the_next_utc_day( + monkeypatch: pytest.MonkeyPatch, + now: datetime, + query: str, + expected_cache_control: str, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + monkeypatch.setattr(common, "utc_now", lambda: now) + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + f"/v2/parameter-values?country_id=us{query}" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == expected_cache_control From 07383802ca97f938ac12a64f3391d055e6f65975 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:51:13 +0400 Subject: [PATCH 065/118] Read v2 metadata in autocommit sessions on the shared pool With the production engine options and a warm pool, the database's statement log showed five statements for each metadata request: the pool's liveness check, BEGIN, the catalog selection, the read and ROLLBACK. Each metadata read is one statement on rows a published catalog version never changes, so it needs no transaction. The default metadata reader now opens its session from a process-owned read-only factory bound to the v2 engine in autocommit mode (sharing its pool), so psycopg sends no BEGIN and no ROLLBACK: three statements per request. Other v2 sessions keep the ordinary mode, which the pool restores when a connection returns. The pool's liveness check is unchanged. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 1 + docs/migration/stage-9-v2-metadata.md | 12 +++++++ policyengine_api/data/v2/database.py | 35 ++++++++++++++++++- .../fastapi_routes/dependencies.py | 4 +-- tests/integration/test_v2_metadata_routes.py | 33 ++++++++++++++++- tests/unit/v2/test_database.py | 23 ++++++++++++ tests/unit/v2/test_metadata_routes.py | 4 ++- 7 files changed, 107 insertions(+), 5 deletions(-) create mode 100644 changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md new file mode 100644 index 000000000..2392c3e57 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -0,0 +1 @@ +v2 metadata reads run in autocommit read-only sessions on the shared pool, so each request sends no BEGIN or ROLLBACK to the database (two fewer round trips per request). diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 0b9873082..ce3efa833 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -75,6 +75,18 @@ an unsupported `country_id` and a malformed `policyengine_version` are rejected with HTTP 422 before any read; before, the last two returned HTTP 400 from the read and unknown parameters were ignored. +## Database round trips per request + +Metadata reads use read-only sessions in autocommit mode on the shared v2 +pool (`get_v2_read_session_factory`): each read is one statement on rows a +published catalog version never changes, so no transaction is needed, and the +database receives no `BEGIN` before the reads and no `ROLLBACK` when the +session closes. Measured with statement logging in a local database, a page of +500 parameters or one parameter's values took five statements per request +before (the pool's liveness check, `BEGIN`, the catalog selection, the read +and `ROLLBACK`) and three after. The pool's liveness check +(`pool_pre_ping`) is unchanged. + ## Response caching Every successful metadata response carries a strong `ETag`, a hash of its diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 07b88ad1a..3885ab94d 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -28,7 +28,7 @@ @dataclass class _V2EngineState: - """The process-owned v2 engine and Session factory. + """The process-owned v2 engine and Session factories. ``engine_pid`` is the ID of the process that created the engine, and ``engine_url_fingerprint`` identifies its target. ``_state_lock`` guards @@ -39,6 +39,7 @@ class _V2EngineState: engine_pid: int | None = None engine_url_fingerprint: tuple[object, ...] | None = None session_factory: sessionmaker[Session] | None = None + read_session_factory: sessionmaker[Session] | None = None _state_lock = Lock() @@ -82,6 +83,22 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: + """Build a factory of read-only Sessions that run without a transaction. + + The sessions share ``engine``'s pool in autocommit mode, so psycopg sends + each statement on its own: no BEGIN before the first read and no ROLLBACK + when the session closes. Use them only for reads whose statements need no + common snapshot, such as reads of a published catalog version, which never + changes. The pool restores the ordinary mode when a connection returns. + """ + return sessionmaker( + bind=engine.execution_options(isolation_level="AUTOCOMMIT"), + class_=Session, + expire_on_commit=False, + ) + + def _discard_inherited_state(current_pid: int) -> None: """Drop pool state inherited through a process fork without touching parent FDs.""" if _engine_state.engine is not None: @@ -90,6 +107,7 @@ def _discard_inherited_state(current_pid: int) -> None: _engine_state.engine_pid = current_pid _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None def get_v2_engine(settings: V2DatabaseSettings | None = None) -> Engine: @@ -130,6 +148,20 @@ def get_v2_session_factory( return _engine_state.session_factory +def get_v2_read_session_factory( + settings: V2DatabaseSettings | None = None, +) -> sessionmaker[Session]: + """Return the process-owned read-only Session factory without opening one. + + See ``build_v2_read_session_factory``. + """ + engine = get_v2_engine(settings) + with _state_lock: + if _engine_state.read_session_factory is None: + _engine_state.read_session_factory = build_v2_read_session_factory(engine) + return _engine_state.read_session_factory + + def close_v2_database() -> None: """Dispose this process's v2 pool and forget its lazy configuration.""" with _state_lock: @@ -139,6 +171,7 @@ def close_v2_database() -> None: _engine_state.engine_pid = None _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None atexit.register(close_v2_database) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 43f6fbf4f..073052be7 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -784,7 +784,7 @@ def _process_metadata_response_cache() -> MetadataResponseCache: def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( - get_v2_session_factory, + get_v2_read_session_factory, ) from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, @@ -792,7 +792,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.services.v2.metadata.services import V2MetadataService return V2MetadataService( - MetadataDatabaseSession(get_v2_session_factory()()), + MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), ) diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 5f5e6af26..e9531fd3f 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -12,7 +12,7 @@ from alembic.config import Config from fastapi.testclient import TestClient from flask import Flask, jsonify -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import Engine, make_url from sqlalchemy.pool import NullPool from sqlmodel import Session, select @@ -543,3 +543,34 @@ def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( ] assert result["node_labels"] == {"gov.example": "Example policy"} assert result["missing_names"] == ["gov.removed.amount"] + + +def test_postgres_metadata_reads_send_no_transaction_statements( + published_engine: Engine, +) -> None: + # Each metadata read is one statement on rows a published catalog never + # changes, so the read session runs in autocommit mode: psycopg then sends + # no BEGIN before the reads and no ROLLBACK when the session closes. + from policyengine_api.data.v2.database import build_v2_read_session_factory + + autocommit_modes: list[bool] = [] + + def record_mode(connection, _cursor, _statement, *_args) -> None: + autocommit_modes.append(connection.connection.dbapi_connection.autocommit) + + event.listen(published_engine, "before_cursor_execute", record_mode) + try: + service = V2MetadataService( + MetadataDatabaseSession(build_v2_read_session_factory(published_engine)()), + running_policyengine_version=POLICYENGINE_VERSION, + ) + result = service.list_parameters("us") + service.close() + finally: + event.remove(published_engine, "before_cursor_execute", record_mode) + + assert [item.name for item in result.items] == ["gov.example.rate"] + assert autocommit_modes + assert all(autocommit_modes) + with published_engine.connect() as connection: + assert connection.connection.dbapi_connection.autocommit is False diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index a00667a49..c1984199c 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -125,3 +125,26 @@ def test_forked_process_state_builds_a_new_engine( assert child_engine is not parent_engine assert database._engine_state.engine_pid == parent_pid + 1 + + +def test_read_sessions_share_the_runtime_pool_without_transactions() -> None: + settings = load_v2_runtime_database_settings(_environment()) + + read_factory = database.get_v2_read_session_factory(settings) + runtime_engine = database.get_v2_engine(settings) + + with read_factory() as session: + bind = session.get_bind() + assert bind.get_execution_options()["isolation_level"] == "AUTOCOMMIT" + assert bind.pool is runtime_engine.pool + assert database.get_v2_read_session_factory(settings) is read_factory + assert runtime_engine.get_execution_options().get("isolation_level") is None + + +def test_read_session_factory_is_rebuilt_with_the_engine() -> None: + settings = load_v2_runtime_database_settings(_environment()) + first = database.get_v2_read_session_factory(settings) + + database.close_v2_database() + + assert database.get_v2_read_session_factory(settings) is not first diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index d77571581..707345d09 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -517,7 +517,9 @@ def query_service(candidate_session, *, running_policyengine_version): captured["version"] = running_policyengine_version return reader - monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) + monkeypatch.setattr( + database, "get_v2_read_session_factory", lambda: lambda: session + ) monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, From bb2071a40eaf6d8230c008d1a61cb85f22681d86 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:57:40 +0400 Subject: [PATCH 066/118] Keep each selected catalog version for the life of the process After the autocommit change, each metadata request still sent three statements: the pool's liveness check, the catalog selection and the read. A published catalog version never changes, so the default metadata reader now keeps every successful selection in a process-wide SelectedCatalogStore, keyed by country and requested version, and later requests skip the selection query: two statements per request in the local statement log. Unsupported countries, invalid versions and absent catalogs are looked up, and fail, each time. Tests pass their own store or none. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 13 ++-- .../fastapi_routes/dependencies.py | 10 +++ .../services/v2/metadata/selected_catalogs.py | 41 ++++++++++ .../services/v2/metadata/services.py | 21 ++++- tests/unit/v2/test_metadata_routes.py | 15 +++- tests/unit/v2/test_metadata_service.py | 78 ++++++++++++++++++- 7 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 policyengine_api/services/v2/metadata/selected_catalogs.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md index 2392c3e57..8eff7221b 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -1 +1 @@ -v2 metadata reads run in autocommit read-only sessions on the shared pool, so each request sends no BEGIN or ROLLBACK to the database (two fewer round trips per request). +v2 metadata reads run in autocommit read-only sessions on the shared pool, and each process keeps the catalog versions it has selected, so a metadata request sends two statements to the database (the pool's liveness check and the read) instead of five. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index ce3efa833..d511af3aa 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -81,11 +81,14 @@ Metadata reads use read-only sessions in autocommit mode on the shared v2 pool (`get_v2_read_session_factory`): each read is one statement on rows a published catalog version never changes, so no transaction is needed, and the database receives no `BEGIN` before the reads and no `ROLLBACK` when the -session closes. Measured with statement logging in a local database, a page of -500 parameters or one parameter's values took five statements per request -before (the pool's liveness check, `BEGIN`, the catalog selection, the read -and `ROLLBACK`) and three after. The pool's liveness check -(`pool_pre_ping`) is unchanged. +session closes. Each process also keeps every catalog version a request has +selected (`SelectedCatalogStore`), since a published version never changes, so +later requests for the same country and version skip the selection query; +failed selections are not kept. Measured with statement logging in a local +database, a page of 500 parameters or one parameter's values took five +statements per request before (the pool's liveness check, `BEGIN`, the catalog +selection, the read and `ROLLBACK`) and two after (the liveness check and the +read). The pool's liveness check (`pool_pre_ping`) is unchanged. ## Response caching diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 073052be7..0be11f8fc 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -16,6 +16,9 @@ from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -776,6 +779,12 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_selected_catalogs() -> SelectedCatalogStore: + """Return the catalog versions this process's metadata reads have selected.""" + return SelectedCatalogStore() + + @lru_cache(maxsize=1) def _process_metadata_response_cache() -> MetadataResponseCache: """Return the whole catalog-version responses this process keeps.""" @@ -794,6 +803,7 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: return V2MetadataService( MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), + selected_catalogs=_process_selected_catalogs(), ) diff --git a/policyengine_api/services/v2/metadata/selected_catalogs.py b/policyengine_api/services/v2/metadata/selected_catalogs.py new file mode 100644 index 000000000..832705432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/selected_catalogs.py @@ -0,0 +1,41 @@ +"""A process-wide store of the catalog versions metadata reads have selected.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + +CatalogSelectionKey = tuple[str, str | None] +"""A country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + + +class SelectedCatalogStore: + """Keep each catalog version a request has selected, for the process's life. + + A published catalog version never changes, so once a request has found it, + later requests for the same country and version need not query for it + again. Only successful selections are kept: an unsupported country, an + invalid version or an absent catalog is looked up, and fails, each time. + The kept records are detached, fully loaded rows that are only read. + """ + + def __init__(self) -> None: + """Start with no selections kept.""" + self._selections: dict[CatalogSelectionKey, SelectedCatalog] = {} + self._lock = threading.Lock() + + def get(self, key: CatalogSelectionKey) -> SelectedCatalog | None: + """Return the selection kept for ``key``, or None.""" + with self._lock: + return self._selections.get(key) + + def keep(self, key: CatalogSelectionKey, selected: SelectedCatalog) -> None: + """Keep ``selected`` as the selection for ``key``.""" + with self._lock: + self._selections[key] = selected diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index cdae4758e..c5a63a223 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -91,6 +91,9 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) class V2MetadataService: @@ -107,12 +110,18 @@ def __init__( database_session: MetadataDatabaseSession, *, running_policyengine_version: str, + selected_catalogs: SelectedCatalogStore | None = None, ) -> None: - """Bind to a request's session and the running PolicyEngine.py version.""" + """Bind to a request's session and the running PolicyEngine.py version. + + With ``selected_catalogs``, a catalog version found once is taken from + that store instead of being queried again. + """ self._database_session = database_session self._running_policyengine_version = validate_policyengine_version( running_policyengine_version ) + self._selected_catalogs = selected_catalogs def close(self) -> None: """Close the request's database session.""" @@ -121,12 +130,20 @@ def close(self) -> None: def _select_catalog( self, country_id: str, policyengine_version: str | None ) -> SelectedCatalog: - return read_metadata_catalog( + key = (country_id, policyengine_version) + if self._selected_catalogs is not None: + kept = self._selected_catalogs.get(key) + if kept is not None: + return kept + selected = read_metadata_catalog( self._database_session.session, country_id=country_id, running_policyengine_version=self._running_policyengine_version, policyengine_version=policyengine_version, ) + if self._selected_catalogs is not None: + self._selected_catalogs.keep(key, selected) + return selected def _select_paginated_catalog( self, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 707345d09..96d504804 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -27,6 +27,9 @@ RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, MetadataCatalogParameter, @@ -512,9 +515,12 @@ def test_default_reader_uses_the_installed_policyengine_version( captured = {} reader = object() - def query_service(candidate_session, *, running_policyengine_version): + def query_service( + candidate_session, *, running_policyengine_version, selected_catalogs + ): captured["session"] = candidate_session.session captured["version"] = running_policyengine_version + captured["selected_catalogs"] = selected_catalogs return reader monkeypatch.setattr( @@ -533,7 +539,12 @@ def query_service(candidate_session, *, running_policyengine_version): route_dependencies._running_policyengine_version.cache_clear() assert result is reader - assert captured == {"session": session, "version": "5.2.0"} + assert captured["session"] is session + assert captured["version"] == "5.2.0" + assert isinstance(captured["selected_catalogs"], SelectedCatalogStore) + assert ( + route_dependencies._process_selected_catalogs() is captured["selected_catalogs"] + ) def test_default_factory_and_close_failures_preserve_success( diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index fc0827b06..d07e94bde 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -33,6 +33,9 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( @@ -239,10 +242,13 @@ def _us_model_version(session: Session) -> TaxBenefitModelVersion: ).one() -def _service(session: Session) -> V2MetadataService: +def _service( + session: Session, *, selected_catalogs: SelectedCatalogStore | None = None +) -> V2MetadataService: return V2MetadataService( MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, ) @@ -1009,6 +1015,76 @@ def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] +def test_a_kept_catalog_selection_saves_its_query_on_later_reads( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + + first, first_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + second, second_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + + assert second == first + assert len(first_statements) == 2 + assert len(second_statements) == 1 + assert "tax_benefit_model_versions" not in second_statements[0] + + +def test_failed_catalog_selections_are_not_kept(catalog_session: Session) -> None: + store = SelectedCatalogStore() + service = _service(catalog_session, selected_catalogs=store) + + for _attempt in range(2): + with pytest.raises(MetadataCatalogVersionNotFoundError): + service.list_parameters("us", "9.9.9") + with pytest.raises(UnsupportedPreviewCountryError): + service.list_parameters("ca") + with pytest.raises(InvalidPolicyEngineVersionError): + service.list_parameters("us", "not a version") + + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + assert service.list_parameters("us", "9.9.9").policyengine_version == "9.9.9" + + +def test_each_country_and_version_keeps_its_own_selection( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + service = _service(catalog_session, selected_catalogs=store) + + running_us = service.get_model_by_country("us") + named_us = service.get_model_by_country("us", "9.9.9") + running_uk = service.get_model_by_country("uk") + + assert (running_us.policyengine_version, named_us.policyengine_version) == ( + POLICYENGINE_VERSION, + "9.9.9", + ) + assert named_us.model_version.current_law_id == 7 + assert running_uk.model.name == "policyengine-uk" + assert service.get_model_by_country("us", "9.9.9") == named_us + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 15355a4efb32997a682677724ee93d7453842e5d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:01:24 +0400 Subject: [PATCH 067/118] Keep each whole-version response's ETag, under the version it was read from The response cache kept only bodies, so the caching route hashed the 3.9 MB US variables body again on every request, and a request without a version and one naming the running version kept two copies. Each kept response now carries its ETag, computed once when it is kept, which the caching route reuses, and responses are kept under the exact version they were read from: a request without a version records which version the process runs, so it and an explicit request for that version share one response. The route helper is generic over the result type, so mypy checks each operation's result against its response model. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 13 +- .../v2/metadata/catalog_routes.py | 15 +- .../fastapi_routes/v2/metadata/common.py | 66 ++++++--- .../v2/metadata/response_cache.py | 133 +++++++++++++----- tests/unit/v2/test_metadata_response_cache.py | 93 ++++++++---- tests/unit/v2/test_metadata_routes.py | 36 ++++- 6 files changed, 258 insertions(+), 98 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d511af3aa..d8978e7cb 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -107,12 +107,13 @@ weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. Each API process also keeps the serialized bodies of `/v2/model-catalog` and -`/v2/model-catalog/variables` once built, for up to eight route, country and -requested-version combinations (least recently used dropped first; a request -without `policyengine_version` is keyed to the process's running version, -which never changes while it runs). Only the first successful request per -process reads the database; concurrent first requests wait for one read, and -a failed read is not kept. Measured locally against the full 6.10.0 US +`/v2/model-catalog/variables` once built, each with its `ETag` computed once, +for up to eight route, country and version combinations (least recently used +dropped first). A response is kept under the exact version it was read from, +so a request without `policyengine_version` (the process's running version, +which never changes while it runs) and one naming that version share it. Only +the first successful request per process reads the database; concurrent first +requests wait for one read, and a failed read is not kept. Measured locally against the full 6.10.0 US catalog, the variables route takes about 160 ms when read and about 25 ms when kept, most of it compression. diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index bef2ba326..6f2259785 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -18,8 +18,13 @@ MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, + MetadataResourceSuccessResponse, ) from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery +from policyengine_api.services.v2.metadata.types import ( + MetadataCatalogVariablesResult, + MetadataModelCatalogResult, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -61,10 +66,11 @@ def build_v2_model_catalog_router( ) def get_model_catalog( query: CatalogQueryDependency, - ) -> MetadataModelCatalogResponse | Response: + ) -> MetadataResourceSuccessResponse[MetadataModelCatalogResult] | Response: return read_whole_version_resource( dependencies, - ("model-catalog", query.country_id, query.policyengine_version), + "model-catalog", + query, MetadataModelCatalogResponse, lambda reader: reader.get_model_catalog( query.country_id, query.policyengine_version @@ -85,10 +91,11 @@ def get_model_catalog( ) def get_catalog_variables( query: CatalogQueryDependency, - ) -> MetadataCatalogVariablesResponse | Response: + ) -> MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] | Response: return read_whole_version_resource( dependencies, - ("model-catalog/variables", query.country_id, query.policyengine_version), + "model-catalog/variables", + query, MetadataCatalogVariablesResponse, lambda reader: reader.get_catalog_variables( query.country_id, query.policyengine_version diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index ce5314c0d..d10f4072b 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,8 @@ from __future__ import annotations import contextlib -import hashlib from datetime import UTC, datetime, timedelta -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, Protocol, TypeVar from fastapi.routing import APIRoute from pydantic import BaseModel @@ -23,6 +22,14 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + body_entity_tag, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataResourceSuccessResponse, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -38,9 +45,7 @@ NativeRouteDependencies, V2MetadataResourceReader, ) - from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( - ResponseKey, - ) + from policyengine_api.query_parameters import CatalogQuery ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { @@ -70,8 +75,18 @@ } +class CatalogVersionResult(Protocol): + """A whole-version result, which names the catalog version it was read from.""" + + @property + def policyengine_version(self) -> str: + """The exact PolicyEngine.py version of the catalog read.""" + ... + + ResponseT = TypeVar("ResponseT", bound=BaseModel) ReadT = TypeVar("ReadT") +CatalogResultT = TypeVar("CatalogResultT", bound=CatalogVersionResult) EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 """How long a response for a named catalog version may be reused (a day). @@ -138,7 +153,7 @@ def with_cache_headers(request: Request, response: Response) -> Response: midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with no body. """ - entity_tag = f'"{hashlib.sha256(response.body).hexdigest()[:32]}"' + entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( EXPLICIT_VERSION_MAX_AGE_SECONDS if "policyengine_version" in request.query_params @@ -180,31 +195,42 @@ def read_resource( def read_whole_version_resource( dependencies: NativeRouteDependencies, - key: ResponseKey, - response_type: type[ResponseT], - operation: Callable[[V2MetadataResourceReader], object], -) -> ResponseT | Response: + route: str, + query: CatalogQuery, + response_type: type[MetadataResourceSuccessResponse[CatalogResultT]], + operation: Callable[[V2MetadataResourceReader], CatalogResultT], +) -> MetadataResourceSuccessResponse[CatalogResultT] | Response: """Serve a whole catalog-version response, built once per process when kept. - With the dependencies' response cache, the serialized body for ``key`` is - read and built only on the first successful request; failures are served - as ``read_resource`` serves them and never kept. Without a cache this is + With the dependencies' response cache, the serialized body and its ETag + for ``route``, the query's country and the version it selects are read + and built only on the first successful request; failures are served as + ``read_resource`` serves them and never kept. Without a cache this is ``read_resource``. """ cache = dependencies.v2_metadata_response_cache if cache is None: return read_resource(dependencies, response_type, operation) - def build() -> bytes | JSONResponse: + def build() -> BuiltResponse | JSONResponse: result = read_resource(dependencies, response_type, operation) - if isinstance(result, BaseModel): - return result.model_dump_json().encode() + if isinstance(result, MetadataResourceSuccessResponse): + return BuiltResponse( + body=result.model_dump_json().encode(), + policyengine_version=result.result.policyengine_version, + ) return result - built = cache.get_or_build(key, build) - if isinstance(built, bytes): - return Response(content=built, media_type="application/json") - return built + kept = cache.get_or_build( + route, query.country_id, query.policyengine_version, build + ) + if isinstance(kept, KeptResponse): + return Response( + content=kept.body, + media_type="application/json", + headers={"ETag": kept.entity_tag}, + ) + return kept def run_metadata_read( diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index ae688ae4c..72d3cf9ba 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -1,9 +1,11 @@ -"""A bounded, process-wide store of whole catalog-version response bodies.""" +"""A bounded, process-wide store of whole catalog-version responses.""" from __future__ import annotations +import hashlib import threading from collections import OrderedDict +from dataclasses import dataclass from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -11,63 +13,120 @@ from starlette.responses import JSONResponse -ResponseKey = tuple[str, str, str | None] -"""A route name, a country ID and the requested PolicyEngine.py version. +DEFAULT_RESPONSE_CAPACITY = 8 -A version of None means the running version, which is fixed for the process. -""" +KeptResponseKey = tuple[str, str, str] +"""A route name, a country ID and the exact PolicyEngine.py version served.""" + + +def body_entity_tag(body: bytes) -> str: + """Return the strong ETag of a response body: a hash of its exact bytes.""" + return f'"{hashlib.sha256(body).hexdigest()[:32]}"' + + +@dataclass(frozen=True) +class BuiltResponse: + """A serialized success body and the catalog version it was read from.""" + + body: bytes + policyengine_version: str -DEFAULT_RESPONSE_CAPACITY = 8 + +@dataclass(frozen=True) +class KeptResponse: + """A kept success body with its ETag, computed once when it was kept.""" + + body: bytes + entity_tag: str class MetadataResponseCache: - """Keep the serialized bodies of whole-version responses, built once each. + """Keep whole-version responses, built once each per process. A published catalog version never changes, so a body built for a route, - country and version stays valid for the life of the process. The most - recently used ``capacity`` bodies are kept. Requests for a body being built - wait for that build instead of starting another, and a failed read (a v2 - error response) is returned but not kept, so the next request tries again. + country and version stays valid for the life of the process. Responses + are kept under the exact version they were read from: a request without a + version (the running version, fixed for the process) and one naming that + version share one response. The most recently used ``capacity`` responses + are kept. Requests for a response being built wait for that build instead + of starting another, and a failed read (a v2 error response) is returned + but not kept, so the next request tries again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: - """Keep at most ``capacity`` bodies (at least one).""" + """Keep at most ``capacity`` responses (at least one).""" if capacity < 1: msg = "capacity must be positive" raise ValueError(msg) self._capacity = capacity - self._bodies: OrderedDict[ResponseKey, bytes] = OrderedDict() + self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() + self._running_versions: dict[tuple[str, str], str] = {} self._lock = threading.Lock() - self._build_locks: dict[ResponseKey, threading.Lock] = {} + self._build_locks: dict[tuple[str, str, str | None], threading.Lock] = {} - def _stored(self, key: ResponseKey) -> bytes | None: + def _stored( + self, route: str, country_id: str, requested_version: str | None + ) -> KeptResponse | None: with self._lock: - body = self._bodies.get(key) - if body is not None: - self._bodies.move_to_end(key) - return body + version = ( + requested_version + if requested_version is not None + else self._running_versions.get((route, country_id)) + ) + if version is None: + return None + key = (route, country_id, version) + kept = self._responses.get(key) + if kept is not None: + self._responses.move_to_end(key) + return kept def get_or_build( self, - key: ResponseKey, - build: Callable[[], bytes | JSONResponse], - ) -> bytes | JSONResponse: - """Return the body kept for ``key``, calling ``build`` only when none is.""" - body = self._stored(key) - if body is not None: - return body + route: str, + country_id: str, + requested_version: str | None, + build: Callable[[], BuiltResponse | JSONResponse], + ) -> KeptResponse | JSONResponse: + """Return the kept response, calling ``build`` only when none is kept. + + ``requested_version`` None means the running version. + """ + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + build_key = (route, country_id, requested_version) with self._lock: - build_lock = self._build_locks.setdefault(key, threading.Lock()) + build_lock = self._build_locks.setdefault(build_key, threading.Lock()) with build_lock: - body = self._stored(key) - if body is not None: - return body + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept built = build() with self._lock: - self._build_locks.pop(key, None) - if isinstance(built, bytes): - self._bodies[key] = built - self._bodies.move_to_end(key) - while len(self._bodies) > self._capacity: - self._bodies.popitem(last=False) - return built + self._build_locks.pop(build_key, None) + if not isinstance(built, BuiltResponse): + return built + return self._keep(route, country_id, requested_version, built) + + def _keep( + self, + route: str, + country_id: str, + requested_version: str | None, + built: BuiltResponse, + ) -> KeptResponse: + key = (route, country_id, built.policyengine_version) + with self._lock: + if requested_version is None: + self._running_versions[(route, country_id)] = built.policyengine_version + kept = self._responses.get(key) + if kept is None: + kept = KeptResponse( + body=built.body, entity_tag=body_entity_tag(built.body) + ) + self._responses[key] = kept + self._responses.move_to_end(key) + while len(self._responses) > self._capacity: + self._responses.popitem(last=False) + return kept diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index a96d7b005..9cfc8fd8c 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -1,4 +1,4 @@ -"""The process-wide store of whole catalog-version response bodies.""" +"""The process-wide store of whole catalog-version responses.""" from __future__ import annotations @@ -9,42 +9,79 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, MetadataResponseCache, + body_entity_tag, ) +RUNNING_VERSION = "6.10.0" -def _counting_build(body: bytes, builds: list[str]): - def build() -> bytes: + +def _counting_build( + builds: list[str], *, body: bytes = b'{"a":1}', version: str = RUNNING_VERSION +): + def build() -> BuiltResponse: builds.append("built") - return body + return BuiltResponse(body=body, policyengine_version=version) return build -def test_a_body_is_built_once_per_key_and_reused() -> None: +def test_a_response_is_built_once_and_kept_with_its_entity_tag() -> None: cache = MetadataResponseCache() builds: list[str] = [] - key = ("variables", "us", "6.10.0") - first = cache.get_or_build(key, _counting_build(b'{"a":1}', builds)) - second = cache.get_or_build(key, _counting_build(b'{"b":2}', builds)) + first = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds, body=b"{}") + ) - assert first == b'{"a":1}' - assert second == b'{"a":1}' + assert first == KeptResponse( + body=b'{"a":1}', entity_tag=body_entity_tag(b'{"a":1}') + ) + assert second is first assert builds == ["built"] +@pytest.mark.parametrize("requested_first", [None, RUNNING_VERSION]) +def test_the_running_version_and_its_number_share_one_kept_response( + requested_first: str | None, +) -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + requested_second = RUNNING_VERSION if requested_first is None else None + + first = cache.get_or_build( + "variables", "us", requested_first, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + third = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + + assert second is first + assert third is first + assert builds == (["built"] if requested_first is None else ["built", "built"]) + + def test_different_routes_countries_and_versions_are_kept_apart() -> None: cache = MetadataResponseCache() builds: list[str] = [] - for key in ( + for route, country_id, version in ( ("variables", "us", "6.10.0"), ("variables", "uk", "6.10.0"), - ("variables", "us", None), + ("variables", "us", "6.9.0"), ("model-catalog", "us", "6.10.0"), ): - cache.get_or_build(key, _counting_build(repr(key).encode(), builds)) + cache.get_or_build( + route, country_id, version, _counting_build(builds, version=version) + ) assert len(builds) == 4 @@ -58,44 +95,40 @@ def failing_build() -> JSONResponse: builds.append("failed") return failure - first = cache.get_or_build(("variables", "us", None), failing_build) - second = cache.get_or_build( - ("variables", "us", None), _counting_build(b"{}", builds) - ) + first = cache.get_or_build("variables", "us", None, failing_build) + second = cache.get_or_build("variables", "us", None, _counting_build(builds)) assert first is failure - assert second == b"{}" + assert isinstance(second, KeptResponse) assert builds == ["failed", "built"] -def test_the_least_recently_used_body_is_dropped_beyond_capacity() -> None: +def test_the_least_recently_used_response_is_dropped_beyond_capacity() -> None: cache = MetadataResponseCache(capacity=2) builds: list[str] = [] - oldest, middle, newest = (("variables", country, None) for country in "abc") - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(middle, _counting_build(b"2", builds)) - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(newest, _counting_build(b"3", builds)) - cache.get_or_build(oldest, _counting_build(b"1", builds)) - cache.get_or_build(middle, _counting_build(b"2", builds)) + def get(country_id: str) -> None: + cache.get_or_build("variables", country_id, "6.10.0", _counting_build(builds)) + + for country_id in ("a", "b", "a", "c", "a", "b"): + get(country_id) assert builds == ["built", "built", "built", "built"] -def test_concurrent_requests_for_one_key_build_it_once() -> None: +def test_concurrent_requests_for_one_response_build_it_once() -> None: cache = MetadataResponseCache() builds: list[str] = [] started = threading.Barrier(4) - def slow_build() -> bytes: + def slow_build() -> BuiltResponse: builds.append("built") time.sleep(0.05) - return b"{}" + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) def request() -> None: started.wait() - cache.get_or_build(("variables", "us", None), slow_build) + cache.get_or_build("variables", "us", None, slow_build) threads = [threading.Thread(target=request) for _ in range(4)] for thread in threads: diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 96d504804..c23ca5099 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1087,7 +1087,10 @@ def factory() -> ResourceReader: first_responses = [client.get(path) for path in paths] repeated_responses = [client.get(path) for path in paths] - assert len(readers) == len(paths) + # The explicit request names the running version (4.20.3), so it shares + # the response the request without a version read: three reads in all. + assert len(readers) == len(paths) - 1 + assert first_responses[2].content == first_responses[1].content for first, repeated in zip(first_responses, repeated_responses, strict=True): assert repeated.status_code == 200 assert repeated.content == first.content @@ -1168,3 +1171,34 @@ def test_current_parameter_values_are_reused_only_until_the_next_utc_day( assert response.status_code == 200 assert response.headers["cache-control"] == expected_cache_control + + +def test_a_kept_response_is_served_without_hashing_its_body_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common, response_cache + + hashed_bodies: list[int] = [] + original_entity_tag = response_cache.body_entity_tag + + def counting_entity_tag(body: bytes) -> str: + hashed_bodies.append(len(body)) + return original_entity_tag(body) + + monkeypatch.setattr(response_cache, "body_entity_tag", counting_entity_tag) + monkeypatch.setattr(common, "body_entity_tag", counting_entity_tag) + results, _ids = _resource_results() + client = _client( + lambda: ResourceReader(results), response_cache=MetadataResponseCache() + ) + + first = client.get("/v2/model-catalog/variables?country_id=us") + hashed_after_first = len(hashed_bodies) + second = client.get( + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert hashed_after_first == 1 + assert len(hashed_bodies) == 1 + assert second.headers["etag"] == first.headers["etag"] + assert second.content == first.content From 038d593212db39b4032e0ef07965debc95cfab28 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:18:37 +0400 Subject: [PATCH 068/118] Read every v2 metadata query through the shared query models A malformed policyengine_version returned 422 from the routes that already used the shared query models but 400 from /v2/variables, /v2/regions, /v2/datasets, the model routes, the detail routes, the parameter tree and /v2/economy-options. The API routes guidance makes the shared definitions own each parameter's validation, so every metadata route now reads its query through them (CatalogQuery, CatalogVersionQuery for the route with the country in its path, CatalogPageQuery and resource models for variables, regions and the parameter tree), with the typed dependencies defined once. Unknown and repeated scalar parameters, a missing or unsupported country_id and a malformed version are rejected with 422 before any read on every route. The release check and the migration document now expect 422. Co-Authored-By: Claude Opus 5.5 --- ...1f-116-catalog-routes-on-demand.changed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 20 +++-- .../v2/metadata/catalog_routes.py | 17 ++-- .../v2/metadata/geography_routes.py | 72 ++++++++--------- .../v2/metadata/model_routes.py | 80 ++++++++----------- .../v2/metadata/parameter_routes.py | 57 +++++-------- .../v2/metadata/query_dependencies.py | 53 ++++++++++++ policyengine_api/query_parameters.py | 41 +++++++++- tests/integration/test_live_v2_metadata.py | 2 +- tests/integration/test_v2_metadata_routes.py | 2 +- tests/unit/v2/test_metadata_routes.py | 71 ++++++++++++++++ 11 files changed, 272 insertions(+), 145 deletions(-) create mode 100644 policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md index 62dc783e3..553c9f234 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -1 +1 @@ -/v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. Both read their query through the shared query models, so unknown or repeated parameters, an unsupported country_id and a malformed policyengine_version now return 422. +Every v2 metadata route reads its query through the shared query models, so unknown or repeated parameters, a missing or unsupported country_id and a malformed policyengine_version return 422 before any read (unsupported countries and malformed versions returned 400 before, and unknown parameters were ignored). /v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d8978e7cb..2ecef27b2 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -68,12 +68,18 @@ carry the same `economy` and `household` flags. `GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page only those exact parameters, and `GET /v2/parameter-values` repeated -`parameter_name` keys to page only those parameters' values. Both routes now -read their query through the shared query models (`ParameterCollectionQuery` -and `ParameterValueCollectionQuery`): unknown and repeated scalar parameters, -an unsupported `country_id` and a malformed `policyengine_version` are -rejected with HTTP 422 before any read; before, the last two returned HTTP 400 -from the read and unknown parameters were ignored. +`parameter_name` keys to page only those parameters' values. + +Every v2 metadata route reads its query through the shared query models in +`policyengine_api.query_parameters` (`CatalogQuery`, `CatalogPageQuery` and the +resource-specific models built on them): unknown parameters, repeated scalar +parameters, a missing or unsupported `country_id`, a malformed +`policyengine_version` and out-of-range `offset` or `limit` values are +rejected with HTTP 422 before any read. Before, the routes other than the +`/v2/model-catalog` ones ignored unknown parameters, and an unsupported country +or malformed version returned HTTP 400 from the read. A country given in the +path (`/v2/tax-benefit-models/by-country/{country_id}`) that is not supported +still returns HTTP 400. ## Database round trips per request @@ -353,7 +359,7 @@ A successful response has HTTP 200, `status: "ok"`, `message: null`, and a typed `result`. Collection results contain `policyengine_version`, `items`, `offset`, `limit`, and `has_more`; detail results contain `policyengine_version` and `item`. A malformed or noncanonical explicit -version returns a typed HTTP 400 error. A valid explicit version that has not +version returns a typed HTTP 422 error. A valid explicit version that has not been published for the country returns a typed HTTP 404 error. An absent catalog for the candidate's installed default version returns a typed HTTP 503 service error. None of these outcomes reads or repairs v1. Unsupported diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 6f2259785..1ef94301e 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -2,25 +2,27 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING -from fastapi import APIRouter, Depends +from fastapi import APIRouter from starlette.responses import JSONResponse, Response -from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( ERROR_RESPONSES, CacheableMetadataRoute, read_resource, read_whole_version_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogParameterQueryDependency, + CatalogQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataCatalogParametersResponse, MetadataCatalogVariablesResponse, MetadataModelCatalogResponse, MetadataResourceSuccessResponse, ) -from policyengine_api.query_parameters import CatalogParameterQuery, CatalogQuery from policyengine_api.services.v2.metadata.types import ( MetadataCatalogVariablesResult, MetadataModelCatalogResult, @@ -29,13 +31,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -CatalogQueryDependency = Annotated[ - CatalogQuery, Depends(query_dependency(CatalogQuery)) -] -CatalogParameterQueryDependency = Annotated[ - CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) -] - def build_v2_model_catalog_router( dependencies: NativeRouteDependencies, diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index f5add9640..4665ea67c 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( @@ -13,6 +13,11 @@ CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + RegionCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, @@ -20,14 +25,10 @@ MetadataRegionDetailResponse, MetadataRegionPageResponse, ) -from policyengine_api.services.v2.metadata.types import MetadataRegionType if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_geography_router( dependencies: NativeRouteDependencies, @@ -49,19 +50,16 @@ def build_v2_metadata_geography_router( summary="List logical inputs from a selected PolicyEngine.py catalog", ) def list_datasets( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataDatasetPageResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetPageResponse, lambda reader: reader.list_datasets( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -72,16 +70,15 @@ def list_datasets( ) def get_dataset( dataset_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataDatasetDetailResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetDetailResponse, lambda reader: reader.get_dataset( - country_id, + query.country_id, dataset_id, - policyengine_version, + query.policyengine_version, ), ) @@ -91,21 +88,19 @@ def get_dataset( summary="List regions from a selected PolicyEngine.py catalog", ) def list_regions( - country_id: str, - policyengine_version: str | None = None, - region_type: MetadataRegionType | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: RegionCollectionQueryDependency, ) -> MetadataRegionPageResponse | JSONResponse: return read_resource( dependencies, MetadataRegionPageResponse, lambda reader: reader.list_regions( - country_id, - policyengine_version, - region_type=region_type.value if region_type is not None else None, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + region_type=( + None if query.region_type is None else query.region_type.value + ), + offset=query.offset, + limit=query.limit, ), ) @@ -116,16 +111,15 @@ def list_regions( ) def get_region_by_code( region_code: str, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region_by_code( - country_id, + query.country_id, region_code, - policyengine_version, + query.policyengine_version, ), ) @@ -136,16 +130,15 @@ def get_region_by_code( ) def get_region( region_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region( - country_id, + query.country_id, region_id, - policyengine_version, + query.policyengine_version, ), ) @@ -155,15 +148,14 @@ def get_region( summary="Get compact economy-selection options from a selected catalog", ) def get_economy_options( - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataEconomyOptionsResponse | JSONResponse: return read_resource( dependencies, MetadataEconomyOptionsResponse, lambda reader: reader.get_economy_options( - country_id, - policyengine_version, + query.country_id, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index cbb3a8ef5..63e72ce56 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -2,10 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( @@ -13,6 +13,12 @@ CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + CatalogVersionQueryDependency, + VariableCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, @@ -26,9 +32,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_model_router( dependencies: NativeRouteDependencies, @@ -50,19 +53,16 @@ def build_v2_metadata_model_router( summary="List models for one PolicyEngine.py catalog", ) def list_models( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelPageResponse, lambda reader: reader.list_models( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -73,14 +73,14 @@ def list_models( ) def get_model_by_country( country_id: str, - policyengine_version: str | None = None, + query: CatalogVersionQueryDependency, ) -> MetadataModelSelectionResponse | JSONResponse: return read_resource( dependencies, MetadataModelSelectionResponse, lambda reader: reader.get_model_by_country( country_id, - policyengine_version, + query.policyengine_version, ), ) @@ -91,16 +91,15 @@ def get_model_by_country( ) def get_model( model_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelDetailResponse, lambda reader: reader.get_model( - country_id, + query.country_id, model_id, - policyengine_version, + query.policyengine_version, ), ) @@ -110,19 +109,16 @@ def get_model( summary="List selected PolicyEngine.py model versions", ) def list_model_versions( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelVersionPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionPageResponse, lambda reader: reader.list_model_versions( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -133,16 +129,15 @@ def list_model_versions( ) def get_model_version( version_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelVersionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionDetailResponse, lambda reader: reader.get_model_version( - country_id, + query.country_id, version_id, - policyengine_version, + query.policyengine_version, ), ) @@ -152,21 +147,17 @@ def get_model_version( summary="List variables from a selected PolicyEngine.py catalog", ) def list_variables( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: VariableCollectionQueryDependency, ) -> MetadataVariablePageResponse | JSONResponse: return read_resource( dependencies, MetadataVariablePageResponse, lambda reader: reader.list_variables( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + search=query.search, ), ) @@ -177,16 +168,15 @@ def list_variables( ) def get_variable( variable_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataVariableDetailResponse | JSONResponse: return read_resource( dependencies, MetadataVariableDetailResponse, lambda reader: reader.get_variable( - country_id, + query.country_id, variable_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index ceb22e985..d9ddeb336 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,19 +2,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Depends, Query, Request +from fastapi import APIRouter, Request from starlette.responses import JSONResponse -from policyengine_api.fastapi_routes.query_parameters import query_dependency from policyengine_api.fastapi_routes.v2.metadata.common import ( DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogQueryDependency, + ParameterChildrenQueryDependency, + ParameterCollectionQueryDependency, + ParameterValueCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, MetadataParameterDetailResponse, @@ -22,10 +27,6 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.query_parameters import ( - ParameterCollectionQuery, - ParameterValueCollectionQuery, -) from policyengine_api.services.v2.metadata.types import ( ParameterFilter, ParameterValueFilter, @@ -34,18 +35,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - - -ParameterCollectionQueryDependency = Annotated[ - ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) -] -ParameterValueCollectionQueryDependency = Annotated[ - ParameterValueCollectionQuery, - Depends(query_dependency(ParameterValueCollectionQuery)), -] - def build_v2_metadata_parameter_router( dependencies: NativeRouteDependencies, @@ -90,21 +79,17 @@ def list_parameters( summary="List direct children of one parameter path", ) def list_parameter_children( - country_id: str, - parent_path: str = "", - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: ParameterChildrenQueryDependency, ) -> MetadataParameterChildPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterChildPageResponse, lambda reader: reader.list_parameter_children( - country_id, - policyengine_version, - parent_path=parent_path, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + parent_path=query.parent_path, + offset=query.offset, + limit=query.limit, ), ) @@ -115,16 +100,15 @@ def list_parameter_children( ) def get_parameter( parameter_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterDetailResponse, lambda reader: reader.get_parameter( - country_id, + query.country_id, parameter_id, - policyengine_version, + query.policyengine_version, ), ) @@ -166,16 +150,15 @@ def list_parameter_values( ) def get_parameter_value( value_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterValueDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterValueDetailResponse, lambda reader: reader.get_parameter_value( - country_id, + query.country_id, value_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py new file mode 100644 index 000000000..b52d18d1e --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py @@ -0,0 +1,53 @@ +"""Typed FastAPI dependencies for the shared v2 metadata query contracts. + +Route modules import these aliases into their own globals, because FastAPI +resolves an endpoint's string annotations against its module only. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CatalogPageQuery, + CatalogParameterQuery, + CatalogQuery, + CatalogVersionQuery, + ParameterChildrenQuery, + ParameterCollectionQuery, + ParameterValueCollectionQuery, + RegionCollectionQuery, + VariableCollectionQuery, +) + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] +CatalogVersionQueryDependency = Annotated[ + CatalogVersionQuery, Depends(query_dependency(CatalogVersionQuery)) +] +CatalogPageQueryDependency = Annotated[ + CatalogPageQuery, Depends(query_dependency(CatalogPageQuery)) +] +VariableCollectionQueryDependency = Annotated[ + VariableCollectionQuery, Depends(query_dependency(VariableCollectionQuery)) +] +RegionCollectionQueryDependency = Annotated[ + RegionCollectionQuery, Depends(query_dependency(RegionCollectionQuery)) +] +ParameterChildrenQueryDependency = Annotated[ + ParameterChildrenQuery, Depends(query_dependency(ParameterChildrenQuery)) +] +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index ec614f109..0d94c8484 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -30,6 +30,7 @@ validate_policyengine_version, ) from policyengine_api.services.economic_impact_request import LocalAreaBreakdown +from policyengine_api.services.v2.metadata.types import MetadataRegionType from policyengine_api.spm import SPMSelection from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, @@ -232,6 +233,42 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CatalogVersionQuery(StrictQueryParameters): + """An optional exact catalog version, for routes with the country in the path.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class CatalogPageQuery(CatalogQuery, PaginationQuery): + """Query contract for one page of a catalog's resources.""" + + +class VariableCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's variables.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + + +class RegionCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's regions.""" + + region_type: MetadataRegionType | None = Field( + default=None, description="Keep only regions of this type" + ) + + +class ParameterChildrenQuery(CatalogPageQuery): + """Query contract for a page of one parameter path's direct children.""" + + parent_path: str = Field( + default="", + description="Dotted path whose direct children are listed; empty for the root", + ) + + MAXIMUM_PARAMETER_NAMES = 100 ParameterName = Annotated[ str, @@ -249,7 +286,7 @@ class CatalogParameterQuery(CatalogQuery): ) -class ParameterCollectionQuery(CatalogQuery, PaginationQuery): +class ParameterCollectionQuery(CatalogPageQuery): """Query contract for a page of a catalog's parameters.""" search: str | None = Field( @@ -263,7 +300,7 @@ class ParameterCollectionQuery(CatalogQuery, PaginationQuery): ) -class ParameterValueCollectionQuery(CatalogQuery, PaginationQuery): +class ParameterValueCollectionQuery(CatalogPageQuery): """Query contract for a page of a catalog's current-law parameter values.""" parameter_id: ResourceId | None = None diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 803e43b05..1b2b8e183 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -110,7 +110,7 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: params={"country_id": "us", "policyengine_version": "not a version"}, ) - assert response.status_code == 400, response.text[:500] + assert response.status_code == 422, response.text[:500] payload = response.json() assert payload["status"] == "error" assert payload["message"] diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index e9531fd3f..fa5ea761b 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -328,7 +328,7 @@ def test_postgres_resources_distinguish_invalid_and_absent_versions( params={"country_id": "us", "policyengine_version": "4.99.0"}, ) - assert invalid.status_code == 400 + assert invalid.status_code == 422 assert invalid.json()["status"] == "error" assert invalid.json()["message"] assert absent.status_code == 404 diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index c23ca5099..d3b5ab349 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1202,3 +1202,74 @@ def counting_entity_tag(body: bytes) -> str: assert len(hashed_bodies) == 1 assert second.headers["etag"] == first.headers["etag"] assert second.content == first.content + + +def _query_rejections(path: str) -> list[str]: + separator = "&" if "?" in path else "?" + rejections = [ + f"{path}{separator}policyengine_version=not%20a%20version", + f"{path}{separator}unexpected=1", + f"{path}{separator}policyengine_version=4.20.3&policyengine_version=4.20.3", + ] + if "country_id=us" in path: + rejections.append(path.replace("country_id=us", "country_id=ca")) + rejections.append( + path.replace("country_id=us&", "").replace("?country_id=us", "") + ) + return rejections + + +def test_every_metadata_route_rejects_invalid_queries_before_reading() -> None: + _results, ids = _resource_results() + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + client = _client(factory) + for path in _every_resource_path(ids): + for rejected in _query_rejections(path): + response = client.get(rejected) + assert response.status_code == 422, (rejected, response.text) + assert response.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert calls == [] + + +def test_every_metadata_route_declares_the_shared_catalog_query() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + parameters = { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + if parameter["in"] == "query" + } + version = parameters["policyengine_version"] + assert version["required"] is False, path + assert version["schema"]["anyOf"][0]["maxLength"] == 128, path + if path != "/v2/tax-benefit-models/by-country/{country_id}": + country = parameters["country_id"] + assert country["required"] is True, path + assert country["schema"]["enum"] == ["us", "uk"], path From ea169b046a0679e7b53bf6e212e6faa47897dffa Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:27:26 +0400 Subject: [PATCH 069/118] Type the new metadata reads and remove their duplicated lookups The current-law value and node-label connectors returned list[Any]; they now return CurrentLawValueRow and ParameterNodeLabelRow named tuples, so mypy checks every use of their fields. get_model_catalog re-implemented get_economy_options's national-region and default-dataset lookup; both now share _validated_national_dataset with a national_region transformation, and differ only in how they load the dataset. The part parameter paths and the found parameter names are computed once in the service and passed to the transformations instead of being recomputed there. Co-Authored-By: Claude Opus 5.5 --- .../reads_parameter_nodes.py | 8 +- .../database_connectors/reads_parameters.py | 13 ++- .../services/v2/metadata/services.py | 103 +++++++++++------- .../services/v2/metadata/transformations.py | 42 ++++--- .../services/v2/metadata/types.py | 29 ++++- tests/unit/v2/test_metadata_service.py | 32 ++++++ 6 files changed, 160 insertions(+), 67 deletions(-) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py index 154a1a6eb..24968399a 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -2,12 +2,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from sqlmodel import Session, col, select from policyengine_api.data.v2.models import ParameterNode from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import ParameterNodeLabelRow if TYPE_CHECKING: from collections.abc import Collection @@ -16,14 +17,14 @@ def read_parameter_node_labels( session: Session, *, model_version_id: UUID, node_names: Collection[str] -) -> list[Any]: +) -> list[ParameterNodeLabelRow]: """Return the name and stored label of each named node of a model version. Names the version has no node for are absent. Rows are in name order. """ if not node_names: return [] - return read_rows( + rows = read_rows( session, select(col(ParameterNode.name), col(ParameterNode.label)) .where( @@ -32,3 +33,4 @@ def read_parameter_node_labels( ) .order_by(col(ParameterNode.name)), ) + return [ParameterNodeLabelRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 39ab71bd5..eadd87be9 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -2,13 +2,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import sqlalchemy as sa from sqlmodel import Session, col, select from policyengine_api.data.v2.models import Parameter, ParameterValue from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import CurrentLawValueRow if TYPE_CHECKING: from collections.abc import Collection @@ -170,16 +171,15 @@ def read_parameter_value( def read_current_law_values_by_parameter_name( session: Session, *, model_version_id: UUID, parameter_names: Collection[str] -) -> list[Any]: +) -> list[CurrentLawValueRow]: """Return the current-law values of the named parameters of a model version. - Each row has the parameter's name, the value's value_json, start_date and - end_date. Values tied to a policy or dynamic are excluded. Rows are ordered - by parameter name, then start date. + Values tied to a policy or dynamic are excluded. Rows are ordered by + parameter name, then start date. """ if not parameter_names: return [] - return read_rows( + rows = read_rows( session, select( col(Parameter.name), @@ -196,3 +196,4 @@ def read_current_law_values_by_parameter_name( ) .order_by(col(Parameter.name), col(ParameterValue.start_date)), ) + return [CurrentLawValueRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index c5a63a223..017e3e5b4 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -55,11 +55,13 @@ metadata_parameter_value, metadata_region, metadata_variable, + national_region, page_result, parameter_value_selection, part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( + FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParametersResult, MetadataCatalogVariablesResult, @@ -85,9 +87,10 @@ ) if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from uuid import UUID + from policyengine_api.data.v2.models import Dataset, Region from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) @@ -554,27 +557,17 @@ def get_economy_options( region, its default input dataset, the current-law id or the years. """ selected = self._select_catalog(country_id, policyengine_version) - regions = read_regions( - self._database_session.session, - model_version_id=selected.model_version.id, - ) - national_region = next( - (region for region in regions if region.code == country_id), None - ) - national_dataset = ( - read_input_dataset( - self._database_session.session, - model_version_id=selected.model_version.id, - dataset_id=national_region.default_dataset_id, - ) - if national_region is not None - else None - ) - dataset = validate_economy_options( + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + dataset = _validated_national_dataset( selected, country_id=country_id, regions=regions, - national_dataset=national_dataset, + find_dataset=lambda dataset_id: read_input_dataset( + session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), ) return metadata_economy_options( selected, regions=regions, national_dataset=dataset @@ -598,28 +591,19 @@ def get_model_catalog( datasets = read_input_datasets( session, model_version_id=selected.model_version.id ) - national_dataset = None - if regions: - national_region = next( - (region for region in regions if region.code == country_id), None - ) - national_dataset = validate_economy_options( + national_dataset = ( + _validated_national_dataset( selected, country_id=country_id, regions=regions, - national_dataset=( - next( - ( - dataset - for dataset in datasets - if dataset.id == national_region.default_dataset_id - ), - None, - ) - if national_region is not None - else None + find_dataset=lambda dataset_id: next( + (dataset for dataset in datasets if dataset.id == dataset_id), + None, ), ) + if regions + else None + ) return metadata_model_catalog( selected, regions=regions, @@ -640,13 +624,17 @@ def get_catalog_variables( variables = read_all_variables( session, model_version_id=selected.model_version.id ) + part_paths = part_parameter_paths(variables) part_values = read_current_law_values_by_parameter_name( session, model_version_id=selected.model_version.id, - parameter_names=part_parameter_paths(variables), + parameter_names=part_paths, ) return metadata_catalog_variables( - selected, variables=variables, part_values=part_values + selected, + variables=variables, + part_paths=part_paths, + part_values=part_values, ) def get_catalog_parameters( @@ -669,21 +657,52 @@ def get_catalog_parameters( parameters = read_parameters_by_name( session, model_version_id=model_version_id, parameter_names=requested_names ) - found_names = {parameter.name for parameter in parameters} + found_parameters = FoundParameters( + parameters=parameters, + names={parameter.name for parameter in parameters}, + ) value_rows = read_current_law_values_by_parameter_name( - session, model_version_id=model_version_id, parameter_names=found_names + session, + model_version_id=model_version_id, + parameter_names=found_parameters.names, ) node_rows = read_parameter_node_labels( session, model_version_id=model_version_id, node_names={ - ancestor for name in found_names for ancestor in ancestor_paths(name) + ancestor + for name in found_parameters.names + for ancestor in ancestor_paths(name) }, ) return metadata_catalog_parameters( selected, requested_names=requested_names, - parameters=parameters, + found_parameters=found_parameters, value_rows=value_rows, node_rows=node_rows, ) + + +def _validated_national_dataset( + selected: SelectedCatalog, + *, + country_id: str, + regions: list[Region], + find_dataset: Callable[[UUID], Dataset | None], +) -> Dataset: + """Return the national region's default dataset, validated for economy choices. + + find_dataset loads a dataset of the selected version by id. Raise + MetadataCatalogUnavailableError when the national region, its dataset or + the model-version options are missing. + """ + region = national_region(regions, country_id) + return validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + None if region is None else find_dataset(region.default_dataset_id) + ), + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 4b9167845..938027741 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -7,6 +7,8 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, + FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParameter, MetadataCatalogParametersResult, @@ -30,6 +32,7 @@ MetadataRegionType, MetadataTimePeriodOption, MetadataVariable, + ParameterNodeLabelRow, ParameterValueSelection, ) @@ -138,16 +141,20 @@ def part_parameter_paths(variables: list[Variable]) -> set[str]: def metadata_catalog_variables( - selected: SelectedCatalog, *, variables: list[Variable], part_values: list[Any] + selected: SelectedCatalog, + *, + variables: list[Variable], + part_paths: set[str], + part_values: list[CurrentLawValueRow], ) -> MetadataCatalogVariablesResult: """Build every variable of the selected version and its part parameters' values. - part_values holds rows of read_current_law_values_by_parameter_name, in - parameter-name and start-date order. Each path a variable names gets an - entry, empty when no row carries its name. + part_paths holds part_parameter_paths of the variables, and part_values the + current-law values of those parameters in name and start-date order. Each + path gets an entry, empty when no value carries its name. """ values_by_path = _dated_values_by_parameter_name( - part_values, parameter_names=part_parameter_paths(variables) + part_values, parameter_names=part_paths ) return MetadataCatalogVariablesResult( policyengine_version=selected.policyengine_version, @@ -158,12 +165,11 @@ def metadata_catalog_variables( def _dated_values_by_parameter_name( - value_rows: list[Any], *, parameter_names: set[str] + value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] ) -> dict[str, list[MetadataDatedValue]]: """Group value rows by parameter name, with an empty list for each other name. - value_rows holds rows of read_current_law_values_by_parameter_name, in - parameter-name and start-date order. + value_rows are in parameter-name and start-date order. """ values_by_name: dict[str, list[MetadataDatedValue]] = { name: [] for name in sorted(parameter_names) @@ -211,19 +217,20 @@ def metadata_catalog_parameters( selected: SelectedCatalog, *, requested_names: set[str], - parameters: list[Parameter], - value_rows: list[Any], - node_rows: list[Any], + found_parameters: FoundParameters, + value_rows: list[CurrentLawValueRow], + node_rows: list[ParameterNodeLabelRow], ) -> MetadataCatalogParametersResult: """Build the requested parameters with their values and ancestor labels. - value_rows holds rows of read_current_law_values_by_parameter_name and - node_rows rows of read_parameter_node_labels. + value_rows are the found parameters' current-law values and node_rows the + stored labels of their ancestor nodes. """ + parameters = found_parameters.parameters + found_names = found_parameters.names values_by_name = _dated_values_by_parameter_name( - value_rows, parameter_names={parameter.name for parameter in parameters} + value_rows, parameter_names=found_names ) - found_names = {parameter.name for parameter in parameters} return MetadataCatalogParametersResult( policyengine_version=selected.policyengine_version, catalog_format=selected.model_version.catalog_format, @@ -348,6 +355,11 @@ def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild] return items +def national_region(regions: list[Region], country_id: str) -> Region | None: + """Return the country's national region (its code is the country ID), or None.""" + return next((region for region in regions if region.code == country_id), None) + + def metadata_economy_options( selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset ) -> MetadataEconomyOptionsResult: diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 36dee6c43..fecb656fa 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,11 +5,14 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import Generic, Literal, TypeVar +from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue +if TYPE_CHECKING: + from policyengine_api.data.v2.models import Parameter + @dataclass(frozen=True) class ParameterFilter: @@ -50,6 +53,30 @@ class ParameterValueSelection: parameter_names: tuple[str, ...] | None = None +class CurrentLawValueRow(NamedTuple): + """One current-law value of a parameter, read with the parameter's name.""" + + name: str + value_json: Any + start_date: datetime + end_date: datetime | None + + +@dataclass(frozen=True) +class FoundParameters: + """The stored parameters a by-name read found, and their names.""" + + parameters: list[Parameter] + names: set[str] + + +class ParameterNodeLabelRow(NamedTuple): + """A parameter node's name and its stored label (None when it has none).""" + + name: str + label: str | None + + class StrictResponseModel(BaseModel): """Base for metadata results that rejects unknown fields and non-finite numbers.""" diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index d07e94bde..458ab34a4 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -39,10 +39,12 @@ from policyengine_api.services.v2.metadata.services import V2MetadataService from policyengine_api.services.v2.metadata.transformations import ancestor_paths from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, MetadataDatedValue, MetadataEntity, MetadataEnumValue, ParameterFilter, + ParameterNodeLabelRow, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -1085,6 +1087,36 @@ def test_each_country_and_version_keeps_its_own_selection( assert service.get_model_by_country("us", "9.9.9") == named_us +def test_value_and_node_label_reads_return_typed_rows( + catalog_session: Session, +) -> None: + from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 + read_parameter_node_labels, + ) + from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( # noqa: E501 + read_current_law_values_by_parameter_name, + ) + + model_version = _us_model_version(catalog_session) + values = read_current_law_values_by_parameter_name( + catalog_session, + model_version_id=model_version.id, + parameter_names={"gov.example.rate"}, + ) + labels = read_parameter_node_labels( + catalog_session, + model_version_id=model_version.id, + node_names={"gov.example", "gov.absent"}, + ) + + assert [type(row) for row in values] == [CurrentLawValueRow, CurrentLawValueRow] + assert [(row.name, row.value_json) for row in values] == [ + ("gov.example.rate", 0.1), + ("gov.example.rate", 0.2), + ] + assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 6e7e52ed9f78a76ccc03f06c3e51f2fd0ace1423 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:29:48 +0400 Subject: [PATCH 070/118] Let browser scripts read metadata responses' ETag The CORS middleware exposed only the request and observability IDs and WWW-Authenticate, so a script on another origin could not read the ETag of a v2 metadata response to revalidate it. ETag joins Access-Control-Expose-Headers (Cache-Control is readable without it). Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 4 +++- policyengine_api/asgi_factory.py | 2 ++ tests/unit/v2/test_metadata_routes.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 2ecef27b2..c39707c93 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -110,7 +110,9 @@ version, which changes on deploy, and may be reused for five minutes reused only until the next UTC midnight, whichever lifetime would otherwise apply. A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses -carry neither header. +carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so +browser scripts on other origins can read it (`Cache-Control` is readable +without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 1b986188e..cba4c00bc 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -480,6 +480,8 @@ def create_asgi_app( REQUEST_ID_HEADER, OBSERVABILITY_ID_HEADER, "WWW-Authenticate", + # v2 metadata responses carry an ETag a client may revalidate with. + "ETag", ], allow_credentials=False, max_age=600, diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index d3b5ab349..df0465c9d 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1273,3 +1273,22 @@ def test_every_metadata_route_declares_the_shared_catalog_query() -> None: country = parameters["country_id"] assert country["required"] is True, path assert country["schema"]["enum"] == ["us", "uk"], path + + +def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + "/v2/model-catalog?country_id=us", + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert "etag" in exposed_headers From 09bbd994c59626b5acd5c115116ab88b7349092e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:31:05 +0400 Subject: [PATCH 071/118] Record the slow existing routes' times on each release, even after a failure The release live test now also times a page of 500 parameters and one parameter's values, the two routes measured slow in production on 2026-10-21, and writes every time it reached to the job summary from a finally block, so a release whose check fails still records its measurements. The second variables request is labelled as possibly served by another instance, since its time need not be a kept response's. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 13 +-- tests/integration/test_live_v2_metadata.py | 93 ++++++++++++++-------- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index c39707c93..50ab0d623 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -370,11 +370,14 @@ return typed client errors. The same test loads each country's catalog in parts as a client starting cold would: `GET /v2/model-catalog` for the running version, then -`/v2/model-catalog/variables` for that exact version twice (the second served -from the process's kept body) and once more with `If-None-Match`, which must -return 304, then `/v2/model-catalog/parameters` for one parameter. It appends -each request's time and body size to the GitHub job summary, so every staging -and production release records these route times against its own catalog. +`/v2/model-catalog/variables` for that exact version twice (the second request +may be served by another instance, so its time is not necessarily a kept +response's) and once more with `If-None-Match`, which must return 304, then a +page of 500 parameters, one parameter's values (the two routes measured at +0.8–2.6 s and 0.5–0.65 s in production on 2026-10-21) and that parameter by +name. It appends each request's time and body size to the GitHub job summary, +even when an assertion fails, so every staging and production release records +these route times against its own catalog. Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 1b2b8e183..00ff143e2 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -154,60 +154,85 @@ def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> No expected_version = _installed_policyengine_version() route_times: list[tuple[str, float, httpx.Response]] = [] - catalog_response, seconds = _timed_get( - api_client, "/v2/model-catalog", params={"country_id": country_id} - ) - route_times.append( - ("/v2/model-catalog (running version)", seconds, catalog_response) + def timed_get(label: str, path: str, **request_options: object) -> httpx.Response: + response, seconds = _timed_get(api_client, path, **request_options) + route_times.append((label, seconds, response)) + return response + + # The times are recorded even when an assertion fails, so every release + # keeps the measurements it reached. + try: + _check_catalog_parts(timed_get, country_id, expected_version) + finally: + _record_route_times(country_id, route_times) + + +def _check_catalog_parts(timed_get, country_id: str, expected_version: str) -> None: + catalog = _assert_ok( + timed_get( + "/v2/model-catalog (running version)", + "/v2/model-catalog", + params={"country_id": country_id}, + ), + expected_version, ) - catalog = _assert_ok(catalog_response, expected_version) assert catalog["regions"] assert catalog["economy_options"]["region"] assert "variables" not in catalog pinned = {"country_id": country_id, "policyengine_version": expected_version} - variables_response, seconds = _timed_get( - api_client, "/v2/model-catalog/variables", params=pinned - ) - route_times.append(("/v2/model-catalog/variables", seconds, variables_response)) - repeated_variables, seconds = _timed_get( - api_client, "/v2/model-catalog/variables", params=pinned + variables_response = timed_get( + "/v2/model-catalog/variables", "/v2/model-catalog/variables", params=pinned ) - route_times.append( - ("/v2/model-catalog/variables (repeated)", seconds, repeated_variables) + second_variables = timed_get( + "/v2/model-catalog/variables (second request; another instance may serve it)", + "/v2/model-catalog/variables", + params=pinned, ) variables = _assert_ok(variables_response, expected_version) assert variables["variables"] assert variables_response.headers["cache-control"] == "public, max-age=86400" - assert repeated_variables.headers["etag"] == variables_response.headers["etag"] - revalidated, seconds = _timed_get( - api_client, + assert second_variables.headers["etag"] == variables_response.headers["etag"] + revalidated = timed_get( + "/v2/model-catalog/variables (revalidated)", "/v2/model-catalog/variables", params=pinned, headers={"If-None-Match": variables_response.headers["etag"]}, ) - route_times.append( - ("/v2/model-catalog/variables (revalidated)", seconds, revalidated) - ) assert revalidated.status_code == 304 - parameter_name = _assert_ok( - api_client.get("/v2/parameters", params={**pinned, "limit": 1}), + # The two existing routes measured slow in production on 2026-10-21. + page = _assert_ok( + timed_get( + "/v2/parameters (page of 500)", + "/v2/parameters", + params={**pinned, "limit": 500}, + ), expected_version, - )["items"][0]["name"] - parameters_response, seconds = _timed_get( - api_client, - "/v2/model-catalog/parameters", - params={**pinned, "name": parameter_name}, ) - route_times.append( - ("/v2/model-catalog/parameters (1 name)", seconds, parameters_response) + first_parameter = page["items"][0] + one_parameter_values = _assert_ok( + timed_get( + "/v2/parameter-values (one parameter)", + "/v2/parameter-values", + params={**pinned, "parameter_id": first_parameter["id"]}, + ), + expected_version, + ) + assert all( + value["parameter_id"] == first_parameter["id"] + for value in one_parameter_values["items"] + ) + + parameters = _assert_ok( + timed_get( + "/v2/model-catalog/parameters (1 name)", + "/v2/model-catalog/parameters", + params={**pinned, "name": first_parameter["name"]}, + ), + expected_version, ) - parameters = _assert_ok(parameters_response, expected_version) assert [parameter["name"] for parameter in parameters["parameters"]] == [ - parameter_name + first_parameter["name"] ] - assert parameters["parameters"][0]["values"] assert parameters["missing_names"] == [] - - _record_route_times(country_id, route_times) From b97982681d155e1806618ffe293d1900d2572a3b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:21:55 +0400 Subject: [PATCH 072/118] Read current parameter values and set their lifetime from one instant The parameter-values route marked a current=true request as depending on the day but passed no instant to the read: the service took the day from its own clock during the read, and the lifetime came from another clock reading after it. A read just before UTC midnight whose headers were written just after got the old day's values with a lifetime of nearly a day. The route now takes the request's instant once, passes it to the read as the filter's now, and records it; the caching route gives the response the seconds left until the UTC midnight after that instant, measured when the headers are written, and zero once that midnight has passed. No other metadata read depends on the time. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 8 +-- .../fastapi_routes/v2/metadata/common.py | 42 ++++++++++----- .../v2/metadata/parameter_routes.py | 8 +-- tests/integration/test_v2_metadata_routes.py | 51 +++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 24 +++++++++ .../v2/test_parameter_value_listing_route.py | 12 ++++- 6 files changed, 124 insertions(+), 21 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 50ab0d623..baf6b24f4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -106,9 +106,11 @@ route's response format or with the day it is read on, and the `ETag` lets a client revalidate after the day. A request without it selects the running version, which changes on deploy, and may be reused for five minutes (`max-age=300`). A response that depends on the request's UTC day, which is -`GET /v2/parameter-values?current=true` (the values in effect today), may be -reused only until the next UTC midnight, whichever lifetime would otherwise -apply. A request whose `If-None-Match` names the current `ETag` (strong or +`GET /v2/parameter-values?current=true` (the values in effect today), takes +the request's instant once: the read selects that day's values, and the +response may be reused only until the UTC midnight after that instant, +whichever lifetime would otherwise apply (`max-age=0` if its headers are +written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d10f4072b..ba7a74418 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -103,11 +103,12 @@ def policyengine_version(self) -> str: """ -DEPENDS_ON_UTC_DAY = "depends_on_utc_day" -"""The ``request.state`` attribute a route sets when its result depends on the day. +UTC_DAY_READ = "utc_day_read" +"""The ``request.state`` attribute holding the instant a day-dependent read used. -Such a response, for example current values on the request's UTC day, may be -reused only until the next UTC midnight, whatever version it names. +A route whose result depends on the UTC day, such as current parameter values, +sets it with ``take_utc_instant_for_read`` and reads with that same instant; +its response may then be reused only until the next UTC midnight after it. """ @@ -116,11 +117,26 @@ def utc_now() -> datetime: return datetime.now(UTC) -def _seconds_until_next_utc_day(moment: datetime) -> int: - next_day = (moment + timedelta(days=1)).replace( +def take_utc_instant_for_read(request: Request) -> datetime: + """Take the instant a day-dependent read uses, and record it on the request. + + The read and its response's cache lifetime then use the same UTC day. + """ + instant = utc_now() + setattr(request.state, UTC_DAY_READ, instant) + return instant + + +def _seconds_left_in_utc_day(read_instant: datetime, now: datetime) -> int: + """Return the seconds from ``now`` to the UTC midnight after ``read_instant``. + + Zero when that midnight has passed, so values read for one day are never + reused into the next. + """ + next_midnight = (read_instant.astimezone(UTC) + timedelta(days=1)).replace( hour=0, minute=0, second=0, microsecond=0 ) - return max(int((next_day - moment).total_seconds()), 0) + return max(int((next_midnight - now).total_seconds()), 0) class CacheableMetadataRoute(APIRoute): @@ -148,10 +164,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: The ETag is a hash of the exact body, so it changes with any change to the content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the - running version for five minutes, and one whose route marked it as - depending on the day (``DEPENDS_ON_UTC_DAY``) no later than the next UTC - midnight. A request whose ``If-None-Match`` names the ETag gets a 304 with - no body. + running version for five minutes, and one whose route read for a UTC day + (``UTC_DAY_READ``) no later than the midnight after that day. A request + whose ``If-None-Match`` names the ETag gets a 304 with no body. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -159,8 +174,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: if "policyengine_version" in request.query_params else RUNNING_VERSION_MAX_AGE_SECONDS ) - if getattr(request.state, DEPENDS_ON_UTC_DAY, False): - max_age = min(max_age, _seconds_until_next_utc_day(utc_now())) + read_instant = getattr(request.state, UTC_DAY_READ, None) + if read_instant is not None: + max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): return Response(status_code=304, headers=headers) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index d9ddeb336..406d470ab 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -9,10 +9,10 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - DEPENDS_ON_UTC_DAY, ERROR_RESPONSES, CacheableMetadataRoute, read_resource, + take_utc_instant_for_read, ) from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( CatalogQueryDependency, @@ -121,8 +121,9 @@ def list_parameter_values( request: Request, query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: - # Current values are those in effect on the request's UTC day. - setattr(request.state, DEPENDS_ON_UTC_DAY, query.current) + # Current values are those in effect on the request's UTC day; the + # read and the response's lifetime use the same instant. + now = take_utc_instant_for_read(request) if query.current else None return read_resource( dependencies, MetadataParameterValuePageResponse, @@ -132,6 +133,7 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + now=now, parameter_names=( None if query.parameter_name is None diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index fa5ea761b..4930e3abb 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -574,3 +574,54 @@ def record_mode(connection, _cursor, _statement, *_args) -> None: assert all(autocommit_modes) with published_engine.connect() as connection: assert connection.connection.dbapi_connection.autocommit is False + + +@pytest.mark.parametrize( + "case", + [ + # Read for 31 December, headers written after midnight: not reusable. + ( + datetime(2025, 12, 31, 23, 59, 59, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, 1, tzinfo=UTC), + [0.1], + "public, max-age=0", + ), + ( + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + [0.1], + "public, max-age=43200", + ), + ( + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + [0.2], + "public, max-age=86400", + ), + ], +) +def test_postgres_current_values_and_their_lifetime_use_one_instant( + published_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + case: tuple[datetime, datetime, list[float], str], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant, header_instant, expected_values, expected_cache_control = case + instants = iter([read_instant, header_instant]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + + response = _client(published_engine).get( + "/v2/parameter-values", + params={ + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "current": "true", + }, + ) + + assert response.status_code == 200 + assert [item["value"] for item in response.json()["result"]["items"]] == ( + expected_values + ) + assert response.headers["cache-control"] == expected_cache_control diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index df0465c9d..e91e6440b 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1292,3 +1292,27 @@ def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: for value in response.headers["access-control-expose-headers"].split(",") } assert "etag" in exposed_headers + + +def test_current_values_read_just_before_midnight_expire_at_that_midnight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The read and the response headers must use the same day: values read for + # 31 December may not be reused into 1 January, even when the headers are + # written after midnight. + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant = datetime(2026, 12, 31, 23, 59, 59, tzinfo=UTC) + instants = iter([read_instant, datetime(2027, 1, 1, 0, 0, 1, tzinfo=UTC)]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3¤t=true" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=0" + _name, _args, kwargs = reader.calls[0] + assert kwargs["value_filter"].now == read_instant diff --git a/tests/unit/v2/test_parameter_value_listing_route.py b/tests/unit/v2/test_parameter_value_listing_route.py index 39ac27b72..a07aa4ca5 100644 --- a/tests/unit/v2/test_parameter_value_listing_route.py +++ b/tests/unit/v2/test_parameter_value_listing_route.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import UTC, datetime + import pytest from policyengine_api.services.v2.metadata.types import ParameterValueFilter @@ -9,7 +11,13 @@ from .test_metadata_routes import ResourceReader, _client, _resource_results -def test_listing_forwards_every_query_parameter() -> None: +def test_listing_forwards_every_query_parameter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + request_instant = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) + monkeypatch.setattr(common, "utc_now", lambda: request_instant) results, ids = _resource_results() reader = ResourceReader(results) @@ -32,7 +40,7 @@ def test_listing_forwards_every_query_parameter() -> None: ("uk", "4.19.0"), { "value_filter": ParameterValueFilter( - parameter_id=ids["parameter_id"], current=True + parameter_id=ids["parameter_id"], current=True, now=request_instant ), "offset": 5, "limit": 7, From b9d734e3767f724326d3a18a0164cbe2dfad7f2a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:23:41 +0400 Subject: [PATCH 073/118] Vary every cacheable metadata response by Origin The API reflects every origin in Access-Control-Allow-Origin, and the CORS middleware passes a request without Origin straight through, so such a response carried neither that header nor Vary: Origin while marked public for up to a day. A shared cache storing it could serve it to a later cross-origin request, which the browser rejects. Every cacheable metadata response, and every 304, now carries Vary: Origin, added to any existing Vary value such as compression's Accept-Encoding rather than replacing it. Failed reads are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 7 +++- .../fastapi_routes/v2/metadata/common.py | 24 +++++++++-- tests/unit/v2/test_metadata_routes.py | 40 +++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index baf6b24f4..fa28c9a5a 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -112,7 +112,12 @@ response may be reused only until the UTC midnight after that instant, whichever lifetime would otherwise apply (`max-age=0` if its headers are written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses -carry neither header. `ETag` is listed in `Access-Control-Expose-Headers`, so +carry neither header. Every cacheable response, and every 304, carries +`Vary: Origin` (added to any other `Vary` value, such as compression's +`Accept-Encoding`): the API reflects every origin in +`Access-Control-Allow-Origin` and a request without `Origin` gets no such +header, so a shared cache must keep responses apart by `Origin`. `ETag` is +listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable without that). diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index ba7a74418..d9eb49159 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -165,8 +165,9 @@ def with_cache_headers(request: Request, response: Response) -> Response: content, including the catalog format. A request naming its catalog with ``policyengine_version`` may be reused for a day, one that takes the running version for five minutes, and one whose route read for a UTC day - (``UTC_DAY_READ``) no later than the midnight after that day. A request - whose ``If-None-Match`` names the ETag gets a 304 with no body. + (``UTC_DAY_READ``) no later than the midnight after that day. Every such + response, and the 304 a request whose ``If-None-Match`` names the ETag + gets instead, varies by ``Origin``. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -179,11 +180,28 @@ def with_cache_headers(request: Request, response: Response) -> Response: max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): - return Response(status_code=304, headers=headers) + not_modified = Response(status_code=304, headers=headers) + not_modified.headers["Vary"] = _vary_with_origin(None) + return not_modified response.headers.update(headers) + response.headers["Vary"] = _vary_with_origin(response.headers.get("vary")) return response +def _vary_with_origin(existing: str | None) -> str: + """Return a Vary value that keeps ``existing`` and names Origin once. + + The API reflects every origin in Access-Control-Allow-Origin, and a request + without Origin gets no such header, so a shared cache must store responses + per Origin value or it could serve one to a cross-origin request that the + browser then rejects. + """ + values = [value.strip() for value in (existing or "").split(",") if value.strip()] + if not any(value.lower() == "origin" for value in values): + values.append("Origin") + return ", ".join(values) + + def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: if if_none_match is None: return False diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e91e6440b..bedd4a326 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1316,3 +1316,43 @@ def test_current_values_read_just_before_midnight_expire_at_that_midnight( assert response.headers["cache-control"] == "public, max-age=0" _name, _args, kwargs = reader.calls[0] assert kwargs["value_filter"].now == read_instant + + +def _vary_values(response) -> set[str]: + return { + value.strip().lower() + for value in response.headers.get("vary", "").split(",") + if value.strip() + } + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_metadata_responses_vary_by_origin(origin: str | None) -> None: + # The API reflects any origin, so a shared cache must not reuse a response + # stored for a request without Origin (no Access-Control-Allow-Origin) for + # a cross-origin one, or the reverse. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {} if origin is None else {"Origin": origin} + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + large = client.get("/v2/model-catalog?country_id=us", headers=headers) + revalidated = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": large.headers["etag"]}, + ) + + assert revalidated.status_code == 304 + for response in (small, large, revalidated): + assert "origin" in _vary_values(response) + assert "accept-encoding" in _vary_values(large) + + +def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get("/v2/variables?country_id=us") + + assert response.status_code == 404 + assert "origin" not in _vary_values(response) From b921ec2ecc08a72849dee1e94dd214b0a90bcef1 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:26:05 +0400 Subject: [PATCH 074/118] Document metadata caching headers, 304 and the 400 case in OpenAPI The metadata routes' OpenAPI entries described 400 as an invalid PolicyEngine.py version, which now returns 422; 400 remains only for an unsupported country in the path of the by-country route. They also declared no 304 and none of the caching headers, so the app's generated client and readers could not see them. Every metadata route now documents ETag, Cache-Control and Vary on its 200 response and a 304 without a body. The statement counts in the changelog and the migration document now say exactly which requests send two statements. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.fixed.md | 2 +- docs/migration/stage-9-v2-metadata.md | 7 ++- .../v2/metadata/catalog_routes.py | 4 +- .../fastapi_routes/v2/metadata/common.py | 43 ++++++++++++++++++- .../v2/metadata/geography_routes.py | 4 +- .../v2/metadata/model_routes.py | 4 +- .../v2/metadata/parameter_routes.py | 4 +- tests/unit/v2/test_metadata_routes.py | 30 +++++++++++++ 8 files changed, 85 insertions(+), 13 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md index 8eff7221b..4c325f3a5 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -1 +1 @@ -v2 metadata reads run in autocommit read-only sessions on the shared pool, and each process keeps the catalog versions it has selected, so a metadata request sends two statements to the database (the pool's liveness check and the read) instead of five. +v2 metadata reads run in autocommit sessions on the shared pool, and each process keeps the catalog versions it has selected. A request that makes one read for a version the process has already selected, such as a page of parameters or one parameter's values, now sends two statements to the database (the pool's liveness check and the read) instead of five; each further read in a request adds one statement. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index fa28c9a5a..4cfee0b24 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -93,8 +93,11 @@ later requests for the same country and version skip the selection query; failed selections are not kept. Measured with statement logging in a local database, a page of 500 parameters or one parameter's values took five statements per request before (the pool's liveness check, `BEGIN`, the catalog -selection, the read and `ROLLBACK`) and two after (the liveness check and the -read). The pool's liveness check (`pool_pre_ping`) is unchanged. +selection, the read and `ROLLBACK`) and two after, once the process has +selected that version (the liveness check and the read). A route that makes +several reads, such as `/v2/model-catalog` (regions and datasets), sends one +statement per read after the liveness check. The pool's liveness check +(`pool_pre_ping`) is unchanged. ## Response caching diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py index 1ef94301e..968c802c4 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -8,7 +8,7 @@ from starlette.responses import JSONResponse, Response from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, read_whole_version_resource, @@ -42,7 +42,7 @@ def build_v2_model_catalog_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index d9eb49159..3b54fc2fa 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -50,7 +50,11 @@ ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, - "description": "The resource request or PolicyEngine.py version is invalid.", + "description": ( + "The country in the path is not supported. Query parameters are " + "checked first: an invalid one, including the PolicyEngine.py " + "version, returns 422." + ), }, 404: { "model": V2ErrorResponse, @@ -62,7 +66,10 @@ }, 422: { "model": V2ErrorResponse, - "description": "The request parameters do not match the resource schema.", + "description": ( + "The query parameters do not match the resource schema, for example " + "an unknown parameter or a malformed PolicyEngine.py version." + ), }, 500: { "model": V2ErrorResponse, @@ -75,6 +82,38 @@ } +CACHE_HEADER_DESCRIPTIONS: dict[str, dict[str, Any]] = { + "ETag": { + "description": "Strong validator: a hash of the exact response body.", + "schema": {"type": "string"}, + }, + "Cache-Control": { + "description": ( + "public, with max-age 86400 when the request names " + "policyengine_version and 300 otherwise; current parameter values " + "expire at the next UTC midnight." + ), + "schema": {"type": "string"}, + }, + "Vary": { + "description": "Includes Origin, so caches keep responses apart by origin.", + "schema": {"type": "string"}, + }, +} + +METADATA_RESPONSES: dict[int | str, dict[str, Any]] = { + **ERROR_RESPONSES, + 200: {"headers": CACHE_HEADER_DESCRIPTIONS}, + 304: { + "description": ( + "Not modified: If-None-Match names the current ETag. No body is sent." + ), + "headers": CACHE_HEADER_DESCRIPTIONS, + }, +} +"""Every metadata route's documented responses, with its caching headers.""" + + class CatalogVersionResult(Protocol): """A whole-version result, which names the catalog version it was read from.""" diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 4665ea67c..448826c5d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -40,7 +40,7 @@ def build_v2_metadata_geography_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 63e72ce56..3482ee108 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -43,7 +43,7 @@ def build_v2_metadata_model_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 406d470ab..51383ccc0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -9,7 +9,7 @@ from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, CacheableMetadataRoute, read_resource, take_utc_instant_for_read, @@ -46,7 +46,7 @@ def build_v2_metadata_parameter_router( """ router = APIRouter( prefix="/v2", - responses=ERROR_RESPONSES, + responses=METADATA_RESPONSES, route_class=CacheableMetadataRoute, ) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index bedd4a326..792d7c05c 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1356,3 +1356,33 @@ def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: assert response.status_code == 404 assert "origin" not in _vary_values(response) + + +def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + responses = schema["paths"][path]["get"]["responses"] + assert set(responses["200"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "$ref" in responses["200"]["content"]["application/json"]["schema"] + assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "content" not in responses["304"] + assert "policyengine_version" not in responses["400"]["description"] + assert "path" in responses["400"]["description"] From b0e98b6defe998b45494e5c149020e06db631160 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:27:18 +0400 Subject: [PATCH 075/118] Keep a response's build lock until it is stored, and state the cache's size MetadataResponseCache.get_or_build removed the per-key build lock before storing the built response, so a request arriving in that interval found neither and started a second read. The lock is now released only after the response is stored (or the read failed). The concurrency tests no longer depend on timing: one blocks the build on an event, the other starts a second request while the first stores its result. The documentation states the bound's cost: the 6.10.0 US variables body is 3,910,063 bytes uncompressed, so eight entries hold about 11 MB in the expected mix and at most about 31 MB. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 4 +- .../v2/metadata/response_cache.py | 32 +++++++----- tests/unit/v2/test_metadata_response_cache.py | 52 +++++++++++++++---- 3 files changed, 66 insertions(+), 22 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 4cfee0b24..e80f289f9 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -127,7 +127,9 @@ without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, for up to eight route, country and version combinations (least recently used -dropped first). A response is kept under the exact version it was read from, +dropped first; the 6.10.0 US variables body is 3,910,063 bytes uncompressed, +so eight entries hold about 11 MB for both countries' two routes at two +versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only the first successful request per process reads the database; concurrent first diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 72d3cf9ba..22ea2d348 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -48,9 +48,13 @@ class MetadataResponseCache: are kept under the exact version they were read from: a request without a version (the running version, fixed for the process) and one naming that version share one response. The most recently used ``capacity`` responses - are kept. Requests for a response being built wait for that build instead - of starting another, and a failed read (a v2 error response) is returned - but not kept, so the next request tries again. + are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes + uncompressed, so the default of eight holds about 11 MB for both + countries' two whole-version routes at two versions, and at most about + 31 MB if every entry were that large. Requests for a response being built + wait for that build, and its storage, instead of starting another, and a + failed read (a v2 error response) is returned but not kept, so the next + request tries again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: @@ -99,15 +103,19 @@ def get_or_build( with self._lock: build_lock = self._build_locks.setdefault(build_key, threading.Lock()) with build_lock: - kept = self._stored(route, country_id, requested_version) - if kept is not None: - return kept - built = build() - with self._lock: - self._build_locks.pop(build_key, None) - if not isinstance(built, BuiltResponse): - return built - return self._keep(route, country_id, requested_version, built) + try: + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + built = build() + if not isinstance(built, BuiltResponse): + return built + # Stored before the build lock is released, so a request + # waiting on it finds the response instead of reading again. + return self._keep(route, country_id, requested_version, built) + finally: + with self._lock: + self._build_locks.pop(build_key, None) def _keep( self, diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index 9cfc8fd8c..4d6704a36 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -3,7 +3,6 @@ from __future__ import annotations import threading -import time import pytest from starlette.responses import JSONResponse @@ -119,26 +118,61 @@ def get(country_id: str) -> None: def test_concurrent_requests_for_one_response_build_it_once() -> None: cache = MetadataResponseCache() builds: list[str] = [] - started = threading.Barrier(4) + building = threading.Event() + release = threading.Event() - def slow_build() -> BuiltResponse: + def blocked_build() -> BuiltResponse: builds.append("built") - time.sleep(0.05) + building.set() + release.wait() return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) def request() -> None: - started.wait() - cache.get_or_build("variables", "us", None, slow_build) + cache.get_or_build("variables", "us", None, blocked_build) - threads = [threading.Thread(target=request) for _ in range(4)] - for thread in threads: + first = threading.Thread(target=request) + first.start() + building.wait() + others = [threading.Thread(target=request) for _ in range(3)] + for thread in others: thread.start() - for thread in threads: + release.set() + for thread in [first, *others]: thread.join() + # Whether the others waited on the build or arrived after it, none read. assert builds == ["built"] +def test_a_request_arriving_while_a_response_is_stored_waits_for_it() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + second_results: list[object] = [] + second_requests: list[threading.Thread] = [] + original_keep = cache._keep + + def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: + second_request = threading.Thread( + target=lambda: second_results.append( + cache.get_or_build("variables", "us", None, _counting_build(builds)) + ) + ) + second_requests.append(second_request) + second_request.start() + # The second request may not build while this one stores its result: + # it waits on the build, whatever the timing. + second_request.join(timeout=0.2) + return original_keep(*keep_arguments) + + cache._keep = keep_after_another_request_arrives # type: ignore[method-assign] + + first = cache.get_or_build("variables", "us", None, _counting_build(builds)) + second_requests[0].join() + + assert builds == ["built"] + assert second_results == [first] + + def test_capacity_must_be_positive() -> None: with pytest.raises(ValueError, match="capacity"): MetadataResponseCache(capacity=0) From 77a12b55de071e6732d5826c2bed832d788ee0a4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:28:38 +0400 Subject: [PATCH 076/118] Refuse writes through v2 read sessions and state who uses them The autocommit read sessions were described as read-only, but nothing stopped a flush. Autocommit rules out a per-transaction read-only setting, and a connection-level default would carry over to writers through the shared pool, so the factory now refuses every ORM flush with ReadOnlySessionError, and the docstring says exactly that only the metadata reader opens these sessions and that statements passed to execute are not checked. A test pins that the metadata reader factory is the only user. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 12 ++++--- policyengine_api/data/v2/database.py | 25 ++++++++++++--- tests/unit/v2/test_database.py | 45 ++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index e80f289f9..9357eb48c 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -83,11 +83,13 @@ still returns HTTP 400. ## Database round trips per request -Metadata reads use read-only sessions in autocommit mode on the shared v2 -pool (`get_v2_read_session_factory`): each read is one statement on rows a -published catalog version never changes, so no transaction is needed, and the -database receives no `BEGIN` before the reads and no `ROLLBACK` when the -session closes. Each process also keeps every catalog version a request has +Metadata reads use sessions in autocommit mode on the shared v2 pool +(`get_v2_read_session_factory`), which only the metadata reader opens: each +read is one statement on rows a published catalog version never changes, so no +transaction is needed, and the database receives no `BEGIN` before the reads +and no `ROLLBACK` when the session closes. The sessions refuse to flush added +or changed rows (`ReadOnlySessionError`); a statement passed directly to +`execute` is not checked, so the metadata connectors keep to `SELECT`. Each process also keeps every catalog version a request has selected (`SelectedCatalogStore`), since a published version never changes, so later requests for the same country and version skip the selection query; failed selections are not kept. Measured with statement logging in a local diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 3885ab94d..f9df94461 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -8,6 +8,7 @@ from threading import Lock from typing import TYPE_CHECKING +from sqlalchemy import event from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -83,20 +84,34 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +class ReadOnlySessionError(RuntimeError): + """Raised when code tries to write rows through a v2 read session.""" + + +def _refuse_flush(*_flush_arguments: object) -> None: + msg = "v2 read sessions do not write rows" + raise ReadOnlySessionError(msg) + + def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: - """Build a factory of read-only Sessions that run without a transaction. + """Build a factory of Sessions for v2 metadata reads, run without a transaction. The sessions share ``engine``'s pool in autocommit mode, so psycopg sends each statement on its own: no BEGIN before the first read and no ROLLBACK - when the session closes. Use them only for reads whose statements need no - common snapshot, such as reads of a published catalog version, which never - changes. The pool restores the ordinary mode when a connection returns. + when the session closes. Only the metadata reader uses them: each of its + reads is one statement on a published catalog version, which never + changes, so no common snapshot is needed. The sessions refuse to flush + added or changed rows; statements passed directly to ``execute`` are not + checked, so they must stay reads. The pool restores the ordinary mode + when a connection returns. """ - return sessionmaker( + factory = sessionmaker( bind=engine.execution_options(isolation_level="AUTOCOMMIT"), class_=Session, expire_on_commit=False, ) + event.listen(factory, "before_flush", _refuse_flush) + return factory def _discard_inherited_state(current_pid: int) -> None: diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index c1984199c..fbfa92c72 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -1,11 +1,14 @@ """Tests for lazy process-owned v2 engine and SQLModel Session factories.""" from collections.abc import Iterator +from pathlib import Path import pytest -from sqlmodel import Session +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select from policyengine_api.data.v2 import database +from policyengine_api.data.v2.models import V2_METADATA, TaxBenefitModel from policyengine_api.data.v2.settings import ( V2_MIGRATION_DATABASE_URL, V2_RUNTIME_DATABASE_URL, @@ -148,3 +151,43 @@ def test_read_session_factory_is_rebuilt_with_the_engine() -> None: database.close_v2_database() assert database.get_v2_read_session_factory(settings) is not first + + +def test_read_sessions_refuse_to_write_rows() -> None: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + V2_METADATA.create_all(engine) + read_factory = database.build_v2_read_session_factory(engine) + + with read_factory() as session: + session.add(TaxBenefitModel(name="policyengine-test")) + with pytest.raises(database.ReadOnlySessionError): + session.flush() + + with Session(engine) as session: + assert session.exec(select(TaxBenefitModel)).all() == [] + engine.dispose() + + +def test_only_the_metadata_reader_opens_read_sessions() -> None: + package = Path(database.__file__).parents[2] + users = sorted( + str(module.relative_to(package)) + for module in package.rglob("*.py") + if module.name != "database.py" + and ( + "get_v2_read_session_factory" in module.read_text(encoding="utf-8") + or "build_v2_read_session_factory" in module.read_text(encoding="utf-8") + ) + ) + + assert users == ["fastapi_routes/dependencies.py"] + dependencies_source = (package / "fastapi_routes/dependencies.py").read_text( + encoding="utf-8" + ) + reader_factory = dependencies_source.split( + "def _default_v2_metadata_reader_factory" + )[1].split("\ndef ")[0] + assert "get_v2_read_session_factory" in reader_factory + assert dependencies_source.count("get_v2_read_session_factory") == 2 From 87c17b109c23fde4fa2f92e3eec70497d2d7c296 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:30:44 +0400 Subject: [PATCH 077/118] Describe the metadata stores in the layout guide and drop FoundParameters The layout guide listed neither the metadata route package's query_dependencies and response_cache modules nor the service package's selected_catalogs store; it now describes each and where process-wide state belongs. FoundParameters held ORM rows and a derivable name set in the framework-independent types module; it is removed. The service passes the parameters, their values grouped by name (through the now public dated_values_by_parameter_name transformation) and the missing names to the transformation, which no longer recomputes any of them. Co-Authored-By: Claude Opus 5.5 --- .../skills/v2-code-organization.md | 16 +++++++++++++ .../services/v2/metadata/services.py | 23 ++++++++----------- .../services/v2/metadata/transformations.py | 23 ++++++++----------- .../services/v2/metadata/types.py | 13 +---------- 4 files changed, 35 insertions(+), 40 deletions(-) diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index bbe997b65..b32fe2af1 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -21,9 +21,18 @@ user_policies/ metadata/ response_models.py common.py + query_dependencies.py + response_cache.py *_routes.py ``` +`common.py` also holds the route class that gives metadata responses their +caching headers. `query_dependencies.py` defines the typed FastAPI +dependencies for the shared query models once, for every metadata route +module to import. `response_cache.py` keeps serialized whole-version +responses (bodies and ETags) for the process; it is HTTP-level, so it lives +with the routes. + `errors.py` defines the strict error response and serialization function shared by every API v2 resource. Application-wide exception handling distinguishes API v2 from non-v2 requests but must not select error behavior by matching @@ -52,6 +61,13 @@ services/v2// deletes.py ``` +A resource may also add a module for process-wide state its service reuses +across requests, named for what it keeps, such as the metadata package's +`selected_catalogs.py` (each catalog version a request has selected, which +never changes once published). Such a module holds plain data, constructs no +SQL and owns no session; the composition code creates one instance per process +and injects it into the service. + Only add CRUD connector modules for operations the resource supports. Append a specific descriptor when one operation file would become too broad, such as `reads_variables.py` or `reads_parameter_tree.py`. diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 017e3e5b4..4496e3b30 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -42,6 +42,7 @@ ) from policyengine_api.services.v2.metadata.transformations import ( ancestor_paths, + dated_values_by_parameter_name, metadata_catalog_parameters, metadata_catalog_variables, metadata_dataset, @@ -61,7 +62,6 @@ part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( - FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParametersResult, MetadataCatalogVariablesResult, @@ -657,30 +657,25 @@ def get_catalog_parameters( parameters = read_parameters_by_name( session, model_version_id=model_version_id, parameter_names=requested_names ) - found_parameters = FoundParameters( - parameters=parameters, - names={parameter.name for parameter in parameters}, - ) + found_names = {parameter.name for parameter in parameters} value_rows = read_current_law_values_by_parameter_name( - session, - model_version_id=model_version_id, - parameter_names=found_parameters.names, + session, model_version_id=model_version_id, parameter_names=found_names ) node_rows = read_parameter_node_labels( session, model_version_id=model_version_id, node_names={ - ancestor - for name in found_parameters.names - for ancestor in ancestor_paths(name) + ancestor for name in found_names for ancestor in ancestor_paths(name) }, ) return metadata_catalog_parameters( selected, - requested_names=requested_names, - found_parameters=found_parameters, - value_rows=value_rows, + parameters=parameters, + values_by_name=dated_values_by_parameter_name( + value_rows, parameter_names=found_names + ), node_rows=node_rows, + missing_names=requested_names - found_names, ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 938027741..48efad3a1 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -8,7 +8,6 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( CurrentLawValueRow, - FoundParameters, MetadataCanonicalParameterValue, MetadataCatalogParameter, MetadataCatalogParametersResult, @@ -153,7 +152,7 @@ def metadata_catalog_variables( current-law values of those parameters in name and start-date order. Each path gets an entry, empty when no value carries its name. """ - values_by_path = _dated_values_by_parameter_name( + values_by_path = dated_values_by_parameter_name( part_values, parameter_names=part_paths ) return MetadataCatalogVariablesResult( @@ -164,7 +163,7 @@ def metadata_catalog_variables( ) -def _dated_values_by_parameter_name( +def dated_values_by_parameter_name( value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] ) -> dict[str, list[MetadataDatedValue]]: """Group value rows by parameter name, with an empty list for each other name. @@ -216,21 +215,17 @@ def ancestor_paths(parameter_name: str) -> list[str]: def metadata_catalog_parameters( selected: SelectedCatalog, *, - requested_names: set[str], - found_parameters: FoundParameters, - value_rows: list[CurrentLawValueRow], + parameters: list[Parameter], + values_by_name: dict[str, list[MetadataDatedValue]], node_rows: list[ParameterNodeLabelRow], + missing_names: set[str], ) -> MetadataCatalogParametersResult: """Build the requested parameters with their values and ancestor labels. - value_rows are the found parameters' current-law values and node_rows the - stored labels of their ancestor nodes. + values_by_name maps each parameter's name to its current-law values (see + dated_values_by_parameter_name), node_rows are the stored labels of their + ancestor nodes, and missing_names the requested names the version lacks. """ - parameters = found_parameters.parameters - found_names = found_parameters.names - values_by_name = _dated_values_by_parameter_name( - value_rows, parameter_names=found_names - ) return MetadataCatalogParametersResult( policyengine_version=selected.policyengine_version, catalog_format=selected.model_version.catalog_format, @@ -242,7 +237,7 @@ def metadata_catalog_parameters( for parameter in parameters ], node_labels={row.name: row.label for row in node_rows}, - missing_names=sorted(requested_names - found_names), + missing_names=sorted(missing_names), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index fecb656fa..812f73638 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,14 +5,11 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar +from typing import Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue -if TYPE_CHECKING: - from policyengine_api.data.v2.models import Parameter - @dataclass(frozen=True) class ParameterFilter: @@ -62,14 +59,6 @@ class CurrentLawValueRow(NamedTuple): end_date: datetime | None -@dataclass(frozen=True) -class FoundParameters: - """The stored parameters a by-name read found, and their names.""" - - parameters: list[Parameter] - names: set[str] - - class ParameterNodeLabelRow(NamedTuple): """A parameter node's name and its stored label (None when it has none).""" From 4bbe04f4edc9ecc75b18aede96cf386fd418932f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:42:31 +0400 Subject: [PATCH 078/118] Give parameter-tree leaves the economy and household flags /v2/parameters/children built each leaf's parameter summary by hand from a few selected columns and left economy and household out, so a leaf always reported null, which the shared schema reserves for versions published before catalog format 2. The tree read now selects the stored Parameter, and each leaf is built with metadata_parameter, the same builder the parameter routes use, so the two can no longer differ. Co-Authored-By: Claude Opus 5.5 --- .../reads_parameter_tree.py | 8 ++--- .../services/v2/metadata/transformations.py | 36 +++++++------------ tests/unit/v2/test_metadata_service.py | 20 +++++++++++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index cd945768e..49f232a8e 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -114,11 +114,7 @@ def _parameter_children_statement( ), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), - parameter_id.label("parameter_id"), - col(Parameter.label).label("parameter_label"), - col(Parameter.description).label("parameter_description"), - col(Parameter.data_type).label("parameter_data_type"), - col(Parameter.unit).label("parameter_unit"), + Parameter, ) .select_from( paths.outerjoin( @@ -152,7 +148,7 @@ def read_parameter_children( """Return a page of the direct children of parent_path in a model version's tree. Each row has the child's path, label, type (node or parameter), child count - and, for parameters, the parameter's summary columns. The session's dialect + and the stored Parameter (None for a node). The session's dialect selects SQLite or PostgreSQL string functions; limit + 1 rows are fetched so the caller can tell whether another page exists. """ diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 48efad3a1..8234fb8db 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -323,31 +323,21 @@ def metadata_region(region: Region) -> MetadataRegion: def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: """Build parameter-tree children from read_parameter_children rows. - Parameter leaves carry the parameter's summary, and a child with no label is - labelled with the last segment of its path. + Parameter leaves carry the same summary as the parameter routes, and a child + with no label is labelled with the last segment of its path. """ - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) + return [ + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=( + metadata_parameter(row.Parameter) if row.type == "parameter" else None + ), ) - return items + for row in rows + ] def national_region(regions: list[Region], country_id: str) -> Region | None: diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 458ab34a4..fbe09f72c 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1117,6 +1117,26 @@ def test_value_and_node_label_reads_return_typed_rows( assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] +def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( + catalog_session: Session, +) -> None: + service = _service(catalog_session) + + leaf = next( + child + for child in service.list_parameter_children( + "us", parent_path="gov.example" + ).items + if child.type == "parameter" + ) + page_item = service.list_parameters( + "us", parameter_filter=ParameterFilter(names=("gov.example.rate",)) + ).items[0] + + assert leaf.parameter == page_item + assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From db0b4ae418653aaf2706e61656debe60b27de042 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:50:20 +0400 Subject: [PATCH 079/118] Give every request waiting on a whole-version read that read's outcome MetadataResponseCache held a build lock for the whole read, and when the read failed each waiting request took the lock in turn and read again: ten concurrent first requests with a failing read ran ten reads one after another. A waiting request's cleanup could also delete a newer read's lock. Each key now has at most one read in progress; requests arriving meanwhile wait for it and receive its outcome, the kept response or their own copy of its error response (middleware adds request-specific headers, so error responses are not shared). Nothing is kept on failure, so the first request after it reads again, and a read removes only its own entry. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 11 ++- .../fastapi_routes/v2/metadata/common.py | 2 +- .../v2/metadata/response_cache.py | 98 ++++++++++++++----- tests/unit/v2/test_metadata_response_cache.py | 90 +++++++++++++++++ 4 files changed, 173 insertions(+), 28 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 9357eb48c..bf72dc038 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -134,10 +134,13 @@ so eight entries hold about 11 MB for both countries' two routes at two versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only -the first successful request per process reads the database; concurrent first -requests wait for one read, and a failed read is not kept. Measured locally against the full 6.10.0 US -catalog, the variables route takes about 160 ms when read and about 25 ms -when kept, most of it compression. +the first successful request per process reads the database. Requests that +arrive while that read is in progress wait for it and all receive its outcome, +the kept response or a copy of its error response, so one failing read is not +repeated by each of them; a failed read is not kept, so the first request after +it reads again. Measured locally against the full 6.10.0 US catalog, the +variables route takes about 160 ms when read and about 25 ms when kept, most of +it compression. ## Configuration and credential boundaries diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 3b54fc2fa..3e623cf83 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -285,7 +285,7 @@ def read_whole_version_resource( if cache is None: return read_resource(dependencies, response_type, operation) - def build() -> BuiltResponse | JSONResponse: + def build() -> BuiltResponse | Response: result = read_resource(dependencies, response_type, operation) if isinstance(result, MetadataResourceSuccessResponse): return BuiltResponse( diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 22ea2d348..039c9dd03 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -8,11 +8,11 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from starlette.responses import Response + if TYPE_CHECKING: from collections.abc import Callable - from starlette.responses import JSONResponse - DEFAULT_RESPONSE_CAPACITY = 8 KeptResponseKey = tuple[str, str, str] @@ -40,6 +40,35 @@ class KeptResponse: entity_tag: str +@dataclass(frozen=True) +class FailedRead: + """A failed whole-version read, given to the requests that waited on it.""" + + status_code: int + body: bytes + + def response(self) -> Response: + """Return a new error response for one waiting request. + + Each request gets its own response object, because middleware adds + request-specific headers to the response it sends. + """ + return Response( + content=self.body, + status_code=self.status_code, + media_type="application/json", + ) + + +class _InFlightRead: + """One read in progress for a key, and how many requests wait on it.""" + + def __init__(self) -> None: + self.done = threading.Event() + self.outcome: KeptResponse | FailedRead | None = None + self.waiters = 0 + + class MetadataResponseCache: """Keep whole-version responses, built once each per process. @@ -51,10 +80,10 @@ class MetadataResponseCache: are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes uncompressed, so the default of eight holds about 11 MB for both countries' two whole-version routes at two versions, and at most about - 31 MB if every entry were that large. Requests for a response being built - wait for that build, and its storage, instead of starting another, and a - failed read (a v2 error response) is returned but not kept, so the next - request tries again. + 31 MB if every entry were that large. Requests that arrive while a read + for the same key is in progress wait for it, and all receive its outcome: + the kept response, or a copy of its error response. A failed read is not + kept, so the first request after it reads again. """ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: @@ -66,7 +95,7 @@ def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() self._running_versions: dict[tuple[str, str], str] = {} self._lock = threading.Lock() - self._build_locks: dict[tuple[str, str, str | None], threading.Lock] = {} + self._builds: dict[tuple[str, str, str | None], _InFlightRead] = {} def _stored( self, route: str, country_id: str, requested_version: str | None @@ -90,32 +119,55 @@ def get_or_build( route: str, country_id: str, requested_version: str | None, - build: Callable[[], BuiltResponse | JSONResponse], - ) -> KeptResponse | JSONResponse: + build: Callable[[], BuiltResponse | Response], + ) -> KeptResponse | Response: """Return the kept response, calling ``build`` only when none is kept. - ``requested_version`` None means the running version. + ``requested_version`` None means the running version. Requests that + arrive while a read for the same key is in progress wait for it and + receive its outcome: the kept response, or a copy of its error + response. A failed read is not kept, so the next request after it + reads again. """ kept = self._stored(route, country_id, requested_version) if kept is not None: return kept - build_key = (route, country_id, requested_version) + key = (route, country_id, requested_version) with self._lock: - build_lock = self._build_locks.setdefault(build_key, threading.Lock()) - with build_lock: - try: - kept = self._stored(route, country_id, requested_version) - if kept is not None: - return kept + in_flight = self._builds.get(key) + if in_flight is None: + in_flight = _InFlightRead() + self._builds[key] = in_flight + reads_itself = True + else: + in_flight.waiters += 1 + reads_itself = False + if not reads_itself: + in_flight.done.wait() + outcome = in_flight.outcome + if outcome is None: + # The read raised instead of returning a response: try again. + return self.get_or_build(route, country_id, requested_version, build) + return outcome.response() if isinstance(outcome, FailedRead) else outcome + try: + kept = self._stored(route, country_id, requested_version) + if kept is None: built = build() if not isinstance(built, BuiltResponse): + in_flight.outcome = FailedRead( + status_code=built.status_code, body=bytes(built.body) + ) return built - # Stored before the build lock is released, so a request - # waiting on it finds the response instead of reading again. - return self._keep(route, country_id, requested_version, built) - finally: - with self._lock: - self._build_locks.pop(build_key, None) + kept = self._keep(route, country_id, requested_version, built) + in_flight.outcome = kept + return kept + finally: + with self._lock: + # Only this read's own entry is removed; a later read for the + # same key keeps its entry. + if self._builds.get(key) is in_flight: + del self._builds[key] + in_flight.done.set() def _keep( self, diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py index 4d6704a36..a0a788c1c 100644 --- a/tests/unit/v2/test_metadata_response_cache.py +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading +import time import pytest from starlette.responses import JSONResponse @@ -173,6 +174,95 @@ def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: assert second_results == [first] +def _wait_for_waiters(cache: MetadataResponseCache, key: tuple, count: int) -> None: + # Polls until every request has registered as waiting on the read in + # progress, so the result does not depend on thread timing. + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + with cache._lock: + in_flight = cache._builds.get(key) + if in_flight is not None and in_flight.waiters == count: + return + time.sleep(0.001) + pytest.fail(f"{count} requests did not wait on the read in progress") + + +@pytest.mark.parametrize("read_outcome", ["failure", "success"]) +def test_waiting_requests_receive_the_outcome_of_the_one_read( + read_outcome: str, +) -> None: + read_fails = read_outcome == "failure" + cache = MetadataResponseCache() + builds: list[str] = [] + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> BuiltResponse | JSONResponse: + builds.append("built") + release.wait() + if read_fails: + return JSONResponse({"status": "error"}, status_code=503) + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + results: list[object] = [] + + def request() -> None: + results.append(cache.get_or_build(*key, blocked_build)) + + owner = threading.Thread(target=request, daemon=True) + waiters = [threading.Thread(target=request, daemon=True) for _ in range(9)] + try: + owner.start() + while not builds: + time.sleep(0.001) + for thread in waiters: + thread.start() + _wait_for_waiters(cache, key, 9) + finally: + release.set() + for thread in [owner, *waiters]: + thread.join(timeout=10) + + assert builds == ["built"] + assert len(results) == 10 + if read_fails: + assert all(result.status_code == 503 for result in results) + assert {bytes(result.body) for result in results} == {b'{"status":"error"}'} + # Each request gets its own response object, so middleware headers + # added for one request cannot reach another. + assert len({id(result) for result in results}) == 10 + else: + assert all(result is results[0] for result in results) + assert key not in cache._builds + + +def test_a_finished_read_leaves_a_newer_reads_entry_in_place() -> None: + cache = MetadataResponseCache() + building = threading.Event() + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> JSONResponse: + building.set() + release.wait() + return JSONResponse({"status": "error"}, status_code=503) + + owner = threading.Thread( + target=lambda: cache.get_or_build(*key, blocked_build), daemon=True + ) + newer_read = object() + try: + owner.start() + building.wait(timeout=10) + with cache._lock: + cache._builds[key] = newer_read + finally: + release.set() + owner.join(timeout=10) + + assert cache._builds[key] is newer_read + + def test_capacity_must_be_positive() -> None: with pytest.raises(ValueError, match="capacity"): MetadataResponseCache(capacity=0) From ba1b435c66ccd028bc1abf0e775861f4860bca98 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:52:35 +0400 Subject: [PATCH 080/118] Give cacheable 200 and 304 responses the same Vary, each value once with_cache_headers gave a 304 only Vary: Origin, while gzip added Accept-Encoding only to compressed 200 responses; a 304 must carry the Vary its 200 would. Every cacheable metadata response and its 304 now name Origin and Accept-Encoding. Starlette's CORS and gzip middleware append to Vary without checking for values already present, which gave "Origin" twice for cross-origin requests, so a small UniqueVaryMiddleware now wraps the CORS middleware and lists each Vary value once. Tested through the whole middleware stack, with and without Origin. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 11 ++-- policyengine_api/asgi_factory.py | 6 +- .../fastapi_routes/v2/metadata/common.py | 37 +++++++----- policyengine_api/vary_headers.py | 56 +++++++++++++++++++ tests/unit/v2/test_metadata_routes.py | 35 ++++++++++++ 5 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 policyengine_api/vary_headers.py diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index bf72dc038..8f7a8431a 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -118,10 +118,13 @@ whichever lifetime would otherwise apply (`max-age=0` if its headers are written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or weak form, alone or in a list) gets HTTP 304 with no body. Error responses carry neither header. Every cacheable response, and every 304, carries -`Vary: Origin` (added to any other `Vary` value, such as compression's -`Accept-Encoding`): the API reflects every origin in -`Access-Control-Allow-Origin` and a request without `Origin` gets no such -header, so a shared cache must keep responses apart by `Origin`. `ETag` is +`Vary: Origin, Accept-Encoding`, each value once: the API reflects every +origin in `Access-Control-Allow-Origin` and a request without `Origin` gets no +such header, so a shared cache must keep responses apart by `Origin`, and gzip +compresses bodies for clients that accept it. A 304 carries the same `Vary` as +its 200. The CORS and gzip middleware append to `Vary` without checking for +values already present, so the application's outermost middleware +(`UniqueVaryMiddleware`) lists each value once. `ETag` is listed in `Access-Control-Expose-Headers`, so browser scripts on other origins can read it (`Cache-Control` is readable without that). diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index cba4c00bc..8f74b1bb2 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -53,6 +53,7 @@ current_observability_id, generate_request_id, ) +from policyengine_api.vary_headers import UniqueVaryMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterable @@ -471,7 +472,9 @@ def create_asgi_app( app.mount("/", WSGIMiddleware(wsgi_app)) # The public API already permits every web origin. Use the standard ASGI # implementation while preserving the existing reflected-origin response. - return CORSMiddleware( + # UniqueVaryMiddleware wraps it, because CORS and gzip each append to Vary + # values a route may already have named. + cors_app = CORSMiddleware( app=app, allow_origin_regex=".*", allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], @@ -486,3 +489,4 @@ def create_asgi_app( allow_credentials=False, max_age=600, ) + return UniqueVaryMiddleware(cors_app) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 3e623cf83..bdffdd410 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -34,6 +34,7 @@ InvalidMetadataPageError, MetadataResourceNotFoundError, ) +from policyengine_api.vary_headers import unique_vary_values if TYPE_CHECKING: from collections.abc import Callable, Coroutine @@ -96,7 +97,10 @@ "schema": {"type": "string"}, }, "Vary": { - "description": "Includes Origin, so caches keep responses apart by origin.", + "description": ( + "Origin, Accept-Encoding: caches keep responses apart by origin " + "and by compression; a 304 carries the same value." + ), "schema": {"type": "string"}, }, } @@ -206,7 +210,7 @@ def with_cache_headers(request: Request, response: Response) -> Response: running version for five minutes, and one whose route read for a UTC day (``UTC_DAY_READ``) no later than the midnight after that day. Every such response, and the 304 a request whose ``If-None-Match`` names the ETag - gets instead, varies by ``Origin``. + gets instead, varies by ``Origin`` and ``Accept-Encoding``. """ entity_tag = response.headers.get("etag") or body_entity_tag(response.body) max_age = ( @@ -220,25 +224,30 @@ def with_cache_headers(request: Request, response: Response) -> Response: headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): not_modified = Response(status_code=304, headers=headers) - not_modified.headers["Vary"] = _vary_with_origin(None) + not_modified.headers["Vary"] = _cacheable_vary(None) return not_modified response.headers.update(headers) - response.headers["Vary"] = _vary_with_origin(response.headers.get("vary")) + response.headers["Vary"] = _cacheable_vary(response.headers.get("vary")) return response -def _vary_with_origin(existing: str | None) -> str: - """Return a Vary value that keeps ``existing`` and names Origin once. +CACHEABLE_VARY_VALUES = ("Origin", "Accept-Encoding") +"""What every cacheable metadata response, and its 304, varies by. + +The API reflects every origin in Access-Control-Allow-Origin, and a request +without Origin gets no such header, so a shared cache must keep responses +apart by Origin; gzip compresses the body for clients that accept it. A 304 +carries the Vary its 200 would. +""" + + +def _cacheable_vary(existing: str | None) -> str: + """Return ``existing`` with Origin and Accept-Encoding added if absent. - The API reflects every origin in Access-Control-Allow-Origin, and a request - without Origin gets no such header, so a shared cache must store responses - per Origin value or it could serve one to a cross-origin request that the - browser then rejects. + The CORS and gzip middleware may append them again; the application's + outermost middleware (UniqueVaryMiddleware) then lists each value once. """ - values = [value.strip() for value in (existing or "").split(",") if value.strip()] - if not any(value.lower() == "origin" for value in values): - values.append("Origin") - return ", ".join(values) + return unique_vary_values(", ".join([existing or "", *CACHEABLE_VARY_VALUES])) def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: diff --git a/policyengine_api/vary_headers.py b/policyengine_api/vary_headers.py new file mode 100644 index 000000000..98d834086 --- /dev/null +++ b/policyengine_api/vary_headers.py @@ -0,0 +1,56 @@ +"""ASGI middleware that lists each Vary value of a response once.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from starlette.datastructures import MutableHeaders + +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +def unique_vary_values(vary: str) -> str: + """Return ``vary`` with each value listed once, in first-seen order. + + Values are compared without regard to case, as HTTP field names are. + """ + values: list[str] = [] + seen: set[str] = set() + for value in vary.split(","): + name = value.strip() + if name and name.lower() not in seen: + seen.add(name.lower()) + values.append(name) + return ", ".join(values) + + +class UniqueVaryMiddleware: + """List each Vary value of every HTTP response once. + + Starlette's CORS and gzip middleware append Origin and Accept-Encoding to + Vary without checking whether a route already named them. This wraps the + outermost middleware, so the response that leaves the application names + each value once. + """ + + def __init__(self, app: ASGIApp) -> None: + """Wrap ``app``.""" + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Pass the request through, rewriting a duplicated Vary header.""" + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_unique_vary(message: Message) -> None: + if message["type"] == "http.response.start": + message.setdefault("headers", []) + headers = MutableHeaders(scope=message) + vary = headers.get("vary") + if vary is not None: + headers["vary"] = unique_vary_values(vary) + await send(message) + + await self.app(scope, receive, send_with_unique_vary) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 792d7c05c..e0ce842ea 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1386,3 +1386,38 @@ def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: assert "content" not in responses["304"] assert "policyengine_version" not in responses["400"]["description"] assert "path" in responses["400"]["description"] + + +def _vary_value_counts(response) -> dict[str, int]: + counts: dict[str, int] = {} + for value in response.headers.get("vary", "").split(","): + if value.strip(): + counts[value.strip().lower()] = counts.get(value.strip().lower(), 0) + 1 + return counts + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( + origin: str | None, +) -> None: + # Through the whole middleware stack (gzip, then CORS): a 304 must carry + # the Vary its 200 would, and each value appears once. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {"Accept-Encoding": "gzip"} + if origin is not None: + headers["Origin"] = origin + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + compressed = client.get("/v2/model-catalog?country_id=us", headers=headers) + not_modified = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": compressed.headers["etag"]}, + ) + + assert compressed.headers.get("content-encoding") == "gzip" + assert not_modified.status_code == 304 + expected = {"origin": 1, "accept-encoding": 1} + assert _vary_value_counts(small) == expected + assert _vary_value_counts(compressed) == expected + assert _vary_value_counts(not_modified) == expected From ec9a9c7c23f40dcc3bb9108ee4bd0ed4a20b8591 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:54:19 +0400 Subject: [PATCH 081/118] Declare 400 only on the metadata route with the country in its path Every metadata route's OpenAPI entry listed 400 for an unsupported country in the path, but only /v2/tax-benefit-models/by-country/ {country_id} takes the country in its path; on every other route the shared query models reject an invalid country, version or page with 422 before any read, so 400 cannot occur there. The description now sits only on that route. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/v2/metadata/common.py | 6 ++++- .../v2/metadata/model_routes.py | 2 ++ tests/unit/v2/test_metadata_routes.py | 23 +++++++++---------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index bdffdd410..a2dab29ee 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -48,7 +48,7 @@ ) from policyengine_api.query_parameters import CatalogQuery -ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { +PATH_COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, "description": ( @@ -57,6 +57,10 @@ "version, returns 422." ), }, +} +"""The extra response of the one metadata route that takes the country in its path.""" + +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 404: { "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 3482ee108..ae3127725 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -10,6 +10,7 @@ from policyengine_api.fastapi_routes.v2.metadata.common import ( METADATA_RESPONSES, + PATH_COUNTRY_ERROR_RESPONSES, CacheableMetadataRoute, read_resource, ) @@ -69,6 +70,7 @@ def list_models( @router.get( "/tax-benefit-models/by-country/{country_id}", response_model=MetadataModelSelectionResponse, + responses=PATH_COUNTRY_ERROR_RESPONSES, summary="Get a country model and selected PolicyEngine.py version", ) def get_model_by_country( diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index e0ce842ea..bc5eafb7a 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -650,20 +650,15 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: for path in metadata_paths: operation = schema["paths"][path]["get"] - assert set(operation["responses"]) >= { - "200", - "400", - "404", - "405", - "422", - "500", - "503", - } + error_statuses = ["404", "405", "422", "500", "503"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + error_statuses.append("400") + assert set(operation["responses"]) >= {"200", *error_statuses} success_schema = operation["responses"]["200"]["content"]["application/json"][ "schema" ] assert success_schema["$ref"].startswith("#/components/schemas/Metadata") - for status in ("400", "404", "405", "422", "500", "503"): + for status in error_statuses: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] @@ -1384,8 +1379,12 @@ def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: assert "$ref" in responses["200"]["content"]["application/json"]["schema"] assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} assert "content" not in responses["304"] - assert "policyengine_version" not in responses["400"]["description"] - assert "path" in responses["400"]["description"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + assert "path" in responses["400"]["description"] + else: + # Query parameters are validated first (422), so only a country + # given in the path can return 400. + assert "400" not in responses, path def _vary_value_counts(response) -> dict[str, int]: From 069d739f8db38e9472fea99bfee75f5bc9adf0b6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:19 +0400 Subject: [PATCH 082/118] Test autocommit reads on one pooled connection and the production reader The autocommit test used a NullPool engine, so it could not notice a pooled connection left in autocommit mode for the next writer. It now uses a pooled engine with one connection and checks that the writer that takes it next is transactional (with autocommit left on, the test fails). A new test runs the production combination of read sessions, SelectedCatalogStore and the response cache through the routes and counts the statements each request sends. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_v2_metadata_routes.py | 110 +++++++++++++++++-- 1 file changed, 103 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 4930e3abb..e72cbd17f 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -550,30 +550,126 @@ def test_postgres_metadata_reads_send_no_transaction_statements( ) -> None: # Each metadata read is one statement on rows a published catalog never # changes, so the read session runs in autocommit mode: psycopg then sends - # no BEGIN before the reads and no ROLLBACK when the session closes. + # no BEGIN before the reads and no ROLLBACK when the session closes. The + # engine has one pooled connection, so the writer below must get the same + # connection back in the ordinary, transactional mode. + from psycopg.pq import TransactionStatus + from policyengine_api.data.v2.database import build_v2_read_session_factory + del published_engine # publishes the catalog this pooled engine reads + pooled_engine = create_engine(_disposable_url(), pool_size=1, max_overflow=0) + read_connections: list[object] = [] autocommit_modes: list[bool] = [] def record_mode(connection, _cursor, _statement, *_args) -> None: - autocommit_modes.append(connection.connection.dbapi_connection.autocommit) + dbapi_connection = connection.connection.dbapi_connection + read_connections.append(dbapi_connection) + autocommit_modes.append(dbapi_connection.autocommit) - event.listen(published_engine, "before_cursor_execute", record_mode) try: + event.listen(pooled_engine, "before_cursor_execute", record_mode) service = V2MetadataService( - MetadataDatabaseSession(build_v2_read_session_factory(published_engine)()), + MetadataDatabaseSession(build_v2_read_session_factory(pooled_engine)()), running_policyengine_version=POLICYENGINE_VERSION, ) result = service.list_parameters("us") service.close() + event.remove(pooled_engine, "before_cursor_execute", record_mode) + + with Session(pooled_engine) as writer: + writer_connection = writer.connection().connection.dbapi_connection + writer.execute(text("SELECT 1")) + writer_status = writer_connection.info.transaction_status + writer_autocommit = writer_connection.autocommit finally: - event.remove(published_engine, "before_cursor_execute", record_mode) + pooled_engine.dispose() assert [item.name for item in result.items] == ["gov.example.rate"] assert autocommit_modes assert all(autocommit_modes) - with published_engine.connect() as connection: - assert connection.connection.dbapi_connection.autocommit is False + assert {id(connection) for connection in read_connections} == { + id(writer_connection) + } + assert writer_autocommit is False + assert writer_status == TransactionStatus.INTRANS + + +def test_postgres_production_metadata_reader_keeps_catalogs_and_responses( + published_engine: Engine, +) -> None: + # The production combination: autocommit read sessions, the process's + # selected catalogs and its whole-version response cache. + from policyengine_api.data.v2.database import build_v2_read_session_factory + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) + + read_factory = build_v2_read_session_factory(published_engine) + selected_catalogs = SelectedCatalogStore() + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_metadata_reader_factory=lambda: V2MetadataService( + MetadataDatabaseSession(read_factory()), + running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, + ), + v2_metadata_response_cache=MetadataResponseCache(), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + query = {"country_id": "us", "policyengine_version": POLICYENGINE_VERSION} + + event.listen(published_engine, "before_cursor_execute", record_statement) + try: + first_variables = client.get( + "/v2/model-catalog/variables", params={"country_id": "us"} + ) + statements_for_first = len(statements) + repeated_variables = client.get("/v2/model-catalog/variables", params=query) + statements_for_repeat = len(statements) - statements_for_first + parameters = client.get( + "/v2/parameters", params={"country_id": "us", "name": "gov.example.rate"} + ) + statements_for_parameters = ( + len(statements) - statements_for_first - statements_for_repeat + ) + finally: + event.remove(published_engine, "before_cursor_execute", record_statement) + + assert first_variables.status_code == 200 + assert repeated_variables.content == first_variables.content + assert repeated_variables.headers["etag"] == first_variables.headers["etag"] + # The running version's selection and the variables read (the fixture's + # variables name no parameter-defined parts, so no value read); then + # nothing for the explicit request, which shares the kept response; then + # only the parameter read, the running version's selection being kept. + assert statements_for_first == 2 + assert statements_for_repeat == 0 + assert statements_for_parameters == 1 + assert [item["name"] for item in parameters.json()["result"]["items"]] == [ + "gov.example.rate" + ] + assert selected_catalogs.get(("us", None)) is not None @pytest.mark.parametrize( From 67785b5a8dd007b777b830f47a451ca9145bf4a5 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:56 +0400 Subject: [PATCH 083/118] Bring the 1.16 changelog fragment in line with the caching behavior The fragment did not mention Vary, the readable ETag or how requests waiting on a whole-version read are answered, and described the current-value lifetime loosely. It now states each as the code does. Co-Authored-By: Claude Opus 5.5 --- changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index a80f72ae6..09a26ee81 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,5 +1,5 @@ -Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the next UTC midnight for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. -Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. From 1e93eaaddd9243dc690b07fd14221def3b999700 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:06:40 +0400 Subject: [PATCH 084/118] Import the connector modules in a test instead of suppressing line length Two imports in the typed-rows test named the connector functions by long dotted paths and suppressed E501. The test now imports the two connector modules and calls the functions through them, so it needs no suppression. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index fbe09f72c..798c6ce02 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1090,20 +1090,18 @@ def test_each_country_and_version_keeps_its_own_selection( def test_value_and_node_label_reads_return_typed_rows( catalog_session: Session, ) -> None: - from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 - read_parameter_node_labels, - ) - from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( # noqa: E501 - read_current_law_values_by_parameter_name, + from policyengine_api.services.v2.metadata.database_connectors import ( + reads_parameter_nodes, + reads_parameters, ) model_version = _us_model_version(catalog_session) - values = read_current_law_values_by_parameter_name( + values = reads_parameters.read_current_law_values_by_parameter_name( catalog_session, model_version_id=model_version.id, parameter_names={"gov.example.rate"}, ) - labels = read_parameter_node_labels( + labels = reads_parameter_nodes.read_parameter_node_labels( catalog_session, model_version_id=model_version.id, node_names={"gov.example", "gov.absent"}, From 02e120a0f509cca2fbe7c9b9d5651da6b8f76379 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:44:06 +0400 Subject: [PATCH 085/118] Serve each variable's unit and definition period The app's v3 household builder and breakdown need each variable's unit (currency formatting) and definition period (per-month or per-year labels and household validation), which catalog format 2 now stores. Every variable response, in /v2/model-catalog/variables and in /v2/variables pages, now carries unit and definition_period, declared in OpenAPI and null for versions published before catalog format 2. The 6.10.0 US variables body grows from 3,910,063 to 4,190,824 bytes uncompressed (0.589 MB compressed, from 0.583 MB), and the documented bound of the response store is updated to match. Co-Authored-By: Claude Opus 5.5 --- ...n-1f-116-catalog-routes-on-demand.added.md | 2 +- docs/migration/stage-9-v2-metadata.md | 10 +-- .../v2/metadata/response_cache.py | 6 +- .../services/v2/metadata/transformations.py | 2 + .../services/v2/metadata/types.py | 11 ++- tests/integration/test_v2_metadata_routes.py | 2 + tests/unit/v2/test_metadata_routes.py | 12 ++++ tests/unit/v2/test_metadata_service.py | 67 +++++++++++++++++++ 8 files changed, 101 insertions(+), 11 deletions(-) diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md index 09a26ee81..bbfc742cb 100644 --- a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -1,5 +1,5 @@ Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. -GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, and enum labels), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, enum labels, unit and definition period), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8f7a8431a..3785f5fbc 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -45,8 +45,10 @@ parameters are rejected. `GET /v2/model-catalog/variables` (same query) returns every variable of the version in name order, in one response, with the catalog format 2 fields: `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, -`subtracts_parameter_path` and `enum_values` (each value's name and its label, -in the type the country package declares). Where a variable's `adds` or +`subtracts_parameter_path`, `enum_values` (each value's name and its label, +in the type the country package declares), `unit` (for example +`currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which +the household builder uses for formatting and per-period labels. Where a variable's `adds` or `subtracts` list comes from a parameter, `part_parameter_values` maps that parameter's path to its current-law values in start-date order (`start_date`, `end_date`, `value`), so a client picks the list for the year @@ -132,8 +134,8 @@ without that). Each API process also keeps the serialized bodies of `/v2/model-catalog` and `/v2/model-catalog/variables` once built, each with its `ETag` computed once, for up to eight route, country and version combinations (least recently used -dropped first; the 6.10.0 US variables body is 3,910,063 bytes uncompressed, -so eight entries hold about 11 MB for both countries' two routes at two +dropped first; the 6.10.0 US variables body is 4,190,824 bytes uncompressed, +so eight entries hold about 12 MB for both countries' two routes at two versions). A response is kept under the exact version it was read from, so a request without `policyengine_version` (the process's running version, which never changes while it runs) and one naming that version share it. Only diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py index 039c9dd03..1a1ad0b2d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -77,10 +77,10 @@ class MetadataResponseCache: are kept under the exact version they were read from: a request without a version (the running version, fixed for the process) and one naming that version share one response. The most recently used ``capacity`` responses - are kept: the US variables body of the 6.10.0 catalog is 3,910,063 bytes - uncompressed, so the default of eight holds about 11 MB for both + are kept: the US variables body of the 6.10.0 catalog is 4,190,824 bytes + uncompressed, so the default of eight holds about 12 MB for both countries' two whole-version routes at two versions, and at most about - 31 MB if every entry were that large. Requests that arrive while a read + 34 MB if every entry were that large. Requests that arrive while a read for the same key is in progress wait for it, and all receive its outcome: the kept response, or a copy of its error response. A failed read is not kept, so the first request after it reads again. diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 8234fb8db..00c197cac 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -126,6 +126,8 @@ def metadata_variable(variable: Variable) -> MetadataVariable: for enum_value in variable.enum_values ] ), + unit=variable.unit, + definition_period=variable.definition_period, ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 812f73638..52e8be0a0 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -119,9 +119,12 @@ class MetadataVariable(StrictResponseModel): subtracted from this one. When the model takes that list from a parameter, adds_parameter_path or subtracts_parameter_path names the parameter, whose dated values give the list for each year (adds and - subtracts hold its value on 2025-01-01). The catalog format 2 fields - (is_input_variable, hidden_input, module_name, the two parameter paths - and enum_values) are null in versions published before that format. + subtracts hold its value on 2025-01-01). unit is the country package's + unit (for example ``currency-USD``) and definition_period the period a + value covers (``year``, ``month`` or ``eternity``). The catalog format 2 + fields (is_input_variable, hidden_input, module_name, the two parameter + paths, enum_values, unit and definition_period) are null in versions + published before that format. """ id: UUID @@ -140,6 +143,8 @@ class MetadataVariable(StrictResponseModel): adds_parameter_path: str | None = None subtracts_parameter_path: str | None = None enum_values: list[MetadataEnumValue] | None = None + unit: str | None = None + definition_period: str | None = None class MetadataParameterSummary(StrictResponseModel): diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index e72cbd17f..7f479d183 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -488,6 +488,8 @@ def test_postgres_catalog_variables_carry_part_parameter_values_by_date( assert result["variables"][0]["module_name"] == ( "household.income.employment_income" ) + assert result["variables"][0]["unit"] == "currency-USD" + assert result["variables"][0]["definition_period"] == "year" assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" assert result["part_parameter_values"] == { "gov.example.parts": [ diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index bc5eafb7a..176fcc568 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -1420,3 +1420,15 @@ def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( assert _vary_value_counts(small) == expected assert _vary_value_counts(compressed) == expected assert _vary_value_counts(not_modified) == expected + + +def test_openapi_declares_each_variables_unit_and_definition_period() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + variable_properties = schema["components"]["schemas"]["MetadataVariable"][ + "properties" + ] + for field in ("unit", "definition_period"): + assert {"type": "string"} in variable_properties[field]["anyOf"] + assert {"type": "null"} in variable_properties[field]["anyOf"] diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 798c6ce02..454d90aad 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -124,6 +124,8 @@ def _insert_country( for enum_value in record.enum_values ] ), + unit=record.unit, + definition_period=record.definition_period, ) for record in country.variables ) @@ -1135,6 +1137,71 @@ def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) +def _uk_model_version(session: Session) -> TaxBenefitModelVersion: + return session.exec( + select(TaxBenefitModelVersion) + .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) + .where( + TaxBenefitModel.name == "policyengine-uk", + TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + ) + ).one() + + +def test_variables_carry_their_unit_and_definition_period( + catalog_session: Session, +) -> None: + uk_version = _uk_model_version(catalog_session) + catalog_session.add_all( + [ + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ), + # A variable published before catalog format 2 stored neither. + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="age", + entity="person", + data_type="int", + default_value=0, + ), + ] + ) + catalog_session.commit() + service = _service(catalog_session) + + us_variables = service.get_catalog_variables("us").variables + uk_variables = { + variable.name: variable + for variable in service.get_catalog_variables("uk").variables + } + us_page = service.list_variables("us").items + + assert [(item.unit, item.definition_period) for item in us_variables] == [ + ("currency-USD", "year") + ] + assert (us_page[0].unit, us_page[0].definition_period) == ( + "currency-USD", + "year", + ) + assert ( + uk_variables["universal_credit"].unit, + uk_variables["universal_credit"].definition_period, + ) == ("currency-GBP", "month") + assert (uk_variables["age"].unit, uk_variables["age"].definition_period) == ( + None, + None, + ) + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 32eb33615b9b8fde26a7ca47beb5b8c387814e8f Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:47:13 +0400 Subject: [PATCH 086/118] Require the catalog format 2 fields in the metadata response schemas The format 2 fields on MetadataVariable (input and hidden-input flags, module, the two parameter paths, enum values, unit and definition period) and on MetadataParameterSummary (economy and household) defaulted to None, so the OpenAPI schema left them out of required although every response includes them, and a generated client would type them as possibly missing. They are now required and nullable; every constructor already passes them, and the route test fixtures now do too. A test checks the OpenAPI required lists. Co-Authored-By: Claude Opus 5.5 --- .../services/v2/metadata/types.py | 20 ++++----- tests/unit/v2/test_metadata_routes.py | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 52e8be0a0..63a77944e 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -137,14 +137,14 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None - is_input_variable: bool | None = None - hidden_input: bool | None = None - module_name: str | None = None - adds_parameter_path: str | None = None - subtracts_parameter_path: str | None = None - enum_values: list[MetadataEnumValue] | None = None - unit: str | None = None - definition_period: str | None = None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: list[MetadataEnumValue] | None + unit: str | None + definition_period: str | None class MetadataParameterSummary(StrictResponseModel): @@ -161,8 +161,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None - economy: bool | None = None - household: bool | None = None + economy: bool | None + household: bool | None class MetadataCanonicalParameterValue(StrictResponseModel): diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 176fcc568..037b2952f 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -126,6 +126,14 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: default_value=0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter = MetadataParameterSummary( id=parameter_id, @@ -134,6 +142,8 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: description=None, data_type="float", unit="/1", + economy=True, + household=False, ) parameter_value = MetadataCanonicalParameterValue( id=parameter_value_id, @@ -1432,3 +1442,34 @@ def test_openapi_declares_each_variables_unit_and_definition_period() -> None: for field in ("unit", "definition_period"): assert {"type": "string"} in variable_properties[field]["anyOf"] assert {"type": "null"} in variable_properties[field]["anyOf"] + + +def test_openapi_requires_every_catalog_format_2_field_as_nullable() -> None: + # Every response includes these fields (null for versions published before + # catalog format 2), so a generated client must not type them as missing. + results, _ids = _resource_results() + schemas = ( + _client(lambda: ResourceReader(results)) + .get("/v2/openapi.json") + .json()["components"]["schemas"] + ) + format_2_fields = { + "MetadataVariable": { + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + }, + "MetadataParameterSummary": {"economy", "household"}, + "MetadataCatalogParameter": {"economy", "household"}, + } + + for schema_name, fields in format_2_fields.items(): + schema = schemas[schema_name] + assert set(schema["properties"]) <= set(schema["required"]), schema_name + for field in fields: + assert {"type": "null"} in schema["properties"][field]["anyOf"], field From 504f57e5d28c2de666e64a5f2d64953a428f3015 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:47:35 +0400 Subject: [PATCH 087/118] Look up test model versions through one helper taking the model name _uk_model_version duplicated _us_model_version except for the model name. Both are replaced by _model_version(session, model_name, version), which the tests call with the model they need. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 41 ++++++++++---------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 454d90aad..5772b8187 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -235,13 +235,15 @@ def _add_country_version( session.commit() -def _us_model_version(session: Session) -> TaxBenefitModelVersion: +def _model_version( + session: Session, model_name: str, version: str = POLICYENGINE_VERSION +) -> TaxBenefitModelVersion: return session.exec( select(TaxBenefitModelVersion) .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) .where( - TaxBenefitModel.name == "policyengine-us", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + TaxBenefitModel.name == model_name, + TaxBenefitModelVersion.version == version, ) ).one() @@ -259,7 +261,7 @@ def _service( def test_resource_collection_uses_bounded_pagination_without_counting( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") catalog_session.add( Variable( id=uuid4(), @@ -351,7 +353,7 @@ def record_statement(_connection, _cursor, statement, *_args) -> None: def test_parameter_values_are_separate_canonical_resources( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -412,7 +414,7 @@ def test_current_parameter_value_uses_inclusive_effective_dates( def test_parameter_children_are_loaded_one_level_at_a_time( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") nested_node = ParameterNode( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -547,7 +549,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( def test_dataset_collection_excludes_outputs_and_storage_references( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") output_id = uuid4() catalog_session.add( Dataset( @@ -661,7 +663,7 @@ def test_model_catalog_without_regions_offers_no_economy_calculation( ) -> None: # A household-only country publishes no regions or datasets; its catalog # still serves entities and basic inputs, with empty economy choices. - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") for region in catalog_session.exec( select(Region).where(Region.tax_benefit_model_version_id == model_version.id) ).all(): @@ -686,7 +688,7 @@ def test_model_catalog_without_regions_offers_no_economy_calculation( def test_model_catalog_published_before_format_2_has_empty_new_fields( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") model_version.catalog_format = None model_version.entities = None model_version.basic_inputs = None @@ -714,7 +716,7 @@ def test_model_catalog_with_regions_requires_the_national_region( def _add_variable_with_parameter_named_parts(session: Session) -> None: - model_version = _us_model_version(session) + model_version = _model_version(session, "policyengine-us") parts = Parameter( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -858,7 +860,7 @@ def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> N def _add_bracket_schedule(session: Session) -> None: - model_version = _us_model_version(session) + model_version = _model_version(session, "policyengine-us") session.add_all( [ ParameterNode( @@ -967,7 +969,7 @@ def test_named_parameters_list_names_the_version_lacks( def test_named_parameters_exclude_policy_and_dynamic_values( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -1097,7 +1099,7 @@ def test_value_and_node_label_reads_return_typed_rows( reads_parameters, ) - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") values = reads_parameters.read_current_law_values_by_parameter_name( catalog_session, model_version_id=model_version.id, @@ -1137,21 +1139,10 @@ def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) -def _uk_model_version(session: Session) -> TaxBenefitModelVersion: - return session.exec( - select(TaxBenefitModelVersion) - .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) - .where( - TaxBenefitModel.name == "policyengine-uk", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, - ) - ).one() - - def test_variables_carry_their_unit_and_definition_period( catalog_session: Session, ) -> None: - uk_version = _uk_model_version(catalog_session) + uk_version = _model_version(catalog_session, "policyengine-uk") catalog_session.add_all( [ Variable( From fc8f61f9bc1449d42be60c80a0b8c2820255b4e3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:48:13 +0400 Subject: [PATCH 088/118] Test that both variables routes serve unit and definition period Through the routes, with a response store: /v2/model-catalog/variables (read, then served from the stored body) and /v2/variables pages carry unit and definition_period for the US, for a UK variable with its own values, and as null keys for a version whose catalog_format is null. Co-Authored-By: Claude Opus 5.5 --- tests/unit/v2/test_metadata_service.py | 88 ++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 5772b8187..b2f35d7be 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -1193,6 +1193,94 @@ def test_variables_carry_their_unit_and_definition_period( ) +@pytest.mark.parametrize( + ("country_id", "version", "expected_catalog_format", "expected_fields"), + [ + ("us", None, 2, {"employment_income": ("currency-USD", "year")}), + ( + "uk", + None, + 2, + { + "employment_income": ("currency-USD", "year"), + "universal_credit": ("currency-GBP", "month"), + }, + ), + # A version published before catalog format 2 stored neither field. + ("us", "9.9.9", None, {"employment_income": (None, None)}), + ], +) +def test_both_variables_routes_serve_unit_and_definition_period( + catalog_session: Session, + country_id: str, + version: str | None, + expected_catalog_format: int | None, + expected_fields: dict[str, tuple[str | None, str | None]], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + + from .test_metadata_routes import _client + + catalog_session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=_model_version( + catalog_session, "policyengine-uk" + ).id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ) + ) + catalog_session.commit() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + earlier_version = _model_version(catalog_session, "policyengine-us", "9.9.9") + earlier_version.catalog_format = None + for variable in catalog_session.exec( + select(Variable).where( + Variable.tax_benefit_model_version_id == earlier_version.id + ) + ).all(): + variable.unit = None + variable.definition_period = None + catalog_session.commit() + client = _client( + lambda: _service(catalog_session), response_cache=MetadataResponseCache() + ) + query = f"country_id={country_id}" + ( + "" if version is None else f"&policyengine_version={version}" + ) + + whole_version_bodies = [ + client.get(f"/v2/model-catalog/variables?{query}").json()["result"] + for _attempt in ("read", "stored") + ] + page = client.get(f"/v2/variables?{query}").json()["result"] + + for result in whole_version_bodies: + assert result["catalog_format"] == expected_catalog_format + served = { + variable["name"]: (variable["unit"], variable["definition_period"]) + for variable in result["variables"] + } + assert served == expected_fields + paged = { + item["name"]: (item["unit"], item["definition_period"]) + for item in page["items"] + } + assert paged == expected_fields + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( From 3885872ae2a9eb525b1581d6e78aff8c15f5d935 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 26 Oct 2026 21:48:24 +0400 Subject: [PATCH 089/118] Say the same in the Stage 9 guide about who reads unit and definition period One paragraph said the household builder uses the two fields for formatting and per-period labels, while the guide's table says the builder and the breakdown read them. The paragraph now says the builder and breakdown read them, for currency formatting, per-period labels and household validation. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 3785f5fbc..8e9ea0123 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -48,8 +48,9 @@ version in name order, in one response, with the catalog format 2 fields: `subtracts_parameter_path`, `enum_values` (each value's name and its label, in the type the country package declares), `unit` (for example `currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which -the household builder uses for formatting and per-period labels. Where a variable's `adds` or -`subtracts` list comes from a parameter, `part_parameter_values` maps that +the household builder and breakdown read (currency formatting, per-period +labels and household validation). Where a variable's `adds` or `subtracts` list +comes from a parameter, `part_parameter_values` maps that parameter's path to its current-law values in start-date order (`start_date`, `end_date`, `value`), so a client picks the list for the year it shows instead of the list resolved on 2025-01-01; a path whose parameter From 8417d1c6b3edbd60e1f3d34c9cd304ec01536f3f Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 20:41:45 +0400 Subject: [PATCH 090/118] Record each catalog version's SPM availability from the simulation service Catalog versions gain spm_availability: whether the version offers SPM settings, with their JSON schema and the certified defaults, or {"available": false}. After publishing, catalog initialization reads the simulation service's public versions answer and records it on each published version that has none yet; a US version offers SPM when the service lists a canonical capability with pinned defaults for its PolicyEngine.py version, and other countries never do. A bundle pins its SPM release, so a recorded value never changes, and versions published earlier get it on the next run. The publication step now receives SIMULATION_ENTRYPOINT_URL. The three database steps of initialization are grouped as PublicationSteps. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/seed-v2-database.yml | 3 + .github/workflows/v2-integration-check.yml | 2 +- changelog.d/catalog-spm-availability.added.md | 1 + ...f3e8f662_store_catalog_spm_availability.py | 32 ++++ .../data/v2/catalog/initialization.py | 44 ++++- .../data/v2/catalog/spm_availability.py | 159 ++++++++++++++++++ policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/metadata.py | 5 + .../test_v2_catalog_spm_availability.py | 80 +++++++++ tests/unit/v2/test_alembic_v2.py | 3 +- tests/unit/v2/test_catalog_initialization.py | 17 +- .../unit/v2/test_catalog_spm_availability.py | 134 +++++++++++++++ tests/unit/v2/test_metadata_deployment.py | 6 + tests/unit/v2/test_models.py | 9 + 14 files changed, 481 insertions(+), 16 deletions(-) create mode 100644 changelog.d/catalog-spm-availability.added.md create mode 100644 migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py create mode 100644 policyengine_api/data/v2/catalog/spm_availability.py create mode 100644 tests/integration/test_v2_catalog_spm_availability.py create mode 100644 tests/unit/v2/test_catalog_spm_availability.py diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index 61af8978c..723b0bcb2 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -69,3 +69,6 @@ jobs: # Reads the certified UK data release, whose repository is private, # to check each new catalog version's region filters. HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + # Each version's SPM availability comes from this environment's + # simulation service's public versions answer. + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 6bf58aae2..599726735 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -78,7 +78,7 @@ jobs: env: RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication, resource routes and the countries list - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations diff --git a/changelog.d/catalog-spm-availability.added.md b/changelog.d/catalog-spm-availability.added.md new file mode 100644 index 000000000..bdf508050 --- /dev/null +++ b/changelog.d/catalog-spm-availability.added.md @@ -0,0 +1 @@ +Each published catalog version now records whether it offers SPM settings, with their schema and certified defaults, from the simulation service's public versions answer; the catalog publication step reads `SIMULATION_ENTRYPOINT_URL`, and versions published earlier get it on the next publication run. diff --git a/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py b/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py new file mode 100644 index 000000000..a8722ee1f --- /dev/null +++ b/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py @@ -0,0 +1,32 @@ +"""store catalog spm availability + +Revision ID: aeb5f3e8f662 +Revises: fb1a3e40d3d6 +Create Date: 2026-10-27 20:34:25.487436 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "aeb5f3e8f662" +down_revision: str | None = "fb1a3e40d3d6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "tax_benefit_model_versions", + sa.Column("spm_availability", sa.JSON(), nullable=True), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("tax_benefit_model_versions", "spm_availability") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 978e40e47..036b055d9 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -5,6 +5,7 @@ import json import os import sys +from dataclasses import dataclass from typing import TYPE_CHECKING from sqlalchemy import Engine, create_engine @@ -27,6 +28,9 @@ RegionDataVerificationError, verify_region_data, ) +from policyengine_api.data.v2.catalog.spm_availability import ( + record_published_spm_availability, +) from policyengine_api.data.v2.settings import ( V2ConfigurationError, V2DatabaseSettings, @@ -96,25 +100,47 @@ def verify_unpublished_region_data( ) +@dataclass(frozen=True) +class PublicationSteps: + """What initialization does with the database, in order, after extraction. + + ``verify_region_data`` checks the regions' certified data before anything + is written, ``publish`` writes the catalog, and ``record_spm_availability`` + then records each version's SPM availability from the simulation service + (``spm_availability``). + """ + + verify_region_data: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = verify_unpublished_region_data + publish: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( + publish_catalog + ) + record_spm_availability: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = record_published_spm_availability + + def initialize_catalog( environ: Mapping[str, str] | None = None, *, extractor: Callable[[], NormalizedCatalog] = extract_installed_catalog, engine_builder: Callable[[V2DatabaseSettings], Engine] = build_data_write_engine, - publisher: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( - publish_catalog - ), - region_data_verifier: Callable[ - [Engine, NormalizedCatalog, Mapping[str, str] | None], None - ] = verify_unpublished_region_data, + steps: PublicationSteps | None = None, ) -> PublicationEvidence: - """Load only row-write settings, extract fully, verify, then publish explicitly.""" + """Load only row-write settings, extract fully, verify, then publish explicitly. + + After publishing, record each version's SPM availability. + """ + selected_steps = steps or PublicationSteps() settings = load_v2_data_write_database_settings(environ) catalog = extractor() engine = engine_builder(settings) try: - region_data_verifier(engine, catalog, environ) - return publisher(engine, catalog) + selected_steps.verify_region_data(engine, catalog, environ) + evidence = selected_steps.publish(engine, catalog) + selected_steps.record_spm_availability(engine, catalog, environ) + return evidence finally: engine.dispose() diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py new file mode 100644 index 000000000..673759971 --- /dev/null +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -0,0 +1,159 @@ +"""Record whether each published catalog version offers SPM settings. + +The simulation service's public ``GET /versions`` lists, under +``spm_capabilities``, the certified SPM capability of each PolicyEngine.py +bundle it runs: its contract version and pinned defaults. The Public API +already checks economy requests' SPM selections against it. A US catalog +version offers SPM when the service reports a canonical capability for its +PolicyEngine.py version; other countries never do. + +Publication records it after the catalog itself, on each version that has +none yet, so versions published before it existed get it on the next run. A +bundle pins its SPM release, so a recorded value never changes. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any + +import httpx +import sqlalchemy as sa +from pydantic import ValidationError + +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) +from policyengine_api.data.v2.models import TaxBenefitModel, TaxBenefitModelVersion +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from sqlalchemy import Connection, Engine + + from policyengine_api.data.v2.catalog.records import NormalizedCatalog + +SIMULATION_ENTRYPOINT_URL = "SIMULATION_ENTRYPOINT_URL" +SPM_COUNTRY_ID = "us" +SPM_CONTRACT_VERSION = "canonical-spm-v1" +VERSIONS_TIMEOUT_SECONDS = 30.0 +UNAVAILABLE: dict[str, Any] = {"available": False} +MODELS = TaxBenefitModel.__table__ +MODEL_VERSIONS = TaxBenefitModelVersion.__table__ + + +def spm_availability( + country_id: str, policyengine_version: str, versions: object +) -> dict[str, Any]: + """Return ``{available, settings_schema, defaults}`` for one catalog version. + + Only a US version whose PolicyEngine.py version has a canonical capability + with pinned defaults (forecast artifact and scenario) offers SPM; anything + else is ``{"available": False}``. + """ + if country_id != SPM_COUNTRY_ID or not isinstance(versions, dict): + return dict(UNAVAILABLE) + capabilities = versions.get("spm_capabilities") + capability = ( + capabilities.get(policyengine_version) + if isinstance(capabilities, dict) + else None + ) + if ( + not isinstance(capability, dict) + or set(capability) != {"contract_version", "defaults"} + or capability["contract_version"] != SPM_CONTRACT_VERSION + ): + return dict(UNAVAILABLE) + try: + defaults = SPMSelection.model_validate(capability["defaults"]) + except ValidationError: + return dict(UNAVAILABLE) + if not defaults.forecast_content_sha256 or not defaults.scenario: + return dict(UNAVAILABLE) + return { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": { + name: getattr(defaults, name) for name in SPMSelection.model_fields + }, + } + + +def simulation_entrypoint_url(environ: Mapping[str, str]) -> str: + """Return the simulation service's base URL; raise when it is not set.""" + url = environ.get(SIMULATION_ENTRYPOINT_URL, "").strip().rstrip("/") + if not url: + msg = f"{SIMULATION_ENTRYPOINT_URL} is required to record SPM availability" + raise V2ConfigurationError(msg) + return url + + +def fetch_simulation_versions( + base_url: str, *, client: httpx.Client | None = None +) -> object: + """Read the simulation service's public ``GET /versions`` answer.""" + url = f"{base_url.rstrip('/')}/versions" + try: + if client is None: + response = httpx.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + else: + response = client.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + response.raise_for_status() + return response.json() + except (httpx.HTTPError, ValueError) as error: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) from error + + +def record_spm_availability( + connection: Connection, + *, + model_name: str, + version: str, + availability: Mapping[str, Any], +) -> bool: + """Store a catalog version's SPM availability unless it already has one. + + Return whether the version was changed. + """ + model_ids = sa.select(MODELS.c.id).where(MODELS.c.name == model_name) + result = connection.execute( + sa.update(MODEL_VERSIONS) + .where( + MODEL_VERSIONS.c.model_id.in_(model_ids), + MODEL_VERSIONS.c.version == version, + MODEL_VERSIONS.c.spm_availability.is_(None), + ) + .values(spm_availability=dict(availability)) + ) + return bool(result.rowcount) + + +def record_published_spm_availability( + engine: Engine, + catalog: NormalizedCatalog, + environ: Mapping[str, str] | None = None, + *, + fetch_versions: Callable[[str], object] = fetch_simulation_versions, +) -> None: + """Record each published country's SPM availability for the catalog version. + + Raise V2ConfigurationError without ``SIMULATION_ENTRYPOINT_URL``, and + CatalogPublicationError when the service's versions cannot be read. + """ + versions = fetch_versions( + simulation_entrypoint_url(os.environ if environ is None else environ) + ) + with engine.begin() as connection: + for country in catalog.countries: + record_spm_availability( + connection, + model_name=country.model.name, + version=country.model_version.version, + availability=spm_availability( + country.country_id, catalog.policyengine_version, versions + ), + ) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index 2c7b6986d..77e019ec6 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fb1a3e40d3d6" +V2_ALEMBIC_HEAD_REVISION = "aeb5f3e8f662" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 5431a6abc..5779d65b9 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -113,6 +113,11 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): catalog_format: int | None = None entities: list[CatalogEntity] | None = Field(default=None, sa_type=sa.JSON) basic_inputs: list[str] | None = Field(default=None, sa_type=sa.JSON) + # Whether this version offers SPM settings, with their schema and the + # certified defaults, as the simulation service reports its bundle's SPM + # support: {available, settings_schema, defaults}. Recorded after the + # catalog is published, so a version briefly has none. + spm_availability: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") variables: list["Variable"] = Relationship( diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py new file mode 100644 index 000000000..9df0b3564 --- /dev/null +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -0,0 +1,80 @@ +"""Record SPM availability on published catalog versions in disposable Postgres.""" + +from __future__ import annotations + +from uuid import uuid4 + +import sqlalchemy as sa +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog.spm_availability import ( + MODEL_VERSIONS, + MODELS, + record_spm_availability, +) + +AVAILABLE = {"available": True, "settings_schema": {}, "defaults": {"scenario": "x"}} +UNAVAILABLE = {"available": False} + + +def test_availability_fills_only_the_named_version_and_only_once( + disposable_v2_database_url: str, +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + model_id, version_id, other_version_id = uuid4(), uuid4(), uuid4() + model_name = f"policyengine-us-{model_id.hex[:8]}" + try: + with engine.begin() as connection: + connection.execute(sa.insert(MODELS).values(id=model_id, name=model_name)) + connection.execute( + sa.insert(MODEL_VERSIONS), + [ + { + "id": identifier, + "model_id": model_id, + "version": version, + "current_law_id": 2, + "metadata_time_periods": [2026], + } + for identifier, version in ( + (version_id, "1.450.0"), + (other_version_id, "1.449.0"), + ) + ], + ) + first = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=AVAILABLE, + ) + again = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=UNAVAILABLE, + ) + with engine.connect() as connection: + stored = dict( + connection.execute( + sa.select( + MODEL_VERSIONS.c.id, MODEL_VERSIONS.c.spm_availability + ).where(MODEL_VERSIONS.c.model_id == model_id) + ).all() + ) + finally: + with engine.begin() as connection: + connection.execute( + sa.delete(MODEL_VERSIONS).where(MODEL_VERSIONS.c.model_id == model_id) + ) + connection.execute(sa.delete(MODELS).where(MODELS.c.id == model_id)) + engine.dispose() + + assert first + assert not again + assert stored == {version_id: AVAILABLE, other_version_id: None} diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index b3dbb17f9..fbea9c3ef 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -248,6 +248,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + "fb1a3e40d3d6", "f5c1d9b4685b", SAVED_POLICY_RENAME_REVISION, SAVED_POLICY_RENAME_PARENT_REVISION, @@ -622,7 +623,7 @@ def _countries_revision_source() -> str: for revision in script.walk_revisions() if revision.down_revision == "f5c1d9b4685b" ) - assert countries_revision.revision == V2_ALEMBIC_HEAD_REVISION + assert countries_revision.revision == "fb1a3e40d3d6" assert countries_revision.path is not None return Path(countries_revision.path).read_text(encoding="utf-8") diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index d4d6c22c7..aee1342b5 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -78,12 +78,19 @@ def verify(selected_engine, catalog, environ): environ, ) + def record_spm(selected_engine, catalog, environ): + assert "engine" in observed + observed["spm"] = (selected_engine, catalog.policyengine_version, environ) + result = initialization.initialize_catalog( ENVIRONMENT, extractor=normalized_catalog, engine_builder=build, - publisher=publish, - region_data_verifier=verify, + steps=initialization.PublicationSteps( + verify_region_data=verify, + publish=publish, + record_spm_availability=record_spm, + ), ) assert result == _evidence() @@ -92,6 +99,7 @@ def verify(selected_engine, catalog, environ): "verified": (engine, POLICYENGINE_VERSION, ENVIRONMENT), "engine": engine, "version": POLICYENGINE_VERSION, + "spm": (engine, POLICYENGINE_VERSION, ENVIRONMENT), } assert engine.disposed @@ -112,8 +120,9 @@ def refuse_publication(_engine, _catalog): ENVIRONMENT, extractor=normalized_catalog, engine_builder=lambda _settings: engine, - publisher=refuse_publication, - region_data_verifier=reject, + steps=initialization.PublicationSteps( + verify_region_data=reject, publish=refuse_publication + ), ) assert engine.disposed diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py new file mode 100644 index 000000000..b07f3c6d8 --- /dev/null +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -0,0 +1,134 @@ +"""Each catalog version's SPM availability, from the simulation service's versions.""" + +from __future__ import annotations + +import httpx +import pytest + +from policyengine_api.data.v2.catalog import spm_availability +from policyengine_api.data.v2.catalog.publication_types import CatalogPublicationError +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +POLICYENGINE_VERSION = "6.10.0" +DEFAULTS = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +VERSIONS = { + "policyengine": {"latest": POLICYENGINE_VERSION}, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": DEFAULTS, + } + }, +} + + +def test_a_us_version_with_a_certified_capability_offers_spm() -> None: + availability = spm_availability.spm_availability( + "us", POLICYENGINE_VERSION, VERSIONS + ) + + assert availability == { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": DEFAULTS, + } + + +@pytest.mark.parametrize( + ("country_id", "policyengine_version", "versions"), + [ + ("uk", POLICYENGINE_VERSION, VERSIONS), + ("us", "6.11.0", VERSIONS), + ("us", POLICYENGINE_VERSION, {}), + ("us", POLICYENGINE_VERSION, {"spm_capabilities": []}), + ( + "us", + POLICYENGINE_VERSION, + { + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v0", + "defaults": DEFAULTS, + } + } + }, + ), + ( + "us", + POLICYENGINE_VERSION, + { + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": {**DEFAULTS, "scenario": None}, + } + } + }, + ), + ], + ids=[ + "uk", + "unlisted-version", + "no-capabilities", + "malformed-capabilities", + "other-contract", + "unpinned-defaults", + ], +) +def test_anything_else_offers_no_spm( + country_id: str, policyengine_version: str, versions: object +) -> None: + assert spm_availability.spm_availability( + country_id, policyengine_version, versions + ) == {"available": False} + + +def test_the_versions_come_from_the_simulation_services_public_route() -> None: + requests: list[httpx.Request] = [] + + def answer(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=VERSIONS) + + with httpx.Client(transport=httpx.MockTransport(answer)) as client: + versions = spm_availability.fetch_simulation_versions( + "https://simulation.example/", client=client + ) + + assert versions == VERSIONS + assert [str(request.url) for request in requests] == [ + "https://simulation.example/versions" + ] + assert "authorization" not in requests[0].headers + + +def test_an_unreadable_versions_answer_stops_the_recording() -> None: + with ( + httpx.Client( + transport=httpx.MockTransport(lambda _request: httpx.Response(503)) + ) as client, + pytest.raises(CatalogPublicationError, match="versions"), + ): + spm_availability.fetch_simulation_versions( + "https://simulation.example", client=client + ) + + +def test_recording_needs_the_simulation_service_url() -> None: + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + spm_availability.simulation_entrypoint_url({}) + + assert ( + spm_availability.simulation_entrypoint_url( + {"SIMULATION_ENTRYPOINT_URL": " https://simulation.example/ "} + ) + == "https://simulation.example" + ) diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index a5e51e864..1784a108a 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -54,6 +54,12 @@ def test_reusable_seeding_workflow_separates_database_credentials() -> None: assert "V2_MIGRATION_DATABASE_URL" not in publication assert "V2_RUNTIME_DATABASE_URL" not in workflow assert "scripts/publish_v2_metadata_catalog.py" in publication + # Publication records each version's SPM availability from the + # environment's own simulation service. + assert ( + "SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + in publication + ) assert (REPO / "scripts/publish_v2_metadata_catalog.py").is_file() assert not (REPO / "scripts/initialize_v2_metadata.py").exists() diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 66ac8b49b..88ed2db5e 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -695,6 +695,15 @@ def test_saved_policy_table_is_named_like_the_other_association_tables() -> None } +def test_spm_availability_is_nullable_for_versions_not_yet_recorded() -> None: + """Publication records it after the catalog, so a version may lack it briefly.""" + column = V2_METADATA.tables["tax_benefit_model_versions"].c["spm_availability"] + + assert column.nullable + assert column.server_default is None + assert isinstance(column.type, sa.JSON) + + def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: """Catalog format 2 stores v3's metadata; versions published before it keep null. From fcad8fd6fe3c91c79210cee0417eced3cbbfa256 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 20:49:17 +0400 Subject: [PATCH 091/118] Serve each catalog version's SPM availability at /v2/model-catalog The model catalog result gains spm: available, and when it is, the JSON schema of a household's SPM selection and the certified defaults. A version whose availability is not recorded yet, or that does not offer SPM, is not available. The route contract and the stage 9 guide name the field. Co-Authored-By: Claude Opus 5.5 --- .../model-catalog-spm-availability.added.md | 1 + docs/engineering/migration-contracts.md | 2 +- docs/generated/migration_contracts.json | 3 +- docs/migration/stage-9-v2-metadata.md | 9 +++++- .../services/v2/metadata/transformations.py | 7 +++- .../services/v2/metadata/types.py | 20 ++++++++++-- tests/contract/registry.py | 1 + tests/unit/v2/test_metadata_routes.py | 11 +++++++ tests/unit/v2/test_metadata_service.py | 32 +++++++++++++++++++ 9 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 changelog.d/model-catalog-spm-availability.added.md diff --git a/changelog.d/model-catalog-spm-availability.added.md b/changelog.d/model-catalog-spm-availability.added.md new file mode 100644 index 000000000..b8515016b --- /dev/null +++ b/changelog.d/model-catalog-spm-availability.added.md @@ -0,0 +1 @@ +`GET /v2/model-catalog` now returns `spm`: whether the catalog version offers SPM settings for a household, with their JSON schema and certified defaults when it does. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 67d63c2b1..31614d7d4 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -224,7 +224,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/regions/by-code/state/ca?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.item` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | -| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options`, `result.spm` | | `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | | `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index b6a956745..928b956b3 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1651,7 +1651,8 @@ "result.basic_inputs", "result.regions", "result.datasets", - "result.economy_options" + "result.economy_options", + "result.spm" ] }, { diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8e9ea0123..d02cbd0b4 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -36,7 +36,14 @@ version (with the current-law policy id and the years offered), the entities input dataset, and the economy choices. A catalog without regions, such as a household-only country's, has empty region, dataset and economy-choice lists rather than an error. A version published before catalog format 2 has a null -`catalog_format` and empty entities and basic inputs. The result names the +`catalog_format` and empty entities and basic inputs. `spm` says whether the +version offers SPM settings for a household: `available`, and when it does, +`settings_schema` (the JSON schema of a household's `spm` selection) and +`defaults` (the certified selection). Catalog publication records it from the +simulation service's public `GET /versions`: a US version offers SPM when the +service lists a canonical capability with pinned defaults for its +PolicyEngine.py version, and other countries never do. A version whose +availability is not recorded yet offers none. The result names the selected PolicyEngine.py version, so a client that starts without a version learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 00c197cac..67d860a56 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -29,6 +29,7 @@ MetadataRegion, MetadataRegionOption, MetadataRegionType, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, ParameterNodeLabelRow, @@ -402,7 +403,8 @@ def metadata_model_catalog( Without a national dataset (a catalog without regions) the economy choices are empty. Entities and basic inputs a version published before catalog - format 2 lacks are empty lists. + format 2 lacks are empty lists, and a version whose SPM availability is + not recorded yet offers none. """ model_version = selected.model_version return MetadataModelCatalogResult( @@ -424,4 +426,7 @@ def metadata_model_catalog( if national_dataset is not None else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) ), + spm=MetadataSPMAvailability.model_validate( + model_version.spm_availability or {"available": False} + ), ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 63a77944e..eb15bca55 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -326,14 +326,29 @@ class MetadataEconomyOptions(StrictResponseModel): datasets: list[MetadataDatasetOption] +class MetadataSPMAvailability(StrictResponseModel): + """Whether a catalog version offers SPM settings for a household. + + When ``available``, ``settings_schema`` is the JSON schema of a + household's ``spm`` selection and ``defaults`` the certified selection + that applies to every field a household leaves out. A version without + them (any country but the US, or a bundle without certified SPM support) + is not available, and both are null. + """ + + available: bool + settings_schema: dict[str, JsonValue] | None = None + defaults: dict[str, JsonValue] | None = None + + class MetadataModelCatalogResult(StrictResponseModel): """The small parts of one catalog version, read when a client starts. policyengine_version names the selected catalog version, so a client can request its other parts for that exact version. catalog_format is null, and entities and basic_inputs are empty, for a version published before - catalog format 2. Variables, parameters and their values are served by - their own routes. + catalog format 2. ``spm`` says whether the version offers SPM settings. + Variables, parameters and their values are served by their own routes. """ policyengine_version: str @@ -345,6 +360,7 @@ class MetadataModelCatalogResult(StrictResponseModel): regions: list[MetadataRegion] datasets: list[MetadataDataset] economy_options: MetadataEconomyOptions + spm: MetadataSPMAvailability class MetadataDatedValue(StrictResponseModel): diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 02a53761f..ac187802d 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -703,6 +703,7 @@ class WorkflowContract: "result.regions", "result.datasets", "result.economy_options", + "result.spm", ), route_group="metadata", ), diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 2ba7580ef..a1e4708c1 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -51,6 +51,7 @@ MetadataParameterSummary, MetadataRegion, MetadataRegionOption, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, ParameterFilter, @@ -273,6 +274,11 @@ def detail(item: object) -> MetadataDetailResult: ) ], ), + spm=MetadataSPMAvailability( + available=True, + settings_schema={"type": "object"}, + defaults={"scenario": "ce_trend"}, + ), ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, @@ -789,6 +795,11 @@ def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> assert result["basic_inputs"] == ["employment_income"] assert [region["code"] for region in result["regions"]] == ["us"] assert result["economy_options"]["datasets"][0]["default"] is True + assert result["spm"] == { + "available": True, + "settings_schema": {"type": "object"}, + "defaults": {"scenario": "ce_trend"}, + } assert reader.calls == [("get_model_catalog", ("us", None), {})] assert reader.closed diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index b2f35d7be..6b6a19f32 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -702,6 +702,38 @@ def test_model_catalog_published_before_format_2_has_empty_new_fields( assert result.basic_inputs == [] +SPM_AVAILABLE = { + "available": True, + "settings_schema": {"title": "SPMSelection", "type": "object"}, + "defaults": {"scenario": "ce_trend", "geography_kind": "county"}, +} + + +def test_model_catalog_serves_the_versions_recorded_spm_availability( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + model_version.spm_availability = SPM_AVAILABLE + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.spm.model_dump(mode="json") == SPM_AVAILABLE + + +def test_model_catalog_without_recorded_spm_availability_offers_none( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_model_catalog("uk") + + assert result.spm.model_dump(mode="json") == { + "available": False, + "settings_schema": None, + "defaults": None, + } + + def test_model_catalog_with_regions_requires_the_national_region( catalog_session: Session, ) -> None: From 54071cdf7d252a21c04068fbaa80f8ad273d7604 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 21:25:46 +0400 Subject: [PATCH 092/118] Convert stored households and policies to what the simulation service takes A stored v2 household becomes the situation shape the simulation service's household functions take: people and each group collection by its plural, each group listing the people whose memberships name it, every record named by its ID so a calculated value maps back to its record. The saved SPM choice is read separately. A v2 policy's values become parameter values keyed by YYYY-MM-DD.YYYY-MM-DD period ranges, with an open end as 2100-12-31. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/__init__.py | 1 + .../household_calculations/transformations.py | 79 ++++++++ ...t_household_calculation_transformations.py | 172 ++++++++++++++++++ 3 files changed, 252 insertions(+) create mode 100644 policyengine_api/services/v2/household_calculations/__init__.py create mode 100644 policyengine_api/services/v2/household_calculations/transformations.py create mode 100644 tests/unit/v2/test_household_calculation_transformations.py diff --git a/policyengine_api/services/v2/household_calculations/__init__.py b/policyengine_api/services/v2/household_calculations/__init__.py new file mode 100644 index 000000000..0501ab630 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Household calculations on v2, run by the Simulation Entrypoint.""" diff --git a/policyengine_api/services/v2/household_calculations/transformations.py b/policyengine_api/services/v2/household_calculations/transformations.py new file mode 100644 index 000000000..53911db0f --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/transformations.py @@ -0,0 +1,79 @@ +"""Stored households and policies, in the shapes the Simulation Entrypoint takes. + +A stored v2 household lists its people and each group entity as records, +each with an ``id`` and its ``values``; each person names, for every group +collection, the ``id`` of the group it belongs to. The Simulation Entrypoint's +household functions take the Public API's situation shape instead: ``people`` +maps each person to their values, and each group collection, by its plural, +maps each group to its ``members`` and values. Records keep their ``id`` as +their name, so a calculated value maps back to its record. + +A v2 policy stores each parameter value with its start date and an optional +end date; the household functions take the Public API's policy JSON, each +parameter's values keyed by ``YYYY-MM-DD.YYYY-MM-DD`` period ranges, where an +open end is ``2100-12-31``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Mapping + + from policyengine_api.services.v2.policies.types import PolicyRead + +# Each country's stored group collections and their plural in the situation. +GROUP_COLLECTIONS: dict[str, dict[str, str]] = { + "us": { + "household": "households", + "family": "families", + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "marital_unit": "marital_units", + }, + "uk": {"household": "households", "benunit": "benunits"}, +} +OPEN_END_DATE = "2100-12-31" + + +def household_situation( + country_id: str, household_data: Mapping[str, Any] +) -> dict[str, Any]: + """Return a stored household's people and groups in the situation shape. + + The saved SPM choice is not part of the situation; ``saved_spm_choice`` + reads it. + """ + people = household_data["people"] + situation: dict[str, Any] = { + "people": {person["id"]: dict(person["values"]) for person in people} + } + for collection, plural in GROUP_COLLECTIONS[country_id].items(): + situation[plural] = { + group["id"]: { + "members": [ + person["id"] + for person in people + if person["memberships"].get(collection) == group["id"] + ], + **group["values"], + } + for group in household_data.get(collection, []) + } + return situation + + +def saved_spm_choice(household_data: Mapping[str, Any]) -> object: + """Return the household's saved SPM selection, or None when it has none.""" + return household_data.get("spm") + + +def policy_parameters(policy: PolicyRead) -> dict[str, dict[str, Any]]: + """Return the policy's values by parameter and period range.""" + parameters: dict[str, dict[str, Any]] = {} + for value in policy.parameter_values: + end = OPEN_END_DATE if value.end_date is None else value.end_date.date() + period = f"{value.start_date.date().isoformat()}.{end}" + parameters.setdefault(value.parameter_name, {})[period] = value.value + return parameters diff --git a/tests/unit/v2/test_household_calculation_transformations.py b/tests/unit/v2/test_household_calculation_transformations.py new file mode 100644 index 000000000..6efd6e8f7 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_transformations.py @@ -0,0 +1,172 @@ +"""Stored households and policies, as the Simulation Entrypoint takes them.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from uuid import uuid4 + +from policyengine_api.services.v2.household_calculations.transformations import ( + household_situation, + policy_parameters, + saved_spm_choice, +) +from policyengine_api.services.v2.policies.types import ( + PolicyParameterValueRead, + PolicyRead, +) + +US_HOUSEHOLD = { + "people": [ + { + "id": "person-1", + "source_name": "you", + "values": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-1", + }, + }, + { + "id": "person-2", + "values": {"age": {"2026": 8}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-2", + }, + }, + ], + "household": [{"id": "household-1", "values": {"state_name": {"2026": "CA"}}}], + "family": [{"id": "family-1", "values": {}}], + "tax_unit": [{"id": "tax_unit-1", "values": {"eitc": {"2026": None}}}], + "spm_unit": [{"id": "spm_unit-1", "values": {}}], + "marital_unit": [ + {"id": "marital_unit-1", "values": {}}, + {"id": "marital_unit-2", "values": {}}, + ], + "spm": {"scenario": "ce_trend", "geography_kind": "national"}, +} +UK_HOUSEHOLD = { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 30}}, + "memberships": {"household": "household-1", "benunit": "benunit-1"}, + } + ], + "household": [{"id": "household-1", "values": {"region": {"2026": "WALES"}}}], + "benunit": [{"id": "benunit-1", "values": {"universal_credit": {"2026": None}}}], +} + + +def test_a_us_household_becomes_a_situation_keyed_by_record_id() -> None: + assert household_situation("us", US_HOUSEHOLD) == { + "people": { + "person-1": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "person-2": {"age": {"2026": 8}}, + }, + "households": { + "household-1": { + "members": ["person-1", "person-2"], + "state_name": {"2026": "CA"}, + } + }, + "families": {"family-1": {"members": ["person-1", "person-2"]}}, + "tax_units": { + "tax_unit-1": {"members": ["person-1", "person-2"], "eitc": {"2026": None}} + }, + "spm_units": {"spm_unit-1": {"members": ["person-1", "person-2"]}}, + "marital_units": { + "marital_unit-1": {"members": ["person-1"]}, + "marital_unit-2": {"members": ["person-2"]}, + }, + } + + +def test_a_uk_household_has_households_and_benefit_units() -> None: + assert household_situation("uk", UK_HOUSEHOLD) == { + "people": {"person-1": {"age": {"2026": 30}}}, + "households": { + "household-1": {"members": ["person-1"], "region": {"2026": "WALES"}} + }, + "benunits": { + "benunit-1": { + "members": ["person-1"], + "universal_credit": {"2026": None}, + } + }, + } + + +def test_a_group_without_members_keeps_an_empty_member_list() -> None: + household = {**UK_HOUSEHOLD, "benunit": [*UK_HOUSEHOLD["benunit"]]} + household["benunit"].append({"id": "benunit-2", "values": {}}) + + situation = household_situation("uk", household) + + assert situation["benunits"]["benunit-2"] == {"members": []} + + +def test_the_saved_spm_choice_is_the_documents_selection_or_none() -> None: + assert saved_spm_choice(US_HOUSEHOLD) == { + "scenario": "ce_trend", + "geography_kind": "national", + } + assert saved_spm_choice(UK_HOUSEHOLD) is None + + +def _value(*, name: str, value: object, start: datetime, end: datetime | None): + return PolicyParameterValueRead( + id=uuid4(), + parameter_id=uuid4(), + parameter_name=name, + value=value, + start_date=start, + end_date=end, + ) + + +def test_a_policy_becomes_parameter_values_by_period_range() -> None: + policy = PolicyRead( + id=uuid4(), + country_id="us", + tax_benefit_model_id=uuid4(), + tax_benefit_model_version_id=uuid4(), + created_at=datetime(2026, 1, 1, tzinfo=UTC), + updated_at=datetime(2026, 1, 1, tzinfo=UTC), + parameter_values=( + _value( + name="gov.irs.credits.ctc.amount.base[0].amount", + value=3000, + start=datetime(2026, 1, 1, tzinfo=UTC), + end=None, + ), + _value( + name="gov.irs.credits.ctc.refundable.fully_refundable", + value=True, + start=datetime(2026, 1, 1, tzinfo=UTC), + end=datetime(2027, 12, 31, tzinfo=UTC), + ), + _value( + name="gov.irs.credits.ctc.amount.base[0].amount", + value=3500, + start=datetime(2028, 1, 1, tzinfo=UTC), + end=None, + ), + ), + ) + + assert policy_parameters(policy) == { + "gov.irs.credits.ctc.amount.base[0].amount": { + "2026-01-01.2100-12-31": 3000, + "2028-01-01.2100-12-31": 3500, + }, + "gov.irs.credits.ctc.refundable.fully_refundable": { + "2026-01-01.2027-12-31": True + }, + } From d12b80451cc87ffa4b6d7dc393902f41db557c37 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 21:49:03 +0400 Subject: [PATCH 093/118] Calculate stored households through /v2/household-calculations POST /v2/household-calculations checks the country's countries row allows household calculations (422 naming the country otherwise), reads the stored household and optional policy through the households and policies services, selects the country's running catalog version and sends the situation, saved SPM choice, policy parameters and options (axes, add_missing_variables) to the Simulation Entrypoint. GET /v2/household-calculations/{calculation_id} reads the calculation back with its household, warnings, spm_config, spm_provenance and error. Neither route needs an account. The simulation service is called through the generated simulation client, one per process, with the gateway credentials and correlation headers; a missing or partial configuration, an unreachable service, an error answer or a refusal is a 503. Co-Authored-By: Claude Opus 5.5 --- .../v2-household-calculations.added.md | 1 + docs/migration/stage-11-v2-households.md | 36 +++ .../fastapi_routes/dependencies.py | 52 ++++ .../v2/household_calculations/__init__.py | 1 + .../household_calculations/request_models.py | 27 ++ .../household_calculations/response_models.py | 112 ++++++++ .../v2/household_calculations/routes.py | 147 +++++++++++ policyengine_api/fastapi_routes/v2/routes.py | 4 + .../v2/household_calculations/services.py | 136 ++++++++++ .../simulation_client.py | 171 ++++++++++++ .../v2/household_calculations/types.py | 205 +++++++++++++++ .../v2/household_calculations/validators.py | 47 ++++ pyproject.toml | 4 + .../unit/v2/household_calculation_support.py | 248 ++++++++++++++++++ .../v2/test_household_calculation_routes.py | 224 ++++++++++++++++ .../v2/test_household_calculation_service.py | 246 +++++++++++++++++ ...household_calculation_simulation_client.py | 166 ++++++++++++ tests/unit/v2/test_metadata_routes.py | 2 + uv.lock | 15 ++ 19 files changed, 1844 insertions(+) create mode 100644 changelog.d/v2-household-calculations.added.md create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/request_models.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/response_models.py create mode 100644 policyengine_api/fastapi_routes/v2/household_calculations/routes.py create mode 100644 policyengine_api/services/v2/household_calculations/services.py create mode 100644 policyengine_api/services/v2/household_calculations/simulation_client.py create mode 100644 policyengine_api/services/v2/household_calculations/types.py create mode 100644 policyengine_api/services/v2/household_calculations/validators.py create mode 100644 tests/unit/v2/household_calculation_support.py create mode 100644 tests/unit/v2/test_household_calculation_routes.py create mode 100644 tests/unit/v2/test_household_calculation_service.py create mode 100644 tests/unit/v2/test_household_calculation_simulation_client.py diff --git a/changelog.d/v2-household-calculations.added.md b/changelog.d/v2-household-calculations.added.md new file mode 100644 index 000000000..9a4a83ccc --- /dev/null +++ b/changelog.d/v2-household-calculations.added.md @@ -0,0 +1 @@ +`POST /v2/household-calculations` sends a stored household, under a stored policy or current law, with optional axes, to the simulation service for calculation, and `GET /v2/household-calculations/{calculation_id}` returns its status and, once it succeeded, the calculated household with its warnings and, in the US, the SPM selection applied and its receipt. Neither needs an account; a country whose `countries` row does not allow household calculations is refused with 422. diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index c9a68eb1c..2a58f7dd1 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -256,3 +256,39 @@ Apply the reviewed schema revisions and application image only to independently verified production targets. Enable native v2 use and v1 create copying as separate operational decisions and retain the preceding application revision for immediate rollback. + +## Household Calculations + +`POST /v2/household-calculations` calculates a household stored through +`/v2/households`, under a policy stored through `/v2/policies` or, without +`policy_id`, under current law. The body may add `year` (the household's +`default_year` otherwise), `axes`, `add_missing_variables` and `force`. The +Public API checks the country's `countries` row allows household calculations +(422 otherwise; a household-only country is accepted), reads the household and +policy through the same services as their routes (404 when either is not +stored in that country), selects the country's running catalog version, and +sends the household in the situation shape, its saved SPM choice and the +policy's parameter values to the simulation service, which cannot read these +tables. The answer is 202 with the calculation's `calculation_id` and +`status`. Poll `GET /v2/household-calculations/{calculation_id}` until +`status` is `succeeded`, with `result`, or `failed`, with `error`. A US result +carries `spm_config` and `spm_provenance`. + +The simulation service stores every calculation as a simulation run, so a +calculation no account saved is still stored, with no report. Neither route +needs an account. + +The routes always call the Cloud Run Simulation Entrypoint, whatever +`SIM_ENTRYPOINT` selects, because only it serves v2 calculations. They read: + +- `SIMULATION_ENTRYPOINT_URL`: the `staging` and `production` environment + secret of the same name, passed to the deployment job's `env:` and written + to the Cloud Run revision by `deploy_cloud_run_candidate.sh` when set. +- `GATEWAY_AUTH_ISSUER`, `GATEWAY_AUTH_AUDIENCE`, `GATEWAY_AUTH_CLIENT_ID` and + `GATEWAY_AUTH_CLIENT_SECRET_RESOURCE`: the organization secrets already used + for every simulation call, with `GATEWAY_AUTH_REQUIRED=1`. + +Without the URL, or with gateway auth partly configured, both routes answer +503. They also answer 503 when the simulation service cannot be reached, +answers with an error, or refuses the calculation (for example when no v2 +worker is deployed for the catalog version's PolicyEngine.py version). diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index e6336d5e5..5d0bff862 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -29,6 +29,10 @@ LegacyReferenceChecker, ) from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + HouseholdCalculationRead, + ) from policyengine_api.services.v2.households.types import ( HouseholdPage, HouseholdRead, @@ -444,6 +448,29 @@ def list_households( ... +class V2HouseholdCalculationResourceService(Protocol): + """Route-independent household calculation operations for one request.""" + + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationRead: + """Send a stored household's calculation to the simulation service. + + Raise ``HouseholdCalculationCountryError`` for a country without + household calculations, ``HouseholdNotFoundError`` or + ``PolicyNotFoundError`` for a record not stored in that country, and + the simulation service errors when it cannot calculate. + """ + ... + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation as it stands on the simulation service. + + Raise ``HouseholdCalculationNotFoundError`` when it has none. + """ + ... + + class V2UserPolicyResourceService(Protocol): """Route-independent user-policy operations for one request.""" @@ -857,6 +884,25 @@ def _default_v2_household_service_factory() -> V2HouseholdResourceService: ) +def _default_v2_household_calculation_service_factory() -> ( + V2HouseholdCalculationResourceService +): + from policyengine_api.services.v2.household_calculations.services import ( # noqa: PLC0415 # loaded on the first calculation request only + V2HouseholdCalculationService, + ) + from policyengine_api.services.v2.household_calculations.simulation_client import ( # noqa: PLC0415 # imports the generated simulation client on first use only + process_simulation_household_client, + ) + + return V2HouseholdCalculationService( + countries=_default_v2_country_service_factory(), + households=_default_v2_household_service_factory(), + policies=_default_v2_policy_service_factory(), + catalogs=_default_v2_metadata_reader_factory(), + simulation=process_simulation_household_client(), + ) + + _user_bearer_scheme = HTTPBearer( auto_error=False, scheme_name="UserAccessToken", @@ -1006,6 +1052,9 @@ class NativeRouteDependencies: v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None + v2_household_calculation_service_factory: ( + Callable[[], V2HouseholdCalculationResourceService] | None + ) = None v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( None ) @@ -1041,6 +1090,9 @@ def defaults(cls) -> NativeRouteDependencies: v2_country_service_factory=_default_v2_country_service_factory, v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, + v2_household_calculation_service_factory=( + _default_v2_household_calculation_service_factory + ), v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), v2_user_household_service_factory=( _default_v2_user_household_service_factory diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py new file mode 100644 index 000000000..59e3798f7 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Native FastAPI routes calculating stored v2 households.""" diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py new file mode 100644 index 000000000..c8b0e4427 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py @@ -0,0 +1,27 @@ +"""Strict HTTP request models for v2 household calculations.""" + +from __future__ import annotations + +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, +) + + +class HouseholdCalculationRequest(HouseholdCalculationInput): + """A stored household to calculate under a stored policy, or current law. + + ``household_id`` and ``policy_id`` name records of ``country_id`` stored + through ``/v2/households`` and ``/v2/policies``; without ``policy_id`` the + household is calculated under current law. ``year`` defaults to the + household's ``default_year``. Each group in ``axes`` varies its inputs + together across ``count`` points; several groups form a grid. + ``add_missing_variables`` adds every variable the household lacks before + calculating, and ``force`` calculates again even when a recent identical + calculation exists. + """ + + def to_service_input(self) -> HouseholdCalculationInput: + """Convert the request to the calculation service's input.""" + return HouseholdCalculationInput.model_validate( + self.model_dump(mode="python", exclude_unset=True) + ) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py new file mode 100644 index 000000000..b2b1b160d --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -0,0 +1,112 @@ +"""Strict HTTP response models for v2 household calculations.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, JsonValue + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse + +if TYPE_CHECKING: + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationRead, + ) + + +class StrictHouseholdCalculationAPIModel(BaseModel): + """Base for calculation responses; rejects unknown fields and NaN or infinity.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, from_attributes=True) + + +class HouseholdCalculationOutput(StrictHouseholdCalculationAPIModel): + """A succeeded calculation's household. + + ``household`` is the stored household in the situation shape (people and + groups keyed by their record IDs) with each calculated value filled in by + period. ``warnings`` lists inputs the country model ignored. In the US, + ``spm_config`` is the SPM selection the calculation applied and + ``spm_provenance`` its receipt; both are null elsewhere. + ``policyengine_version`` is the PolicyEngine.py version that calculated it. + """ + + household: dict[str, JsonValue] + warnings: list[str] + spm_config: dict[str, JsonValue] | None + spm_provenance: dict[str, JsonValue] | None + policyengine_version: str | None + + +class HouseholdCalculationError(StrictHouseholdCalculationAPIModel): + """Why a calculation failed; ``detail`` holds the calculation's own error.""" + + code: str + message: str + detail: dict[str, JsonValue] | None + + +class HouseholdCalculationItem(StrictHouseholdCalculationAPIModel): + """A household calculation and its outcome. + + ``status`` moves from ``pending`` through ``dispatching`` and ``running`` + to ``succeeded``, with ``result``, or ``failed``, with ``error``. Poll + ``GET /v2/household-calculations/{calculation_id}`` until it is one of + those two. ``catalog_version_id`` is the catalog version calculated. + """ + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + status: Literal["pending", "dispatching", "running", "succeeded", "failed"] + created_at: datetime + completed_at: datetime | None + result: HouseholdCalculationOutput | None + error: HouseholdCalculationError | None + + @classmethod + def from_read( + cls, calculation: HouseholdCalculationRead + ) -> HouseholdCalculationItem: + """Build the response item from the service's calculation.""" + return cls.model_validate(calculation) + + +class HouseholdCalculationResponse(StrictHouseholdCalculationAPIModel): + """Success envelope: ``status`` "ok", a null ``message`` and the calculation.""" + + status: Literal["ok"] = "ok" + message: None = None + result: HouseholdCalculationItem + + +HOUSEHOLD_CALCULATION_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 404: { + "model": V2ErrorResponse, + "description": "The household, policy or calculation does not exist.", + }, + 422: { + "model": V2ErrorResponse, + "description": ( + "The request does not match the schema, the country does not offer " + "household calculations, or no year can be chosen." + ), + }, + 500: { + "model": V2ErrorResponse, + "description": "The household calculation request failed unexpectedly.", + }, + 503: { + "model": V2ErrorResponse, + "description": ( + "The simulation service, the country's catalog version or v2 " + "storage is unavailable, or the simulation service cannot " + "calculate this household." + ), + }, +} diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py new file mode 100644 index 000000000..c88ce2289 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -0,0 +1,147 @@ +"""Native FastAPI routes submitting stored households for calculation, and polling them. + +The calculation itself runs on the simulation service, which stores every +calculation as a simulation run; a calculation not saved to an account has no +report. The routes need no account. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar +from uuid import UUID + +from fastapi import APIRouter +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + UnsupportedPreviewCountryError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_household_calculation_service_factory, +) +from policyengine_api.fastapi_routes.v2.errors import V2ErrorMapping, run_v2_operation +from policyengine_api.fastapi_routes.v2.household_calculations.request_models import ( + HouseholdCalculationRequest, +) +from policyengine_api.fastapi_routes.v2.household_calculations.response_models import ( + HOUSEHOLD_CALCULATION_ERROR_RESPONSES, + HouseholdCalculationItem, + HouseholdCalculationResponse, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2HouseholdCalculationResourceService, + ) + +OperationT = TypeVar("OperationT") + +HOUSEHOLD_CALCULATION_ERROR_MAPPINGS = ( + V2ErrorMapping( + (HouseholdCalculationCountryError, HouseholdCalculationYearError), 422 + ), + V2ErrorMapping( + ( + HouseholdNotFoundError, + PolicyNotFoundError, + HouseholdCalculationNotFoundError, + ), + 404, + ), + V2ErrorMapping( + (SimulationServiceUnavailableError, SimulationServiceRefusedError), 503 + ), + V2ErrorMapping( + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + UnsupportedPreviewCountryError, + ), + 503, + "The country's catalog version is unavailable", + ), + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 household calculation storage is unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2HouseholdCalculationResourceService]: + if dependencies.v2_household_calculation_service_factory is not None: + return dependencies.v2_household_calculation_service_factory + return _default_v2_household_calculation_service_factory + + +def _calculation_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + return run_v2_operation( + operation, + HOUSEHOLD_CALCULATION_ERROR_MAPPINGS, + fallback_message="V2 household calculation failed", + ) + + +def build_v2_household_calculation_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the household calculation routes without opening any connection.""" + router = APIRouter(prefix="/v2", responses=HOUSEHOLD_CALCULATION_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.post( + "/household-calculations", + response_model=HouseholdCalculationResponse, + status_code=202, + summary="Calculate a stored household under a stored policy", + ) + def submit_household_calculation( + body: HouseholdCalculationRequest, + ) -> HouseholdCalculationResponse | JSONResponse: + def submit() -> HouseholdCalculationResponse: + calculation = service_factory().submit_calculation(body.to_service_input()) + return HouseholdCalculationResponse( + result=HouseholdCalculationItem.from_read(calculation) + ) + + return _calculation_operation(submit) + + @router.get( + "/household-calculations/{calculation_id}", + response_model=HouseholdCalculationResponse, + summary="Read a household calculation and its result", + ) + def get_household_calculation( + calculation_id: UUID, + ) -> HouseholdCalculationResponse | JSONResponse: + def read() -> HouseholdCalculationResponse: + calculation = service_factory().get_calculation(calculation_id) + return HouseholdCalculationResponse( + result=HouseholdCalculationItem.from_read(calculation) + ) + + return _calculation_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index d3b412252..1aa2032b3 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -11,6 +11,9 @@ build_v2_country_router, ) from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( + build_v2_household_calculation_router, +) from policyengine_api.fastapi_routes.v2.households.routes import ( build_v2_household_router, ) @@ -58,6 +61,7 @@ def build_v2_router( router.include_router(build_v2_user_household_router(dependencies)) router.include_router(build_v2_user_policy_router(dependencies)) router.include_router(build_v2_household_router(dependencies)) + router.include_router(build_v2_household_calculation_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) router.include_router(build_v2_country_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py new file mode 100644 index 000000000..3eafaeb71 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -0,0 +1,136 @@ +"""Calculate stored households through the simulation service, and read them. + +Submitting a calculation checks the country's ``countries`` row allows +household calculations (a household-only country does), reads the stored +household and policy through the same services as ``/v2/households`` and +``/v2/policies`` (so a household or policy of another country, or none, is +not found), selects the country's running catalog version, and sends the +simulation service everything it needs: it cannot read these tables. The +simulation service records every calculation, so one not saved to an +account is still stored, as a simulation run with no report. + +Reading a calculation asks the simulation service for it as it stands. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from policyengine_api.services.v2.household_calculations.transformations import ( + household_situation, + policy_parameters, + saved_spm_choice, +) +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationRead, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationNotFoundError, + calculation_year, + require_household_country, +) + +if TYPE_CHECKING: + from uuid import UUID + + from policyengine_api.services.v2.household_calculations.types import ( + CatalogSource, + CountrySource, + HouseholdCalculationInput, + HouseholdSource, + PolicySource, + SimulationHouseholdCalculations, + ) + + +class V2HouseholdCalculationService: + """Submit household calculations to the simulation service and read them.""" + + def __init__( + self, + *, + countries: CountrySource, + households: HouseholdSource, + policies: PolicySource, + catalogs: CatalogSource, + simulation: SimulationHouseholdCalculations, + ) -> None: + """Read through these services for one request.""" + self._countries = countries + self._households = households + self._policies = policies + self._catalogs = catalogs + self._simulation = simulation + + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationRead: + """Send the calculation to the simulation service and return it. + + Raise HouseholdCalculationCountryError, HouseholdNotFoundError, + PolicyNotFoundError or HouseholdCalculationYearError before anything + is sent, and the simulation errors when the service cannot calculate. + """ + country_id = calculation.country_id + require_household_country(self._countries.list_countries(), country_id) + household = self._households.get_household( + country_id=country_id, household_id=calculation.household_id + ) + parameters: dict[str, dict[str, Any]] = {} + if calculation.policy_id is not None: + parameters = policy_parameters( + self._policies.get_policy( + country_id=country_id, policy_id=calculation.policy_id + ) + ) + year = calculation_year(calculation.year, household.default_year) + try: + catalog = self._catalogs.get_model_by_country(country_id) + finally: + self._catalogs.close() + submission: dict[str, Any] = { + "country": country_id, + "catalog_version_id": str(catalog.model_version.id), + "policyengine_version": catalog.policyengine_version, + "year": year, + "household_id": str(household.id), + "household": household_situation(country_id, household.household_data), + "policy": { + "policy_id": ( + None + if calculation.policy_id is None + else str(calculation.policy_id) + ), + "parameters": parameters, + }, + "options": { + "axes": ( + None + if calculation.axes is None + else [ + [ + axis.model_dump(mode="json", exclude_unset=True) + for axis in group + ] + for group in calculation.axes + ] + ), + "add_missing_variables": calculation.add_missing_variables, + }, + "force": calculation.force, + "trigger": "initial", + } + spm = saved_spm_choice(household.household_data) + if spm is not None: + submission["spm"] = spm + return HouseholdCalculationRead.from_simulation( + self._simulation.submit(submission) + ) + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation, or raise HouseholdCalculationNotFoundError.""" + calculation = self._simulation.read(calculation_id) + if calculation is None: + msg = f"Household calculation {calculation_id} not found." + raise HouseholdCalculationNotFoundError(msg) + return HouseholdCalculationRead.from_simulation(calculation) diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py new file mode 100644 index 000000000..1474bd462 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -0,0 +1,171 @@ +"""The simulation service's household calculation routes, through its client. + +Calls go to the Cloud Run Simulation Entrypoint (``SIMULATION_ENTRYPOINT_URL``), +the only simulation service with these routes, with the gateway credentials +the Public API already uses (``GATEWAY_AUTH_*``) and the request's +correlation headers. A 400 is a refusal, whose reason the service gives; any +other failure, including a 503, no answer or missing configuration, means the +service is unavailable. + +One client serves the process, so its connections and its gateway token are +reused across requests. +""" + +from __future__ import annotations + +from functools import lru_cache +from http import HTTPStatus +from typing import TYPE_CHECKING, Any + +import httpx +from policyengine_api_simulation_client import AuthenticatedClient +from policyengine_api_simulation_client.api.default import ( + get_household_calculation, + submit_household_calculation, +) +from policyengine_api_simulation_client.models.household_calculation_submission import ( + HouseholdCalculationSubmission, +) +from policyengine_observability import ( # type: ignore[import-untyped] # the package has no type information + instrument_httpx, +) + +from policyengine_api.libs.gateway_auth import ( + GatewayAuthError, + GatewayAuthTokenProvider, + GatewayBearerAuth, + _require_all_or_none_gateway_auth_env, + gateway_auth_required, +) +from policyengine_api.libs.simulation_entrypoint import ( + _attach_current_request_id, + resolve_simulation_entrypoint_url, +) +from policyengine_api.observability import get_runtime +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) + +if TYPE_CHECKING: + from uuid import UUID + +REQUEST_TIMEOUT_SECONDS = 30.0 +CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" + + +def _answer(response: Any) -> tuple[int, object]: # noqa: ANN401 # the generated client's Response + try: + body = httpx.Response(response.status_code, content=response.content).json() + except ValueError: + body = None + return int(response.status_code), body + + +def _gateway_auth() -> GatewayBearerAuth | None: + """Return the gateway credentials, or None when none are set and none required.""" + _require_all_or_none_gateway_auth_env() + token_provider = GatewayAuthTokenProvider() + if token_provider.configured: + return GatewayBearerAuth(token_provider) + if gateway_auth_required(): + msg = "Gateway auth is required but GATEWAY_AUTH_* is not set." + raise GatewayAuthError(msg) + return None + + +def _refusal_reason(body: object) -> str: + reason = body.get("detail") if isinstance(body, dict) else None + if isinstance(reason, str): + return reason + return "The simulation service refused the household calculation." + + +class SimulationHouseholdClient: + """Submit and read household calculations on the simulation service.""" + + def __init__( + self, base_url: str, *, httpx_args: dict[str, Any] | None = None + ) -> None: + """Call the service at ``base_url`` with these httpx client arguments.""" + http_client = httpx.Client( + base_url=base_url, + timeout=httpx.Timeout(REQUEST_TIMEOUT_SECONDS), + event_hooks={"request": [_attach_current_request_id]}, + **(httpx_args or {}), + ) + instrument_httpx(http_client, get_runtime()) + self._base_url = base_url.rstrip("/") + # The generated routes take an AuthenticatedClient. Its fixed token is + # sent only from an httpx client it builds itself; the one set here + # authenticates through ``httpx_args`` (GatewayBearerAuth, which + # fetches and renews the gateway token), so the empty token is unused. + self._client = AuthenticatedClient( + base_url=base_url, token="" + ).set_httpx_client(http_client) + + @property + def base_url(self) -> str: + """Return the simulation service's base URL, without a trailing slash.""" + return self._base_url + + @classmethod + def from_environment(cls) -> SimulationHouseholdClient: + """Call the configured Cloud Run Simulation Entrypoint with gateway auth. + + Raise SimulationServiceUnavailableError when the URL is missing or + invalid, or gateway auth is partly configured or required but unset. + """ + try: + base_url = resolve_simulation_entrypoint_url( + CLOUD_RUN_SIMULATION_ENTRYPOINT + ) + auth = _gateway_auth() + except (ValueError, GatewayAuthError) as error: + msg = "The simulation service is not configured." + raise SimulationServiceUnavailableError(msg) from error + return cls(base_url, httpx_args={} if auth is None else {"auth": auth}) + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Submit a calculation and return the service's calculation answer.""" + try: + response = submit_household_calculation.sync_detailed( + client=self._client, + body=HouseholdCalculationSubmission.from_dict(submission), + ) + except httpx.HTTPError as error: + msg = "The simulation service could not be reached." + raise SimulationServiceUnavailableError(msg) from error + status, body = _answer(response) + if status == HTTPStatus.BAD_REQUEST: + raise SimulationServiceRefusedError(_refusal_reason(body)) + if status != HTTPStatus.ACCEPTED or not isinstance(body, dict): + msg = f"The simulation service answered {status}." + raise SimulationServiceUnavailableError(msg) + return body + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the service's calculation answer, or None when it has none.""" + try: + response = get_household_calculation.sync_detailed( + calculation_id, client=self._client + ) + except httpx.HTTPError as error: + msg = "The simulation service could not be reached." + raise SimulationServiceUnavailableError(msg) from error + status, body = _answer(response) + if status == HTTPStatus.NOT_FOUND: + return None + if status != HTTPStatus.OK or not isinstance(body, dict): + msg = f"The simulation service answered {status}." + raise SimulationServiceUnavailableError(msg) + return body + + +@lru_cache(maxsize=1) +def process_simulation_household_client() -> SimulationHouseholdClient: + """Return the process's client, built from the environment on first use. + + A configuration error is raised again on the next call, not kept. + """ + return SimulationHouseholdClient.from_environment() diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py new file mode 100644 index 000000000..3cc20ceba --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -0,0 +1,205 @@ +"""Framework-independent data exchanged by the household calculation layers.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol, Self +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +from policyengine_api.query_parameters import CountryId, DefaultYear + +if TYPE_CHECKING: + from collections.abc import Mapping + + from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.households.types import HouseholdRead + from policyengine_api.services.v2.metadata.types import ( + MetadataModelSelectionResult, + ) + from policyengine_api.services.v2.policies.types import PolicyRead + +# The simulation service's limits on axes. +MAXIMUM_AXIS_POINTS = 10_000 +MAXIMUM_AXIS_GROUPS = 8 +MAXIMUM_PARALLEL_AXES = 32 +CalculationStatus = Literal["pending", "dispatching", "running", "succeeded", "failed"] + + +class StrictCalculationModel(BaseModel): + """Base for household calculation data: no unknown fields, finite numbers.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class HouseholdCalculationAxis(StrictCalculationModel): + """One input varied across points. + + The variable ``name`` takes ``count`` evenly spaced values from ``min`` to + ``max`` in ``period`` (the household's default period when omitted), for + the entity member at position ``index`` (the first when omitted). + """ + + name: Annotated[str, Field(min_length=1, max_length=255)] + count: Annotated[int, Field(ge=1, le=MAXIMUM_AXIS_POINTS)] + min: int | float + max: int | float + period: Annotated[str, Field(min_length=1, max_length=64)] | int | None = None + index: Annotated[int, Field(ge=0)] | None = None + + +AxisGroup = Annotated[ + list[HouseholdCalculationAxis], + Field(min_length=1, max_length=MAXIMUM_PARALLEL_AXES), +] + + +class HouseholdCalculationInput(StrictCalculationModel): + """A stored household to calculate under a stored policy, or current law. + + ``year`` defaults to the household's default year. ``axes`` vary inputs + across points; ``add_missing_variables`` adds every yearly, monthly and + eternal variable the household lacks, as ``/calculate-full`` does. + ``force`` calculates again even when a recent calculation can be reused. + """ + + country_id: CountryId + household_id: UUID + policy_id: UUID | None = None + year: DefaultYear | None = None + axes: Annotated[list[AxisGroup], Field(max_length=MAXIMUM_AXIS_GROUPS)] | None = ( + None + ) + add_missing_variables: bool = False + force: bool = False + + +class HouseholdCalculationResult(StrictCalculationModel): + """A succeeded calculation's household and its notes. + + ``household`` is the stored household in the situation shape, its people + and groups named by their record IDs, with each requested value filled + in. ``spm_config`` is the SPM selection the calculation applied and + ``spm_provenance`` its receipt, both null for a country without SPM. + """ + + household: dict[str, JsonValue] + warnings: list[str] + spm_config: dict[str, JsonValue] | None + spm_provenance: dict[str, JsonValue] | None + policyengine_version: str | None + + +class HouseholdCalculationFailure(StrictCalculationModel): + """Why a calculation failed, with the simulation service's error detail.""" + + code: str + message: str + detail: dict[str, JsonValue] | None + + +class HouseholdCalculationRead(StrictCalculationModel): + """A household calculation as it stands: what it calculates and its outcome.""" + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + status: CalculationStatus + created_at: datetime + completed_at: datetime | None + result: HouseholdCalculationResult | None + error: HouseholdCalculationFailure | None + + @classmethod + def from_simulation(cls, calculation: Mapping[str, Any]) -> Self: + """Build the read from the simulation service's calculation answer.""" + result = calculation.get("result") + error = calculation.get("error") + bundle = result.get("bundle") if isinstance(result, dict) else None + return cls( + calculation_id=calculation["calculation_id"], + country_id=calculation["country"], + household_id=calculation["household_id"], + policy_id=calculation["policy_id"], + year=calculation["year"], + catalog_version_id=calculation["catalog_version_id"], + status=calculation["status"], + created_at=calculation["created_at"], + completed_at=calculation["completed_at"], + result=( + None + if result is None + else HouseholdCalculationResult( + household=result["household"], + warnings=result["warnings"], + spm_config=result["spm_config"], + spm_provenance=result["spm_provenance"], + policyengine_version=( + bundle.get("policyengine_version") + if isinstance(bundle, dict) + else None + ), + ) + ), + error=None if error is None else HouseholdCalculationFailure(**error), + ) + + +class SimulationServiceUnavailableError(RuntimeError): + """The simulation service could not be reached, or could not answer.""" + + +class SimulationServiceRefusedError(RuntimeError): + """The simulation service refused the calculation; the message says why.""" + + +class CountrySource(Protocol): + """The countries table (``V2CountryService``).""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and its capability.""" + ... + + +class HouseholdSource(Protocol): + """Stored households (``V2HouseholdService``).""" + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return one household of this country, or raise HouseholdNotFoundError.""" + ... + + +class PolicySource(Protocol): + """Stored policies (``V2PolicyService``).""" + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + """Return one policy of this country, or raise PolicyNotFoundError.""" + ... + + +class CatalogSource(Protocol): + """The country's catalog versions (a v2 metadata reader).""" + + def get_model_by_country(self, country_id: str) -> MetadataModelSelectionResult: + """Return the country's model and the running catalog version.""" + ... + + def close(self) -> None: + """Close the reader's database session.""" + ... + + +class SimulationHouseholdCalculations(Protocol): + """The simulation service's household calculation routes.""" + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Submit a calculation and return it as the service answers.""" + ... + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the calculation as the service answers, or None if unknown.""" + ... diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py new file mode 100644 index 000000000..507590131 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -0,0 +1,47 @@ +"""Checks of a household calculation against what is already loaded.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + from policyengine_api.services.v2.countries.types import CountryRead + + +class HouseholdCalculationCountryError(ValueError): + """The country does not offer household calculations.""" + + +class HouseholdCalculationYearError(ValueError): + """The request names no year and the household has no default year.""" + + +class HouseholdCalculationNotFoundError(LookupError): + """The simulation service has no such household calculation.""" + + +def require_household_country( + countries: Iterable[CountryRead], country_id: str +) -> None: + """Raise HouseholdCalculationCountryError unless the country calculates households. + + A household-only country (household calculations without society-wide + reports) passes. + """ + for country in countries: + if country.id == country_id and country.household_enabled: + return + msg = f"Household calculations are not available for country {country_id}." + raise HouseholdCalculationCountryError(msg) + + +def calculation_year(requested_year: int | None, default_year: int | None) -> int: + """Return the requested year, or else the household's default year.""" + if requested_year is not None: + return requested_year + if default_year is None: + msg = "This household has no default year; send the year to calculate." + raise HouseholdCalculationYearError(msg) + return default_year diff --git a/pyproject.toml b/pyproject.toml index 4d225b491..234fc0e6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dependencies = [ "psycopg[binary]>=3.3,<4", "pyjwt[crypto]>=2.10,<3", "typing-extensions>=4.4,<5", + "policyengine-api-simulation-client>=0.20261027.288", ] [project.optional-dependencies] @@ -249,6 +250,7 @@ runtime-evaluated-base-classes = [ "policyengine_api.data.v2.models.base.TimestampedModel", "policyengine_api.data.v2.models.comparison_runs.Stage12RunTimestamps", "policyengine_api.data.v2.models.reports.GeographicImpactBase", + "policyengine_api.fastapi_routes.v2.household_calculations.response_models.StrictHouseholdCalculationAPIModel", "policyengine_api.fastapi_routes.v2.households.document_models.HouseholdEntityRecord", "policyengine_api.fastapi_routes.v2.households.document_models.StrictHouseholdDocumentModel", "policyengine_api.fastapi_routes.v2.households.response_models.HouseholdSuccessResponse", @@ -273,6 +275,8 @@ runtime-evaluated-base-classes = [ "policyengine_api.query_parameters.PaginationQuery", "policyengine_api.query_parameters.StrictQueryParameters", "policyengine_api.services.v2.associations.types.StrictAssociationModel", + "policyengine_api.services.v2.household_calculations.types.HouseholdCalculationInput", + "policyengine_api.services.v2.household_calculations.types.StrictCalculationModel", "policyengine_api.services.v2.households.types.HouseholdCreationInput", "policyengine_api.services.v2.households.types.StrictHouseholdInput", "policyengine_api.services.v2.metadata.types.MetadataModelVersion", diff --git a/tests/unit/v2/household_calculation_support.py b/tests/unit/v2/household_calculation_support.py new file mode 100644 index 000000000..b28ab2ba0 --- /dev/null +++ b/tests/unit/v2/household_calculation_support.py @@ -0,0 +1,248 @@ +"""Fakes and records shared by the household calculation tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any +from uuid import UUID + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.households.types import HouseholdRead +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.types import ( + PolicyParameterValueRead, + PolicyRead, +) +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError + +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000a1") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000b1") +CALCULATION_ID = UUID("00000000-0000-4000-8000-0000000000c1") +SIMULATION_ID = UUID("00000000-0000-4000-8000-0000000000d1") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000e1") +POLICYENGINE_VERSION = "6.10.0" +CREATED_AT = datetime(2026, 10, 27, 12, 0, tzinfo=UTC) +SPM_CHOICE = {"scenario": "ce_trend", "geography_kind": "national"} +HOUSEHOLD_DATA = { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {"state_name": {"2026": "CA"}}}], + "family": [{"id": "family-1", "values": {}}], + "tax_unit": [{"id": "tax_unit-1", "values": {}}], + "spm_unit": [{"id": "spm_unit-1", "values": {}}], + "marital_unit": [{"id": "marital_unit-1", "values": {}}], + "spm": SPM_CHOICE, +} +SITUATION = { + "people": {"person-1": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}}, + "households": { + "household-1": {"members": ["person-1"], "state_name": {"2026": "CA"}} + }, + "families": {"family-1": {"members": ["person-1"]}}, + "tax_units": {"tax_unit-1": {"members": ["person-1"]}}, + "spm_units": {"spm_unit-1": {"members": ["person-1"]}}, + "marital_units": {"marital_unit-1": {"members": ["person-1"]}}, +} +SPM_CONFIG = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +SPM_PROVENANCE = {"schema_version": "canonical-spm-provenance-v2", "years": ["2026"]} +BUNDLE = { + "policyengine_version": POLICYENGINE_VERSION, + "core_package_version": "3.33.0", + "country_package_name": "policyengine-us", + "country_package_version": "1.450.0", + "bundle_manifest_sha256": "a" * 64, +} + + +def simulation_calculation(**changes: object) -> dict[str, Any]: + """Return the simulation service's answer for a running calculation.""" + answer: dict[str, Any] = { + "calculation_id": str(CALCULATION_ID), + "simulation_id": str(SIMULATION_ID), + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "options": {}, + "catalog_version_id": str(CATALOG_VERSION_ID), + "year": 2026, + "attempt": 1, + "trigger": "initial", + "status": "running", + "created_at": CREATED_AT.isoformat(), + "dispatched_at": CREATED_AT.isoformat(), + "completed_at": None, + "result": None, + "error": None, + } + answer.update(changes) + return answer + + +def succeeded_calculation() -> dict[str, Any]: + """Return the simulation service's answer once the calculation succeeded.""" + return simulation_calculation( + status="succeeded", + completed_at=CREATED_AT.isoformat(), + result={ + "household": { + **SITUATION, + "households": { + "household-1": { + "members": ["person-1"], + "state_name": {"2026": "CA"}, + "household_net_income": {"2026": 27000.5}, + } + }, + }, + "warnings": ["The input `age_group` is deprecated and was ignored."], + "spm_config": SPM_CONFIG, + "spm_provenance": SPM_PROVENANCE, + "bundle": BUNDLE, + }, + ) + + +def household_read(*, default_year: int | None = 2026) -> HouseholdRead: + """Return the stored US household.""" + return HouseholdRead( + id=HOUSEHOLD_ID, + country_id="us", + default_year=default_year, + household_data=HOUSEHOLD_DATA, + created_at=CREATED_AT, + updated_at=CREATED_AT, + ) + + +def policy_read() -> PolicyRead: + """Return the stored US reform with one parameter value.""" + return PolicyRead( + id=POLICY_ID, + country_id="us", + tax_benefit_model_id=CATALOG_VERSION_ID, + tax_benefit_model_version_id=CATALOG_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT, + parameter_values=( + PolicyParameterValueRead( + id=POLICY_ID, + parameter_id=POLICY_ID, + parameter_name="gov.irs.credits.ctc.amount.base[0].amount", + value=3000, + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ), + ) + + +@dataclass +class FakeCountries: + """The countries table: the US with every calculation, the UK households only.""" + + rows: tuple[CountryRead, ...] = ( + CountryRead(id="uk", household_enabled=True, society_wide_enabled=False), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ) + + def list_countries(self) -> list[CountryRead]: + """Return every row.""" + return list(self.rows) + + +class FakeHouseholds: + """Stored households by country and ID.""" + + def __init__(self) -> None: + """Hold the stored US household.""" + self.households = {("us", HOUSEHOLD_ID): household_read()} + self.calls: list[tuple[str, UUID]] = [] + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return the household or raise HouseholdNotFoundError.""" + self.calls.append((country_id, household_id)) + household = self.households.get((country_id, household_id)) + if household is None: + msg = f"Household {household_id} not found." + raise HouseholdNotFoundError(msg) + return household + + +class FakePolicies: + """Stored policies by country and ID.""" + + def __init__(self) -> None: + """Hold the stored US reform.""" + self.policies = {("us", POLICY_ID): policy_read()} + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + """Return the policy or raise PolicyNotFoundError.""" + policy = self.policies.get((country_id, policy_id)) + if policy is None: + msg = f"Policy {policy_id} not found." + raise PolicyNotFoundError(msg) + return policy + + +class FakeCatalogs: + """Each country's current catalog version, and whether it was closed.""" + + def __init__(self) -> None: + """Start open.""" + self.closed = False + + def get_model_by_country(self, country_id: str) -> SimpleNamespace: + """Return the running catalog version's ID and PolicyEngine.py version.""" + return SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + model_version=SimpleNamespace(id=CATALOG_VERSION_ID), + country_id=country_id, + ) + + def close(self) -> None: + """Record the close.""" + self.closed = True + + +class FakeSimulation: + """The simulation service's household routes, as scripted answers.""" + + def __init__(self) -> None: + """Answer submissions with a running calculation.""" + self.submissions: list[dict[str, Any]] = [] + self.answer: dict[str, Any] = simulation_calculation() + self.read_answers: dict[UUID, dict[str, Any]] = {} + self.error: Exception | None = None + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Record the submission and answer it.""" + self.submissions.append(submission) + if self.error is not None: + raise self.error + return self.answer + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the scripted calculation, or None.""" + if self.error is not None: + raise self.error + return self.read_answers.get(calculation_id) diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py new file mode 100644 index 000000000..a6f396bd7 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -0,0 +1,224 @@ +"""The /v2/household-calculations routes: submit a stored household, then poll it.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from flask import Flask + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + HOUSEHOLD_ID, + POLICY_ID, + SPM_CONFIG, + SPM_PROVENANCE, + FakeCatalogs, + FakeCountries, + FakeHouseholds, + FakePolicies, + FakeSimulation, + succeeded_calculation, +) + +PATH = "/v2/household-calculations" + + +class App: + """A client of the native app serving household calculations over fakes.""" + + def __init__(self, countries: FakeCountries | None = None) -> None: + """Build the fakes and the app.""" + self.simulation = FakeSimulation() + self.households = FakeHouseholds() + self.service = V2HouseholdCalculationService( + countries=countries or FakeCountries(), + households=self.households, + policies=FakePolicies(), + catalogs=FakeCatalogs(), + simulation=self.simulation, + ) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_household_calculation_service_factory=lambda: self.service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + self.client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ), + raise_server_exceptions=False, + ) + + +def body(**changes: object) -> dict[str, object]: + """Return a request calculating the stored household under the reform.""" + request: dict[str, object] = { + "country_id": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + } + request.update(changes) + return request + + +def test_a_calculation_is_accepted_without_an_account() -> None: + app = App() + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 202 + payload = response.json() + assert payload["status"] == "ok" + assert payload["result"]["calculation_id"] == str(CALCULATION_ID) + assert payload["result"]["status"] == "running" + assert payload["result"]["household_id"] == str(HOUSEHOLD_ID) + assert payload["result"]["result"] is None + + +def test_a_calculation_with_axes_sends_them() -> None: + app = App() + axes = [[{"name": "employment_income", "count": 3, "min": 0, "max": 50000}]] + + response = app.client.post(PATH, json=body(axes=axes)) + + assert response.status_code == 202 + assert app.simulation.submissions[0]["options"]["axes"] == axes + + +def test_a_succeeded_us_calculation_reads_back_its_spm_choice() -> None: + app = App() + app.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + response = app.client.get(f"{PATH}/{CALCULATION_ID}") + + assert response.status_code == 200 + result = response.json()["result"] + assert result["status"] == "succeeded" + assert result["result"]["spm_config"] == SPM_CONFIG + assert result["result"]["spm_provenance"] == SPM_PROVENANCE + assert result["result"]["warnings"] == [ + "The input `age_group` is deprecated and was ignored." + ] + + +def test_a_country_without_household_calculations_gets_422_naming_it() -> None: + app = App( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 422 + assert "us" in response.json()["message"] + assert app.simulation.submissions == [] + + +def test_a_household_only_country_is_accepted() -> None: + app = App() + stored = app.households.households[("us", HOUSEHOLD_ID)] + app.households.households[("uk", HOUSEHOLD_ID)] = stored.__class__( + **{ + **stored.__dict__, + "country_id": "uk", + "household_data": { + "people": [ + { + "id": "person-1", + "values": {}, + "memberships": { + "household": "household-1", + "benunit": "benunit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {}}], + "benunit": [{"id": "benunit-1", "values": {}}], + }, + } + ) + + response = app.client.post(PATH, json=body(country_id="uk", policy_id=None)) + + assert response.status_code == 202 + + +@pytest.mark.parametrize( + "changes", + [{"household_id": str(uuid4())}, {"policy_id": str(uuid4())}], + ids=["household", "policy"], +) +def test_an_unknown_household_or_policy_is_not_found(changes) -> None: + response = App().client.post(PATH, json=body(**changes)) + + assert response.status_code == 404 + + +def test_an_unknown_calculation_is_not_found() -> None: + assert App().client.get(f"{PATH}/{uuid4()}").status_code == 404 + + +@pytest.mark.parametrize( + ("error", "status_code"), + [ + (SimulationServiceUnavailableError("unreachable"), 503), + (SimulationServiceRefusedError("no household function"), 503), + ], + ids=["unreachable", "refused"], +) +def test_a_simulation_service_that_cannot_calculate_answers_503( + error, status_code +) -> None: + app = App() + app.simulation.error = error + + response = app.client.post(PATH, json=body()) + + assert response.status_code == status_code + + +@pytest.mark.parametrize( + "changes", + [ + {"axes": [[{"name": "employment_income", "count": 0, "min": 0, "max": 1}]]}, + {"country_id": "ca"}, + {"unexpected": True}, + {"year": 1800}, + ], + ids=["axis-count", "country", "unknown-field", "year"], +) +def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None: + app = App() + + response = app.client.post(PATH, json=body(**changes)) + + assert response.status_code == 422 + assert app.simulation.submissions == [] diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py new file mode 100644 index 000000000..b2b16bbc1 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_service.py @@ -0,0 +1,246 @@ +"""Sending a stored household's calculation to the simulation service; reading it.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + CATALOG_VERSION_ID, + HOUSEHOLD_ID, + POLICY_ID, + POLICYENGINE_VERSION, + SITUATION, + SPM_CHOICE, + SPM_CONFIG, + SPM_PROVENANCE, + FakeCatalogs, + FakeCountries, + FakeHouseholds, + FakePolicies, + FakeSimulation, + household_read, + simulation_calculation, + succeeded_calculation, +) + +AXES = [[{"name": "employment_income", "count": 5, "min": 0, "max": 80000}]] + + +class Parts: + """The service's collaborators, kept for the assertions.""" + + def __init__(self, countries: FakeCountries | None = None) -> None: + """Build every fake.""" + self.countries = countries or FakeCountries() + self.households = FakeHouseholds() + self.policies = FakePolicies() + self.catalogs = FakeCatalogs() + self.simulation = FakeSimulation() + self.service = V2HouseholdCalculationService( + countries=self.countries, + households=self.households, + policies=self.policies, + catalogs=self.catalogs, + simulation=self.simulation, + ) + + +def calculation_input(**changes: object) -> HouseholdCalculationInput: + """Return a calculation of the stored US household under the stored reform.""" + values: dict[str, object] = { + "country_id": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": POLICY_ID, + } + values.update(changes) + return HouseholdCalculationInput.model_validate(values) + + +def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: + parts = Parts() + + calculation = parts.service.submit_calculation( + calculation_input(axes=AXES, add_missing_variables=True, force=True) + ) + + (submission,) = parts.simulation.submissions + assert submission == { + "country": "us", + "catalog_version_id": str(CATALOG_VERSION_ID), + "policyengine_version": POLICYENGINE_VERSION, + "year": 2026, + "household_id": str(HOUSEHOLD_ID), + "household": SITUATION, + "spm": SPM_CHOICE, + "policy": { + "policy_id": str(POLICY_ID), + "parameters": { + "gov.irs.credits.ctc.amount.base[0].amount": { + "2026-01-01.2100-12-31": 3000 + } + }, + }, + "options": {"axes": AXES, "add_missing_variables": True}, + "force": True, + "trigger": "initial", + } + assert calculation.calculation_id == CALCULATION_ID + assert calculation.status == "running" + assert calculation.result is None + assert parts.catalogs.closed + + +def test_current_law_and_no_saved_choice_send_neither() -> None: + parts = Parts() + stored = household_read() + data = {key: value for key, value in stored.household_data.items() if key != "spm"} + parts.households.households[("us", HOUSEHOLD_ID)] = stored.__class__( + **{**stored.__dict__, "household_data": data} + ) + + parts.service.submit_calculation(calculation_input(policy_id=None)) + + (submission,) = parts.simulation.submissions + assert "spm" not in submission + assert submission["policy"] == {"policy_id": None, "parameters": {}} + assert submission["options"] == {"axes": None, "add_missing_variables": False} + + +def test_the_year_is_the_requested_one_or_the_households_default() -> None: + parts = Parts() + + parts.service.submit_calculation(calculation_input(year=2027)) + parts.households.households[("us", HOUSEHOLD_ID)] = household_read( + default_year=None + ) + + with pytest.raises(HouseholdCalculationYearError): + parts.service.submit_calculation(calculation_input()) + assert [submission["year"] for submission in parts.simulation.submissions] == [2027] + + +def test_a_country_without_household_calculations_is_refused_by_name() -> None: + parts = Parts( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + + with pytest.raises(HouseholdCalculationCountryError, match="us"): + parts.service.submit_calculation(calculation_input()) + + assert parts.households.calls == [] + assert parts.simulation.submissions == [] + + +def test_a_household_only_country_is_accepted() -> None: + parts = Parts() + uk_household = household_read() + parts.households.households[("uk", HOUSEHOLD_ID)] = uk_household.__class__( + **{ + **uk_household.__dict__, + "country_id": "uk", + "household_data": { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 30}}, + "memberships": { + "household": "household-1", + "benunit": "benunit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {}}], + "benunit": [{"id": "benunit-1", "values": {}}], + }, + } + ) + + parts.service.submit_calculation(calculation_input(country_id="uk", policy_id=None)) + + (submission,) = parts.simulation.submissions + assert submission["country"] == "uk" + assert set(submission["household"]) == {"people", "households", "benunits"} + + +def test_an_unknown_household_or_policy_sends_nothing() -> None: + parts = Parts() + + with pytest.raises(HouseholdNotFoundError): + parts.service.submit_calculation(calculation_input(household_id=uuid4())) + with pytest.raises(PolicyNotFoundError): + parts.service.submit_calculation(calculation_input(policy_id=uuid4())) + + assert parts.simulation.submissions == [] + + +def test_a_us_result_includes_the_spm_choice_applied_and_its_receipt() -> None: + parts = Parts() + parts.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + calculation = parts.service.get_calculation(CALCULATION_ID) + + assert calculation.status == "succeeded" + assert calculation.result is not None + assert calculation.result.spm_config == SPM_CONFIG + assert calculation.result.spm_provenance == SPM_PROVENANCE + assert calculation.result.policyengine_version == POLICYENGINE_VERSION + assert calculation.result.household["households"]["household-1"][ + "household_net_income" + ] == {"2026": 27000.5} + + +def test_a_failed_calculation_reads_back_its_error() -> None: + parts = Parts() + detail = {"code": "invalid_inputs", "message": "Unrecognized inputs: incme"} + parts.simulation.read_answers[CALCULATION_ID] = simulation_calculation( + status="failed", + error={ + "code": "household_calculation_failed", + "message": "The household calculation failed: invalid_inputs.", + "detail": detail, + }, + ) + + calculation = parts.service.get_calculation(CALCULATION_ID) + + assert calculation.status == "failed" + assert calculation.error is not None + assert calculation.error.code == "household_calculation_failed" + assert calculation.error.detail == detail + + +def test_an_unknown_calculation_is_not_found() -> None: + with pytest.raises(HouseholdCalculationNotFoundError): + Parts().service.get_calculation(uuid4()) + + +def test_an_unreachable_simulation_service_is_reported() -> None: + parts = Parts() + parts.simulation.error = SimulationServiceUnavailableError("unreachable") + + with pytest.raises(SimulationServiceUnavailableError): + parts.service.submit_calculation(calculation_input()) diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py new file mode 100644 index 000000000..2f73b5a82 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -0,0 +1,166 @@ +"""The simulation service's household calculation routes, through its client. + +These tests need a simulation client with the household calculation routes. +""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import httpx +import pytest + +from policyengine_api.services.v2.household_calculations.simulation_client import ( + SimulationHouseholdClient, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + CATALOG_VERSION_ID, + HOUSEHOLD_ID, + POLICYENGINE_VERSION, + SITUATION, + simulation_calculation, + succeeded_calculation, +) + +BASE_URL = "https://simulation.example" +SUBMISSION = { + "country": "us", + "catalog_version_id": str(CATALOG_VERSION_ID), + "policyengine_version": POLICYENGINE_VERSION, + "year": 2026, + "household_id": str(HOUSEHOLD_ID), + "household": SITUATION, + "spm": {"scenario": "ce_trend"}, + "policy": {"policy_id": None, "parameters": {}}, + "options": {"axes": None, "add_missing_variables": False}, + "force": False, + "trigger": "initial", +} + + +def client_answering( + status_code: int, payload: object, requests: list +) -> SimulationHouseholdClient: + """Return a client whose requests get one fixed answer.""" + + def answer(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json=payload) + + return SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(answer)} + ) + + +def test_a_submission_is_posted_and_its_calculation_returned() -> None: + requests: list[httpx.Request] = [] + client = client_answering(202, simulation_calculation(), requests) + + calculation = client.submit(SUBMISSION) + + assert calculation == simulation_calculation() + (request,) = requests + assert request.method == "POST" + assert str(request.url) == f"{BASE_URL}/v2/household-calculations" + assert json.loads(request.content) == SUBMISSION + + +def test_a_calculation_is_read_by_id() -> None: + requests: list[httpx.Request] = [] + client = client_answering(200, succeeded_calculation(), requests) + + assert client.read(CALCULATION_ID) == succeeded_calculation() + assert str(requests[0].url) == ( + f"{BASE_URL}/v2/household-calculations/{CALCULATION_ID}" + ) + + +def test_an_unknown_calculation_reads_as_none() -> None: + client = client_answering(404, {"detail": "No such household calculation."}, []) + + assert client.read(uuid4()) is None + + +def test_a_refusal_carries_the_services_reason() -> None: + client = client_answering( + 400, {"detail": "PolicyEngine.py 9.9.9 has no deployed v2 worker"}, [] + ) + + with pytest.raises(SimulationServiceRefusedError, match=r"9\.9\.9"): + client.submit(SUBMISSION) + + +@pytest.mark.parametrize("status_code", [500, 503]) +def test_a_service_that_cannot_answer_is_unavailable(status_code: int) -> None: + client = client_answering(status_code, {"detail": "unavailable"}, []) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +def test_an_unreachable_service_is_unavailable() -> None: + def refuse(request: httpx.Request) -> httpx.Response: + message = "refused" + raise httpx.ConnectError(message, request=request) + + client = SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(refuse)} + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +GATEWAY_AUTH_VARIABLES = ( + "GATEWAY_AUTH_ISSUER", + "GATEWAY_AUTH_AUDIENCE", + "GATEWAY_AUTH_CLIENT_ID", + "GATEWAY_AUTH_CLIENT_SECRET", + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE", + "GATEWAY_AUTH_REQUIRED", +) + + +@pytest.mark.parametrize( + "environment", + [ + {}, + {"SIMULATION_ENTRYPOINT_URL": "not a url"}, + { + "SIMULATION_ENTRYPOINT_URL": BASE_URL, + "GATEWAY_AUTH_ISSUER": "https://issuer.example/", + }, + {"SIMULATION_ENTRYPOINT_URL": BASE_URL, "GATEWAY_AUTH_REQUIRED": "true"}, + ], + ids=["no-url", "invalid-url", "partial-auth", "auth-required"], +) +def test_a_misconfigured_service_is_unavailable( + monkeypatch: pytest.MonkeyPatch, environment: dict[str, str] +) -> None: + for name in ("SIMULATION_ENTRYPOINT_URL", *GATEWAY_AUTH_VARIABLES): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(SimulationServiceUnavailableError, match="not configured"): + SimulationHouseholdClient.from_environment() + + +def test_the_configured_service_is_called_without_auth_when_none_is_set( + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name in GATEWAY_AUTH_VARIABLES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("SIMULATION_ENTRYPOINT_URL", f"{BASE_URL}/") + + client = SimulationHouseholdClient.from_environment() + + assert client.base_url == BASE_URL diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index a1e4708c1..701e2ff2d 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -642,6 +642,8 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/countries", "/v2/households", "/v2/households/{household_id}", + "/v2/household-calculations", + "/v2/household-calculations/{calculation_id}", "/v2/policies", "/v2/policies/{policy_id}", "/v2/user-households", diff --git a/uv.lock b/uv.lock index 14d4b7346..f9bdfaf54 100644 --- a/uv.lock +++ b/uv.lock @@ -2794,6 +2794,7 @@ dependencies = [ { name = "openai" }, { name = "packaging" }, { name = "policyengine", extra = ["models"] }, + { name = "policyengine-api-simulation-client" }, { name = "policyengine-canada" }, { name = "policyengine-il" }, { name = "policyengine-ng" }, @@ -2851,6 +2852,7 @@ requires-dist = [ { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, { name = "policyengine", extras = ["models"], specifier = "==6.10.0" }, + { name = "policyengine-api-simulation-client", specifier = ">=0.20261027.288" }, { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, @@ -2876,6 +2878,19 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "policyengine-api-simulation-client" +version = "0.20261027.288" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/21/145e238dec20e79a1ce7fb4818271e08603fab88656ce4d7e5c7307ab90e/policyengine_api_simulation_client-0.20261027.288.tar.gz", hash = "sha256:5129724a3e5ba4b0dc257d421c32dd1be7f5c4a1ae3c1a9b60b20d7fab58f9f5", size = 44777, upload-time = "2026-10-27T17:02:39.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/27/85e9da0f0561d0548cbe23ec46e4bf63a4b814d7adbe1f0932c514b52191/policyengine_api_simulation_client-0.20261027.288-py3-none-any.whl", hash = "sha256:82b4aacb954de0b8c747b313c1b92ea1d6a028402618f3d819c8298b87977486", size = 99950, upload-time = "2026-10-27T17:02:37.398Z" }, +] + [[package]] name = "policyengine-canada" version = "0.96.3" From f6d1d4e8703dd4a1e28658365ae253c771330f5c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 21:51:08 +0400 Subject: [PATCH 094/118] Check household calculations and their parity with /calculate in staging The parallel live staging job now runs a submit, poll and result cycle for a new US household; six calculations submitted at once against submit and finish time limits; a household across 401 axis points, which must finish; and a parity set of two US and two UK households (one of each with axes) compared value by value with API v1's /calculate on the same candidate. The job's time limit rises from 20 to 30 minutes for the Modal calculations. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/run_live_staging_tests.sh | 2 + .github/workflows/push.yml | 2 +- docs/migration/stage-11-v2-households.md | 25 ++ .../live_household_calculations.py | 222 ++++++++++++++++++ .../test_live_v2_household_calculations.py | 169 +++++++++++++ .../test_live_v2_household_parity.py | 154 ++++++++++++ tests/unit/test_cloud_run_deploy_scripts.py | 6 + 7 files changed, 579 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/integration/live_household_calculations.py create mode 100644 tests/integration/test_live_v2_household_calculations.py create mode 100644 tests/integration/test_live_v2_household_parity.py diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 39bc7a3cb..d7b5f6731 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -8,4 +8,6 @@ python -m pytest \ tests/integration/test_live_budget_window_cache.py \ tests/integration/test_live_calculate.py \ tests/integration/test_live_economy.py \ + tests/integration/test_live_v2_household_calculations.py \ + tests/integration/test_live_v2_household_parity.py \ -v diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e8007f472..d18fa4f53 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -342,7 +342,7 @@ jobs: parallel-live-simulation-tests-staging-cloud-run: name: Run live simulations against the staging candidate runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 needs: deploy-cloud-run-staging if: | (github.repository == 'PolicyEngine/policyengine-api') diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 2a58f7dd1..46c8730d3 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -292,3 +292,28 @@ Without the URL, or with gateway auth partly configured, both routes answer 503. They also answer 503 when the simulation service cannot be reached, answers with an error, or refuses the calculation (for example when no v2 worker is deployed for the catalog version's PolicyEngine.py version). + +### Staging checks + +The parallel live staging job (`run_live_staging_tests.sh`) runs two modules +against each tagged staging candidate, through staging API v1, the staging +Simulation Entrypoint, the v2 household functions on Modal and the staging v2 +database: + +- `test_live_v2_household_calculations.py`: a submit, poll and result cycle + for a new US household; six calculations submitted at once, each accepted + within 10 seconds and finished within 300 seconds; and one household varied + across 401 points of employment income, which must finish. The measured + times appear in the job log as `HouseholdCalculationMeasurement` warnings. +- `test_live_v2_household_parity.py`: two US and two UK households, one of + each with axes, calculated through `/v2/household-calculations` and through + API v1's `/calculate`; every calculated value and warning must match. + +`STAGING_HOUSEHOLD_PARALLEL_COUNT`, `STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, +`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS` and +`STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS` (600 seconds per calculation) +change those limits; none is set in the workflow. The checks see neither the +number of Simulation Entrypoint instances that served the calculations nor +the household functions' memory use: a calculation that runs out of memory +fails, and so fails the check. Both modules fail until the staging Simulation +Entrypoint serves `/v2/household-calculations`. diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py new file mode 100644 index 000000000..6fbf92647 --- /dev/null +++ b/tests/fixtures/integration/live_household_calculations.py @@ -0,0 +1,222 @@ +"""Households and helpers for the live staging household calculation checks. + +They need only httpx and the standard library, because the parallel live test +job installs nothing else. +""" + +from __future__ import annotations + +import math +import os +import time +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + import httpx + +CALCULATION_PATH = "/v2/household-calculations" +CALCULATION_TIMEOUT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS", "600") +) +POLL_INTERVAL_SECONDS = float( + os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") +) +TRANSIENT_STATUS_CODES = {500, 502, 503, 504} +FINISHED_STATUSES = {"succeeded", "failed"} +YEAR = "2026" +COLLECTIONS = { + "us": { + "household": "households", + "family": "families", + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "marital_unit": "marital_units", + }, + "uk": {"household": "households", "benunit": "benunits"}, +} + + +@dataclass(frozen=True) +class LiveHousehold: + """A household whose people all share one group of each kind. + + ``people`` and ``groups`` map variable names to values by period; a null + value asks for the variable to be calculated. ``axes`` vary inputs across + points, as in a ``/calculate`` situation. + """ + + name: str + country_id: str + people: dict[str, dict[str, Any]] + groups: dict[str, dict[str, Any]] = field(default_factory=dict) + axes: list[list[dict[str, Any]]] | None = None + + def document(self) -> dict[str, Any]: + """Return the household as ``/v2/households`` stores it.""" + collections = COLLECTIONS[self.country_id] + document: dict[str, Any] = { + "people": [ + { + "id": person_id, + "values": values, + "memberships": { + collection: f"{collection}-1" for collection in collections + }, + } + for person_id, values in self.people.items() + ] + } + for collection in collections: + document[collection] = [ + {"id": f"{collection}-1", "values": self.groups.get(collection, {})} + ] + return document + + def situation(self) -> dict[str, Any]: + """Return the same household as a ``/calculate`` situation.""" + situation: dict[str, Any] = {"people": self.people} + for collection, plural in COLLECTIONS[self.country_id].items(): + situation[plural] = { + f"{collection}-1": { + "members": list(self.people), + **self.groups.get(collection, {}), + } + } + if self.axes is not None: + situation["axes"] = self.axes + return situation + + +@dataclass(frozen=True) +class FinishedCalculation: + """A calculation's final answer and how long each step took.""" + + item: dict[str, Any] + submit_seconds: float + total_seconds: float + + +def store_household(client: httpx.Client, household: LiveHousehold) -> str: + """Store the household through ``/v2/households`` and return its ID.""" + response = client.post( + "/v2/households", + params={"country_id": household.country_id}, + json={ + "country_id": household.country_id, + "default_year": int(YEAR), + "household_data": household.document(), + }, + ) + assert response.status_code in {200, 201}, response.text + return response.json()["result"]["item"]["id"] + + +def calculate( + client: httpx.Client, + household: LiveHousehold, + household_id: str, + *, + force: bool = False, +) -> FinishedCalculation: + """Submit the stored household's calculation and poll until it finishes.""" + started = time.monotonic() + body: dict[str, Any] = { + "country_id": household.country_id, + "household_id": household_id, + "year": int(YEAR), + "force": force, + } + if household.axes is not None: + body["axes"] = household.axes + response = client.post(CALCULATION_PATH, json=body) + submit_seconds = time.monotonic() - started + assert response.status_code == 202, response.text + item = response.json()["result"] + deadline = started + CALCULATION_TIMEOUT_SECONDS + while item["status"] not in FINISHED_STATUSES: + if time.monotonic() >= deadline: + msg = ( + f"Calculation {item['calculation_id']} did not finish within " + f"{CALCULATION_TIMEOUT_SECONDS:.0f} s; last status {item['status']}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) + polled = client.get(f"{CALCULATION_PATH}/{item['calculation_id']}") + if polled.status_code in TRANSIENT_STATUS_CODES: + continue + assert polled.status_code == 200, polled.text + item = polled.json()["result"] + return FinishedCalculation( + item=item, + submit_seconds=submit_seconds, + total_seconds=time.monotonic() - started, + ) + + +def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: + """Return API v1's ``/calculate`` answer for the same household.""" + response = client.post( + f"/{household.country_id}/calculate", + json={"household": household.situation()}, + ) + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ok", payload + return payload + + +def _same_value(v1_value: object, v2_value: object) -> bool: + if isinstance(v1_value, list) and isinstance(v2_value, list): + return len(v1_value) == len(v2_value) and all( + _same_value(v1_item, v2_item) + for v1_item, v2_item in zip(v1_value, v2_value, strict=True) + ) + # API v1 writes an undefined number as a bare NaN; v2 writes "NaN". + if isinstance(v1_value, float) and math.isnan(v1_value): + return v2_value == "NaN" + if isinstance(v1_value, bool) or isinstance(v2_value, bool): + return v1_value == v2_value + if isinstance(v1_value, int | float) and isinstance(v2_value, int | float): + return math.isclose(v1_value, v2_value, rel_tol=1e-9, abs_tol=1e-6) + return v1_value == v2_value + + +def household_differences( + v1_household: dict[str, Any], v2_household: dict[str, Any] +) -> list[str]: + """Return every value that differs between the two calculated households. + + Each entity, variable and period in either household is compared, so a + value only one of them has is a difference too. + """ + differences: list[str] = [] + for plural in sorted(set(v1_household) | set(v2_household)): + v1_entities = v1_household.get(plural) + v2_entities = v2_household.get(plural) + if not isinstance(v1_entities, dict) or not isinstance(v2_entities, dict): + if v1_entities != v2_entities: + differences.append(f"{plural}: {v1_entities!r} != {v2_entities!r}") + continue + for entity_id in sorted(set(v1_entities) | set(v2_entities)): + v1_variables = v1_entities.get(entity_id) or {} + v2_variables = v2_entities.get(entity_id) or {} + for variable in sorted(set(v1_variables) | set(v2_variables)): + v1_value = v1_variables.get(variable) + v2_value = v2_variables.get(variable) + if variable == "members" or not ( + isinstance(v1_value, dict) and isinstance(v2_value, dict) + ): + if not _same_value(v1_value, v2_value): + differences.append( + f"{plural}.{entity_id}.{variable}: " + f"{v1_value!r} != {v2_value!r}" + ) + continue + differences.extend( + f"{plural}.{entity_id}.{variable}.{period}: " + f"{v1_value.get(period)!r} != {v2_value.get(period)!r}" + for period in sorted(set(v1_value) | set(v2_value)) + if not _same_value(v1_value.get(period), v2_value.get(period)) + ) + return differences diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py new file mode 100644 index 000000000..26a63fc49 --- /dev/null +++ b/tests/integration/test_live_v2_household_calculations.py @@ -0,0 +1,169 @@ +"""Live staging checks of ``/v2/household-calculations``. + +These run against the tagged staging candidate, through staging API v1, the +staging Simulation Entrypoint, the v2 household functions on Modal and the +staging v2 database: + +- a submit, poll and result cycle for a new US household; +- a latency check: several calculations submitted at once, so the Simulation + Entrypoint serves them in parallel, each accepted within + ``STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`` and finished within + ``STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS``; +- a memory check: a household varied across 401 points of employment income, + every variable calculated at each point, which must finish without failing. + +The measured times are reported as a warning, so the job log keeps them. +""" + +from __future__ import annotations + +import os +import statistics +import warnings +from concurrent.futures import ThreadPoolExecutor + +import httpx + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + FinishedCalculation, + LiveHousehold, + calculate, + store_household, +) + +PARALLEL_CALCULATIONS = int(os.environ.get("STAGING_HOUSEHOLD_PARALLEL_COUNT", "6")) +SUBMIT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS", "10") +) +FINISH_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS", "300") +) +MEMORY_CHECK_POINTS = 401 + + +class HouseholdCalculationMeasurement(UserWarning): + """Times measured by the live household calculation checks.""" + + +def _us_household(name: str, employment_income: int) -> LiveHousehold: + return LiveHousehold( + name=name, + country_id="us", + people={ + "adult": { + "age": {YEAR: 40}, + "employment_income": {YEAR: employment_income}, + } + }, + groups={ + "household": { + "state_name": {YEAR: "NY"}, + "household_net_income": {YEAR: None}, + } + }, + ) + + +def _probe_amount(integration_probe_id: str) -> int: + return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 + + +def test_a_us_household_is_submitted_polled_and_read(api_client, integration_probe_id): + household = _us_household("cycle", 51_000 + _probe_amount(integration_probe_id)) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + item = finished.item + assert item["status"] == "succeeded", item["error"] + assert item["household_id"] == household_id + assert item["completed_at"] is not None + result = item["result"] + net_income = result["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert isinstance(net_income, int | float) + assert result["spm_config"] is not None + assert result["spm_provenance"] is not None + assert result["policyengine_version"] + read_again = api_client.get( + f"/v2/household-calculations/{item['calculation_id']}" + ).json()["result"] + assert read_again == item + + +def test_parallel_calculations_finish_within_the_latency_limits( + api_base_url, integration_probe_id +): + probe_amount = _probe_amount(integration_probe_id) + households = [ + _us_household(f"parallel-{index}", 20_000 + 1_000 * index + probe_amount) + for index in range(PARALLEL_CALCULATIONS) + ] + + def run(household: LiveHousehold) -> FinishedCalculation: + with httpx.Client(base_url=api_base_url, timeout=90.0) as client: + return calculate(client, household, store_household(client, household)) + + with ThreadPoolExecutor(max_workers=PARALLEL_CALCULATIONS) as pool: + outcomes = list(pool.map(run, households)) + + submit_times = [outcome.submit_seconds for outcome in outcomes] + finish_times = [outcome.total_seconds for outcome in outcomes] + warnings.warn( + HouseholdCalculationMeasurement( + f"{PARALLEL_CALCULATIONS} parallel household calculations: submit " + f"median {statistics.median(submit_times):.2f} s, longest " + f"{max(submit_times):.2f} s; finish median " + f"{statistics.median(finish_times):.1f} s, longest " + f"{max(finish_times):.1f} s" + ), + stacklevel=1, + ) + assert [outcome.item["status"] for outcome in outcomes] == ["succeeded"] * len( + outcomes + ), [outcome.item["error"] for outcome in outcomes] + assert max(submit_times) <= SUBMIT_LIMIT_SECONDS, submit_times + assert max(finish_times) <= FINISH_LIMIT_SECONDS, finish_times + + +def test_a_household_across_many_points_finishes(api_client, integration_probe_id): + household = LiveHousehold( + name="memory", + country_id="us", + people={"adult": {"age": {YEAR: 45}}}, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": {YEAR: None}, + } + }, + axes=[ + [ + { + "name": "employment_income", + "count": MEMORY_CHECK_POINTS, + "min": 0, + "max": 400_000 + _probe_amount(integration_probe_id), + "period": YEAR, + } + ] + ], + ) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + warnings.warn( + HouseholdCalculationMeasurement( + f"Household across {MEMORY_CHECK_POINTS} points: submit " + f"{finished.submit_seconds:.2f} s, finish {finished.total_seconds:.1f} s" + ), + stacklevel=1, + ) + assert finished.item["status"] == "succeeded", finished.item["error"] + net_income = finished.item["result"]["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert len(net_income) == MEMORY_CHECK_POINTS diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py new file mode 100644 index 000000000..0cebaa53e --- /dev/null +++ b/tests/integration/test_live_v2_household_parity.py @@ -0,0 +1,154 @@ +"""Live staging parity of v2 household calculations with API v1's ``/calculate``. + +Each household is stored through ``/v2/households``, calculated through +``/v2/household-calculations`` (the v2 household function on Modal) and, as +the same situation, through API v1's in-process ``/calculate`` on the same +candidate. Every calculated value must match, axes included. A run-specific +input makes each household new, so no earlier calculation is reused. +""" + +from __future__ import annotations + +import pytest + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + LiveHousehold, + calculate, + calculate_with_v1, + household_differences, + store_household, +) + +CALCULATED = {YEAR: None} + + +def _probe_amount(integration_probe_id: str) -> int: + """Return a small run-specific amount that makes each household new.""" + return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 + + +def _parity_households(probe_amount: int) -> list[LiveHousehold]: + return [ + LiveHousehold( + name="us-single-california", + country_id="us", + people={ + "adult": { + "age": {YEAR: 35}, + "employment_income": {YEAR: 42_000 + probe_amount}, + "income_tax": CALCULATED, + } + }, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + ), + LiveHousehold( + name="us-couple-two-children-texas-axes", + country_id="us", + people={ + "first-adult": {"age": {YEAR: 38}}, + "second-adult": { + "age": {YEAR: 36}, + "employment_income": {YEAR: 15_000 + probe_amount}, + }, + "first-child": {"age": {YEAR: 7}}, + "second-child": {"age": {YEAR: 3}}, + }, + groups={ + "household": { + "state_name": {YEAR: "TX"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 100_000, + "period": YEAR, + } + ] + ], + ), + LiveHousehold( + name="uk-single", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 30}, + "employment_income": {YEAR: 28_000 + probe_amount}, + "income_tax": CALCULATED, + "national_insurance": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + ), + LiveHousehold( + name="uk-couple-child-axes", + country_id="uk", + people={ + "first-adult": {"age": {YEAR: 34}}, + "second-adult": { + "age": {YEAR: 33}, + "employment_income": {YEAR: 9_000 + probe_amount}, + }, + "child": {"age": {YEAR: 4}}, + }, + groups={ + "household": {"household_net_income": CALCULATED}, + "benunit": { + "universal_credit": CALCULATED, + "child_benefit": CALCULATED, + }, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 60_000, + "period": YEAR, + } + ] + ], + ), + ] + + +PARITY_NAMES = [household.name for household in _parity_households(0)] + + +@pytest.mark.parametrize("household_name", PARITY_NAMES) +def test_a_v2_household_calculation_matches_api_v1( + api_client, integration_probe_id, household_name +): + (household,) = [ + candidate + for candidate in _parity_households(_probe_amount(integration_probe_id)) + if candidate.name == household_name + ] + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + v1_answer = calculate_with_v1(api_client, household) + + assert finished.item["status"] == "succeeded", finished.item["error"] + result = finished.item["result"] + differences = household_differences(v1_answer["result"], result["household"]) + assert differences == [], "\n".join(differences[:50]) + assert result["warnings"] == v1_answer.get("warnings", []) + if household.country_id == "us": + assert result["spm_config"] is not None + assert result["spm_provenance"] is not None diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index ee7a48b9f..a9d0dc878 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1894,6 +1894,12 @@ def _assert_live_tests_run_in_parallel_after_readiness( assert "tests/integration/test_live_budget_window_cache.py" in live_test_script assert "tests/integration/test_live_calculate.py" in live_test_script assert "tests/integration/test_live_economy.py" in live_test_script + assert "tests/integration/test_live_v2_household_calculations.py" in ( + live_test_script + ) + assert "tests/integration/test_live_v2_household_parity.py" in live_test_script + # The household calculation checks wait on Modal; 30 minutes leaves room. + assert "timeout-minutes: 30" in parallel_live_tests assert "tests/integration/test_live_v2_policies.py" not in parallel_live_tests assert ( "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" From c66b1d0b0c6cc721297fbd78c9402f71b0248346 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:12:21 +0400 Subject: [PATCH 095/118] Read simulation service answers through the generated models A 422 from the simulation service (a household or options it finds invalid, such as a group no person belongs to) now answers 422 with each invalid field and why, instead of 503 without a reason. A body that is not JSON or not of the documented shape, and a gateway token that cannot be fetched, now answer 503 instead of 500. Each answer is decoded once, by the generated client. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 9 +- .../household_calculations/response_models.py | 4 +- .../v2/household_calculations/routes.py | 8 +- .../simulation_client.py | 117 +++++++++++------- .../v2/household_calculations/types.py | 7 ++ .../v2/test_household_calculation_routes.py | 14 +++ ...household_calculation_simulation_client.py | 73 +++++++++++ 7 files changed, 185 insertions(+), 47 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 46c8730d3..7aee10c96 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -289,9 +289,12 @@ The routes always call the Cloud Run Simulation Entrypoint, whatever for every simulation call, with `GATEWAY_AUTH_REQUIRED=1`. Without the URL, or with gateway auth partly configured, both routes answer -503. They also answer 503 when the simulation service cannot be reached, -answers with an error, or refuses the calculation (for example when no v2 -worker is deployed for the catalog version's PolicyEngine.py version). +503. They also answer 503 when the simulation service cannot be reached, its +gateway token cannot be fetched, it answers with an error or with a body of +another shape, or it refuses the calculation (for example when no v2 worker is +deployed for the catalog version's PolicyEngine.py version). A household or +options the simulation service finds invalid, such as a group no person +belongs to, get 422 with each invalid field and why. ### Staging checks diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index b2b1b160d..d21b25968 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -94,7 +94,9 @@ class HouseholdCalculationResponse(StrictHouseholdCalculationAPIModel): "model": V2ErrorResponse, "description": ( "The request does not match the schema, the country does not offer " - "household calculations, or no year can be chosen." + "household calculations, the household has no default year, or the " + "simulation service finds the household or its options invalid " + "(the message lists why)." ), }, 500: { diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py index c88ce2289..019ba1c41 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -33,6 +33,7 @@ HouseholdCalculationResponse, ) from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, SimulationServiceRefusedError, SimulationServiceUnavailableError, ) @@ -56,7 +57,12 @@ HOUSEHOLD_CALCULATION_ERROR_MAPPINGS = ( V2ErrorMapping( - (HouseholdCalculationCountryError, HouseholdCalculationYearError), 422 + ( + HouseholdCalculationCountryError, + HouseholdCalculationYearError, + SimulationServiceInvalidRequestError, + ), + 422, ), V2ErrorMapping( ( diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 1474bd462..498d23614 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -3,9 +3,11 @@ Calls go to the Cloud Run Simulation Entrypoint (``SIMULATION_ENTRYPOINT_URL``), the only simulation service with these routes, with the gateway credentials the Public API already uses (``GATEWAY_AUTH_*``) and the request's -correlation headers. A 400 is a refusal, whose reason the service gives; any -other failure, including a 503, no answer or missing configuration, means the -service is unavailable. +correlation headers. Answers are read through the client's generated models: +a 400 is a refusal, whose reason the service gives; a 422 lists what the +service finds invalid in the household or its options; anything else, +including a 503, no answer, an answer of another shape, a gateway token that +cannot be fetched or missing configuration, means the service is unavailable. One client serves the process, so its connections and its gateway token are reused across requests. @@ -14,8 +16,7 @@ from __future__ import annotations from functools import lru_cache -from http import HTTPStatus -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar import httpx from policyengine_api_simulation_client import AuthenticatedClient @@ -23,9 +24,18 @@ get_household_calculation, submit_household_calculation, ) +from policyengine_api_simulation_client.models.household_calculation_response import ( + HouseholdCalculationResponse, +) from policyengine_api_simulation_client.models.household_calculation_submission import ( HouseholdCalculationSubmission, ) +from policyengine_api_simulation_client.models.household_route_error import ( + HouseholdRouteError, +) +from policyengine_api_simulation_client.models.http_validation_error import ( + HTTPValidationError, +) from policyengine_observability import ( # type: ignore[import-untyped] # the package has no type information instrument_httpx, ) @@ -43,23 +53,57 @@ ) from policyengine_api.observability import get_runtime from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, SimulationServiceRefusedError, SimulationServiceUnavailableError, ) if TYPE_CHECKING: + from collections.abc import Callable from uuid import UUID + from policyengine_api_simulation_client.types import Response + +ParsedT = TypeVar("ParsedT") + REQUEST_TIMEOUT_SECONDS = 30.0 CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" -def _answer(response: Any) -> tuple[int, object]: # noqa: ANN401 # the generated client's Response +def _call(send: Callable[[], Response[ParsedT]]) -> Response[ParsedT]: + """Send one request; raise SimulationServiceUnavailableError if it cannot be read. + + The generated routes decode the body for each status they document, so a + body that is not JSON or not of the documented shape raises while + decoding. + """ try: - body = httpx.Response(response.status_code, content=response.content).json() - except ValueError: - body = None - return int(response.status_code), body + return send() + except (httpx.HTTPError, GatewayAuthError) as error: + msg = "The simulation service could not be reached." + raise SimulationServiceUnavailableError(msg) from error + except (ValueError, KeyError, TypeError) as error: + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) from error + + +def _invalid_request_reason(error: HTTPValidationError) -> str: + """Return each invalid field and why, without the request's ``body`` prefix.""" + reasons = [] + for item in error.detail if isinstance(error.detail, list) else []: + location = [str(part) for part in item.loc] + if location[:1] == ["body"]: + location = location[1:] + reasons.append(f"{'.'.join(location)}: {item.msg}") + return "The simulation service cannot calculate this household: " + ( + "; ".join(reasons) or "the request is invalid" + ) + + +def _unavailable(status_code: int) -> SimulationServiceUnavailableError: + return SimulationServiceUnavailableError( + f"The simulation service answered {status_code}." + ) def _gateway_auth() -> GatewayBearerAuth | None: @@ -74,13 +118,6 @@ def _gateway_auth() -> GatewayBearerAuth | None: return None -def _refusal_reason(body: object) -> str: - reason = body.get("detail") if isinstance(body, dict) else None - if isinstance(reason, str): - return reason - return "The simulation service refused the household calculation." - - class SimulationHouseholdClient: """Submit and read household calculations on the simulation service.""" @@ -128,38 +165,34 @@ def from_environment(cls) -> SimulationHouseholdClient: def submit(self, submission: dict[str, Any]) -> dict[str, Any]: """Submit a calculation and return the service's calculation answer.""" - try: - response = submit_household_calculation.sync_detailed( - client=self._client, - body=HouseholdCalculationSubmission.from_dict(submission), + body = HouseholdCalculationSubmission.from_dict(submission) + response = _call( + lambda: submit_household_calculation.sync_detailed( + client=self._client, body=body ) - except httpx.HTTPError as error: - msg = "The simulation service could not be reached." - raise SimulationServiceUnavailableError(msg) from error - status, body = _answer(response) - if status == HTTPStatus.BAD_REQUEST: - raise SimulationServiceRefusedError(_refusal_reason(body)) - if status != HTTPStatus.ACCEPTED or not isinstance(body, dict): - msg = f"The simulation service answered {status}." - raise SimulationServiceUnavailableError(msg) - return body + ) + answer = response.parsed + if isinstance(answer, HouseholdCalculationResponse): + return answer.to_dict() + if isinstance(answer, HTTPValidationError): + raise SimulationServiceInvalidRequestError(_invalid_request_reason(answer)) + if isinstance(answer, HouseholdRouteError) and response.status_code == 400: # noqa: PLR2004 # the service's refusal status + raise SimulationServiceRefusedError(answer.detail) + raise _unavailable(response.status_code) def read(self, calculation_id: UUID) -> dict[str, Any] | None: """Return the service's calculation answer, or None when it has none.""" - try: - response = get_household_calculation.sync_detailed( + response = _call( + lambda: get_household_calculation.sync_detailed( calculation_id, client=self._client ) - except httpx.HTTPError as error: - msg = "The simulation service could not be reached." - raise SimulationServiceUnavailableError(msg) from error - status, body = _answer(response) - if status == HTTPStatus.NOT_FOUND: + ) + answer = response.parsed + if isinstance(answer, HouseholdCalculationResponse): + return answer.to_dict() + if isinstance(answer, HouseholdRouteError) and response.status_code == 404: # noqa: PLR2004 # the service's unknown-calculation status return None - if status != HTTPStatus.OK or not isinstance(body, dict): - msg = f"The simulation service answered {status}." - raise SimulationServiceUnavailableError(msg) - return body + raise _unavailable(response.status_code) @lru_cache(maxsize=1) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 3cc20ceba..b94a61323 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -157,6 +157,13 @@ class SimulationServiceRefusedError(RuntimeError): """The simulation service refused the calculation; the message says why.""" +class SimulationServiceInvalidRequestError(ValueError): + """The simulation service finds the household or its options invalid. + + The message lists each invalid field and why, as the service reports it. + """ + + class CountrySource(Protocol): """The countries table (``V2CountryService``).""" diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index a6f396bd7..5cef47c7e 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -19,6 +19,7 @@ V2HouseholdCalculationService, ) from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, SimulationServiceRefusedError, SimulationServiceUnavailableError, ) @@ -222,3 +223,16 @@ def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None assert response.status_code == 422 assert app.simulation.submissions == [] + + +def test_a_household_the_simulation_service_finds_invalid_gets_422_with_why() -> None: + app = App() + app.simulation.error = SimulationServiceInvalidRequestError( + "The simulation service cannot calculate this household: " + "household.households.household-1.members: List should have at least 1 item" + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 422 + assert "members" in response.json()["message"] diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py index 2f73b5a82..e99eb96ca 100644 --- a/tests/unit/v2/test_household_calculation_simulation_client.py +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -11,10 +11,12 @@ import httpx import pytest +from policyengine_api.libs.gateway_auth import GatewayAuthError from policyengine_api.services.v2.household_calculations.simulation_client import ( SimulationHouseholdClient, ) from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, SimulationServiceRefusedError, SimulationServiceUnavailableError, ) @@ -164,3 +166,74 @@ def test_the_configured_service_is_called_without_auth_when_none_is_set( client = SimulationHouseholdClient.from_environment() assert client.base_url == BASE_URL + + +def test_a_household_the_service_finds_invalid_is_reported_with_its_reasons() -> None: + client = client_answering( + 422, + { + "detail": [ + { + "loc": ["body", "household", "households", "h", "members"], + "msg": "List should have at least 1 item after validation", + "type": "too_short", + } + ] + }, + [], + ) + + with pytest.raises( + SimulationServiceInvalidRequestError, + match=r"household\.households\.h\.members: List should have at least 1 item", + ): + client.submit(SUBMISSION) + + +@pytest.mark.parametrize("status_code", [400, 503, 202, 200]) +def test_an_answer_that_is_not_json_is_unavailable(status_code: int) -> None: + def answer(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code, + content=b"Service Unavailable", + headers={"Content-Type": "text/html"}, + ) + + client = SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(answer)} + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +def test_an_answer_of_the_wrong_shape_is_unavailable() -> None: + client = client_answering(202, {"calculation_id": "not a calculation"}, []) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + + +def test_a_gateway_token_that_cannot_be_fetched_is_unavailable() -> None: + class FailingTokenAuth(httpx.Auth): + def auth_flow(self, request: httpx.Request): # httpx's generator protocol + message = "The gateway token request failed" + raise GatewayAuthError(message) + yield request + + client = SimulationHouseholdClient( + BASE_URL, + httpx_args={ + "auth": FailingTokenAuth(), + "transport": httpx.MockTransport( + lambda request: httpx.Response(202, json=simulation_calculation()) + ), + }, + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) From 0f9bf9fbd6eb2be0cc041ca883c10b8998583a9f Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:15:32 +0400 Subject: [PATCH 096/118] Refuse axes the simulation service would refuse, before sending them The axes of one group must share one point count, and the groups' point counts may multiply to at most 10,000, as the simulation service requires. Such requests now get 422 from the Public API before anything is read or sent. Co-Authored-By: Claude Opus 5.5 --- .../v2/household_calculations/types.py | 27 +++++++++++-- .../v2/test_household_calculation_routes.py | 38 ++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index b94a61323..11712acfa 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -2,11 +2,12 @@ from __future__ import annotations +import math from datetime import datetime from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol, Self from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, JsonValue +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator from policyengine_api.query_parameters import CountryId, DefaultYear @@ -20,7 +21,8 @@ ) from policyengine_api.services.v2.policies.types import PolicyRead -# The simulation service's limits on axes. +# The simulation service's limits on axes: the product of the groups' point +# counts is at most MAXIMUM_AXIS_POINTS. MAXIMUM_AXIS_POINTS = 10_000 MAXIMUM_AXIS_GROUPS = 8 MAXIMUM_PARALLEL_AXES = 32 @@ -59,7 +61,9 @@ class HouseholdCalculationInput(StrictCalculationModel): """A stored household to calculate under a stored policy, or current law. ``year`` defaults to the household's default year. ``axes`` vary inputs - across points; ``add_missing_variables`` adds every yearly, monthly and + across points: the axes of one group vary together, so they share one + point count, and the groups form a grid of at most 10,000 points. + ``add_missing_variables`` adds every yearly, monthly and eternal variable the household lacks, as ``/calculate-full`` does. ``force`` calculates again even when a recent calculation can be reused. """ @@ -74,6 +78,23 @@ class HouseholdCalculationInput(StrictCalculationModel): add_missing_variables: bool = False force: bool = False + @field_validator("axes") + @classmethod + def limit_axis_points( + cls, axes: list[list[HouseholdCalculationAxis]] | None + ) -> list[list[HouseholdCalculationAxis]] | None: + """Refuse groups of uneven point counts and grids of too many points.""" + if axes is None: + return None + for group in axes: + if len({axis.count for axis in group}) != 1: + msg = "the axes of one group must share one point count" + raise ValueError(msg) + if math.prod(group[0].count for group in axes) > MAXIMUM_AXIS_POINTS: + msg = f"the axis groups may vary at most {MAXIMUM_AXIS_POINTS} points" + raise ValueError(msg) + return axes + class HouseholdCalculationResult(StrictCalculationModel): """A succeeded calculation's household and its notes. diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 5cef47c7e..836de39bd 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -210,11 +210,32 @@ def test_a_simulation_service_that_cannot_calculate_answers_503( "changes", [ {"axes": [[{"name": "employment_income", "count": 0, "min": 0, "max": 1}]]}, + { + "axes": [ + [ + {"name": "employment_income", "count": 3, "min": 0, "max": 1}, + {"name": "age", "count": 4, "min": 20, "max": 60}, + ] + ] + }, + { + "axes": [ + [{"name": "employment_income", "count": 101, "min": 0, "max": 1}], + [{"name": "age", "count": 100, "min": 20, "max": 60}], + ] + }, {"country_id": "ca"}, {"unexpected": True}, {"year": 1800}, ], - ids=["axis-count", "country", "unknown-field", "year"], + ids=[ + "axis-count", + "parallel-axes-of-different-counts", + "too-many-axis-points", + "country", + "unknown-field", + "year", + ], ) def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None: app = App() @@ -236,3 +257,18 @@ def test_a_household_the_simulation_service_finds_invalid_gets_422_with_why() -> assert response.status_code == 422 assert "members" in response.json()["message"] + + +def test_axis_groups_within_the_point_limit_are_accepted() -> None: + app = App() + axes = [ + [ + {"name": "employment_income", "count": 100, "min": 0, "max": 1}, + {"name": "self_employment_income", "count": 100, "min": 0, "max": 1}, + ], + [{"name": "age", "count": 100, "min": 20, "max": 60}], + ] + + response = app.client.post(PATH, json=body(axes=axes)) + + assert response.status_code == 202 From 0c9b65c63f44163dd32db58bdee5c338988638a5 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:18:40 +0400 Subject: [PATCH 097/118] Calculate each household for its default year, and take no force The request no longer takes `year`: the calculation's year is the household's default year, the year its values name, so a calculation is never labelled with a year it did not calculate. It no longer takes `force` either, so callers without an account cannot skip the 28-day reuse of a succeeded calculation and start a new Modal call for each request. The stage 11 guide notes the year the household function uses when it adds missing variables. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 13 +++++++++++-- .../household_calculations/request_models.py | 10 +++++----- .../v2/household_calculations/services.py | 6 ++++-- .../v2/household_calculations/types.py | 8 +++----- .../v2/household_calculations/validators.py | 10 ++++------ .../live_household_calculations.py | 4 ---- .../v2/test_household_calculation_routes.py | 4 +++- .../v2/test_household_calculation_service.py | 19 ++++++++++++++----- 8 files changed, 44 insertions(+), 30 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 7aee10c96..c43665420 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -261,8 +261,10 @@ for immediate rollback. `POST /v2/household-calculations` calculates a household stored through `/v2/households`, under a policy stored through `/v2/policies` or, without -`policy_id`, under current law. The body may add `year` (the household's -`default_year` otherwise), `axes`, `add_missing_variables` and `force`. The +`policy_id`, under current law, for the household's `default_year`. The body +may add `axes` and `add_missing_variables`. An identical calculation that +succeeded in the last 28 days is returned again; callers cannot force a new +one. The Public API checks the country's `countries` row allows household calculations (422 otherwise; a household-only country is accepted), reads the household and policy through the same services as their routes (404 when either is not @@ -274,6 +276,13 @@ tables. The answer is 202 with the calculation's `calculation_id` and `status` is `succeeded`, with `result`, or `failed`, with `error`. A US result carries `spm_config` and `spm_provenance`. +With `add_missing_variables`, the v2 household function adds each missing +variable for the year it would choose for `/calculate-full`: the first period +of the age of a person named `you`, else the current calendar year (UTC). The +Public API names people by their record IDs, so for a household whose +`default_year` is not the current year the added variables are requested for +the current year until the household function takes the calculation's year. + The simulation service stores every calculation as a simulation run, so a calculation no account saved is still stored, with no report. Neither route needs an account. diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py index c8b0e4427..eb78c9236 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py @@ -12,12 +12,12 @@ class HouseholdCalculationRequest(HouseholdCalculationInput): ``household_id`` and ``policy_id`` name records of ``country_id`` stored through ``/v2/households`` and ``/v2/policies``; without ``policy_id`` the - household is calculated under current law. ``year`` defaults to the - household's ``default_year``. Each group in ``axes`` varies its inputs - together across ``count`` points; several groups form a grid. + household is calculated under current law, for the household's + ``default_year``. Each group in ``axes`` varies its inputs together across + ``count`` points; several groups form a grid of at most 10,000 points. ``add_missing_variables`` adds every variable the household lacks before - calculating, and ``force`` calculates again even when a recent identical - calculation exists. + calculating. An identical calculation that succeeded in the last 28 days + is returned again instead of being repeated. """ def to_service_input(self) -> HouseholdCalculationInput: diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 3eafaeb71..728ada8cc 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -83,7 +83,7 @@ def submit_calculation( country_id=country_id, policy_id=calculation.policy_id ) ) - year = calculation_year(calculation.year, household.default_year) + year = calculation_year(household.default_year) try: catalog = self._catalogs.get_model_by_country(country_id) finally: @@ -117,7 +117,9 @@ def submit_calculation( ), "add_missing_variables": calculation.add_missing_variables, }, - "force": calculation.force, + # A calculation is reused for 28 days after it succeeded; callers + # without an account must not start new Modal calls at will. + "force": False, "trigger": "initial", } spm = saved_spm_choice(household.household_data) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 11712acfa..78d50f8ed 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator -from policyengine_api.query_parameters import CountryId, DefaultYear +from policyengine_api.query_parameters import CountryId if TYPE_CHECKING: from collections.abc import Mapping @@ -60,23 +60,21 @@ class HouseholdCalculationAxis(StrictCalculationModel): class HouseholdCalculationInput(StrictCalculationModel): """A stored household to calculate under a stored policy, or current law. - ``year`` defaults to the household's default year. ``axes`` vary inputs + The calculation's year is the household's default year, the year its + values name. ``axes`` vary inputs across points: the axes of one group vary together, so they share one point count, and the groups form a grid of at most 10,000 points. ``add_missing_variables`` adds every yearly, monthly and eternal variable the household lacks, as ``/calculate-full`` does. - ``force`` calculates again even when a recent calculation can be reused. """ country_id: CountryId household_id: UUID policy_id: UUID | None = None - year: DefaultYear | None = None axes: Annotated[list[AxisGroup], Field(max_length=MAXIMUM_AXIS_GROUPS)] | None = ( None ) add_missing_variables: bool = False - force: bool = False @field_validator("axes") @classmethod diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index 507590131..5475e3328 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -15,7 +15,7 @@ class HouseholdCalculationCountryError(ValueError): class HouseholdCalculationYearError(ValueError): - """The request names no year and the household has no default year.""" + """The household has no default year to calculate.""" class HouseholdCalculationNotFoundError(LookupError): @@ -37,11 +37,9 @@ def require_household_country( raise HouseholdCalculationCountryError(msg) -def calculation_year(requested_year: int | None, default_year: int | None) -> int: - """Return the requested year, or else the household's default year.""" - if requested_year is not None: - return requested_year +def calculation_year(default_year: int | None) -> int: + """Return the household's default year, the year its values name.""" if default_year is None: - msg = "This household has no default year; send the year to calculate." + msg = "This household has no default year to calculate." raise HouseholdCalculationYearError(msg) return default_year diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 6fbf92647..cc59fbe15 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -116,16 +116,12 @@ def calculate( client: httpx.Client, household: LiveHousehold, household_id: str, - *, - force: bool = False, ) -> FinishedCalculation: """Submit the stored household's calculation and poll until it finishes.""" started = time.monotonic() body: dict[str, Any] = { "country_id": household.country_id, "household_id": household_id, - "year": int(YEAR), - "force": force, } if household.axes is not None: body["axes"] = household.axes diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 836de39bd..6ab92cfc6 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -226,7 +226,8 @@ def test_a_simulation_service_that_cannot_calculate_answers_503( }, {"country_id": "ca"}, {"unexpected": True}, - {"year": 1800}, + {"year": 2026}, + {"force": True}, ], ids=[ "axis-count", @@ -235,6 +236,7 @@ def test_a_simulation_service_that_cannot_calculate_answers_503( "country", "unknown-field", "year", + "force", ], ) def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None: diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index b2b16bbc1..30242c867 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -78,7 +78,7 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: parts = Parts() calculation = parts.service.submit_calculation( - calculation_input(axes=AXES, add_missing_variables=True, force=True) + calculation_input(axes=AXES, add_missing_variables=True) ) (submission,) = parts.simulation.submissions @@ -99,7 +99,7 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: }, }, "options": {"axes": AXES, "add_missing_variables": True}, - "force": True, + "force": False, "trigger": "initial", } assert calculation.calculation_id == CALCULATION_ID @@ -124,17 +124,26 @@ def test_current_law_and_no_saved_choice_send_neither() -> None: assert submission["options"] == {"axes": None, "add_missing_variables": False} -def test_the_year_is_the_requested_one_or_the_households_default() -> None: +def test_the_year_is_the_households_default_year() -> None: parts = Parts() + parts.households.households[("us", HOUSEHOLD_ID)] = household_read( + default_year=2027 + ) + + parts.service.submit_calculation(calculation_input()) + + assert parts.simulation.submissions[0]["year"] == 2027 - parts.service.submit_calculation(calculation_input(year=2027)) + +def test_a_household_without_a_default_year_is_refused() -> None: + parts = Parts() parts.households.households[("us", HOUSEHOLD_ID)] = household_read( default_year=None ) with pytest.raises(HouseholdCalculationYearError): parts.service.submit_calculation(calculation_input()) - assert [submission["year"] for submission in parts.simulation.submissions] == [2027] + assert parts.simulation.submissions == [] def test_a_country_without_household_calculations_is_refused_by_name() -> None: From aaf0df33a4ffca6e972f2398554d59054407784e Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:21:59 +0400 Subject: [PATCH 098/118] Open the catalog reader only to submit, and reuse the household collections A household calculation's catalog reader, with its database session, is now opened only once a submission needs the catalog version, and closed after it; polls and refused submissions open none. The situation's group collections come from the households service's own lists, and the live checks share one helper for their run-specific amounts. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/dependencies.py | 2 +- .../v2/household_calculations/services.py | 16 +++++++++---- .../household_calculations/transformations.py | 22 ++++++++++------- .../live_household_calculations.py | 5 ++++ .../test_live_v2_household_calculations.py | 13 ++++------ .../test_live_v2_household_parity.py | 18 ++++++-------- .../v2/test_household_calculation_routes.py | 2 +- .../v2/test_household_calculation_service.py | 24 ++++++++++++++++--- 8 files changed, 64 insertions(+), 38 deletions(-) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d0bff862..9521876ae 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -898,7 +898,7 @@ def _default_v2_household_calculation_service_factory() -> ( countries=_default_v2_country_service_factory(), households=_default_v2_household_service_factory(), policies=_default_v2_policy_service_factory(), - catalogs=_default_v2_metadata_reader_factory(), + catalog_reader_factory=_default_v2_metadata_reader_factory, simulation=process_simulation_household_client(), ) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 728ada8cc..4c5d7fca8 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -31,6 +31,7 @@ ) if TYPE_CHECKING: + from collections.abc import Callable from uuid import UUID from policyengine_api.services.v2.household_calculations.types import ( @@ -52,14 +53,18 @@ def __init__( countries: CountrySource, households: HouseholdSource, policies: PolicySource, - catalogs: CatalogSource, + catalog_reader_factory: Callable[[], CatalogSource], simulation: SimulationHouseholdCalculations, ) -> None: - """Read through these services for one request.""" + """Read through these services for one request. + + ``catalog_reader_factory`` opens a catalog reader, with its own + database session, only once a submission needs the catalog version. + """ self._countries = countries self._households = households self._policies = policies - self._catalogs = catalogs + self._catalog_reader_factory = catalog_reader_factory self._simulation = simulation def submit_calculation( @@ -84,10 +89,11 @@ def submit_calculation( ) ) year = calculation_year(household.default_year) + catalog_reader = self._catalog_reader_factory() try: - catalog = self._catalogs.get_model_by_country(country_id) + catalog = catalog_reader.get_model_by_country(country_id) finally: - self._catalogs.close() + catalog_reader.close() submission: dict[str, Any] = { "country": country_id, "catalog_version_id": str(catalog.model_version.id), diff --git a/policyengine_api/services/v2/household_calculations/transformations.py b/policyengine_api/services/v2/household_calculations/transformations.py index 53911db0f..51de6eb10 100644 --- a/policyengine_api/services/v2/household_calculations/transformations.py +++ b/policyengine_api/services/v2/household_calculations/transformations.py @@ -18,21 +18,25 @@ from typing import TYPE_CHECKING, Any +from policyengine_api.services.v2.households.transformations import ( + LEGACY_COLLECTION_NAMES, +) +from policyengine_api.services.v2.households.validators import ( + SUPPORTED_ENTITY_COLLECTIONS, +) + if TYPE_CHECKING: from collections.abc import Mapping from policyengine_api.services.v2.policies.types import PolicyRead -# Each country's stored group collections and their plural in the situation. +# Each country's stored group collections, as the households service accepts +# them, and their plural in the situation. GROUP_COLLECTIONS: dict[str, dict[str, str]] = { - "us": { - "household": "households", - "family": "families", - "tax_unit": "tax_units", - "spm_unit": "spm_units", - "marital_unit": "marital_units", - }, - "uk": {"household": "households", "benunit": "benunits"}, + country_id: { + collection: LEGACY_COLLECTION_NAMES[collection] for collection in collections + } + for country_id, collections in SUPPORTED_ENTITY_COLLECTIONS.items() } OPEN_END_DATE = "2100-12-31" diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index cc59fbe15..b3dc8f936 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -97,6 +97,11 @@ class FinishedCalculation: total_seconds: float +def probe_amount(integration_probe_id: str) -> int: + """Return a small run-specific amount that makes each household new.""" + return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 + + def store_household(client: httpx.Client, household: LiveHousehold) -> str: """Store the household through ``/v2/households`` and return its ID.""" response = client.post( diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py index 26a63fc49..278575621 100644 --- a/tests/integration/test_live_v2_household_calculations.py +++ b/tests/integration/test_live_v2_household_calculations.py @@ -29,6 +29,7 @@ FinishedCalculation, LiveHousehold, calculate, + probe_amount, store_household, ) @@ -65,12 +66,8 @@ def _us_household(name: str, employment_income: int) -> LiveHousehold: ) -def _probe_amount(integration_probe_id: str) -> int: - return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 - - def test_a_us_household_is_submitted_polled_and_read(api_client, integration_probe_id): - household = _us_household("cycle", 51_000 + _probe_amount(integration_probe_id)) + household = _us_household("cycle", 51_000 + probe_amount(integration_probe_id)) household_id = store_household(api_client, household) finished = calculate(api_client, household, household_id) @@ -96,9 +93,9 @@ def test_a_us_household_is_submitted_polled_and_read(api_client, integration_pro def test_parallel_calculations_finish_within_the_latency_limits( api_base_url, integration_probe_id ): - probe_amount = _probe_amount(integration_probe_id) + run_amount = probe_amount(integration_probe_id) households = [ - _us_household(f"parallel-{index}", 20_000 + 1_000 * index + probe_amount) + _us_household(f"parallel-{index}", 20_000 + 1_000 * index + run_amount) for index in range(PARALLEL_CALCULATIONS) ] @@ -145,7 +142,7 @@ def test_a_household_across_many_points_finishes(api_client, integration_probe_i "name": "employment_income", "count": MEMORY_CHECK_POINTS, "min": 0, - "max": 400_000 + _probe_amount(integration_probe_id), + "max": 400_000 + probe_amount(integration_probe_id), "period": YEAR, } ] diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index 0cebaa53e..f310add83 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -17,18 +17,14 @@ calculate, calculate_with_v1, household_differences, + probe_amount, store_household, ) CALCULATED = {YEAR: None} -def _probe_amount(integration_probe_id: str) -> int: - """Return a small run-specific amount that makes each household new.""" - return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 - - -def _parity_households(probe_amount: int) -> list[LiveHousehold]: +def _parity_households(run_amount: int) -> list[LiveHousehold]: return [ LiveHousehold( name="us-single-california", @@ -36,7 +32,7 @@ def _parity_households(probe_amount: int) -> list[LiveHousehold]: people={ "adult": { "age": {YEAR: 35}, - "employment_income": {YEAR: 42_000 + probe_amount}, + "employment_income": {YEAR: 42_000 + run_amount}, "income_tax": CALCULATED, } }, @@ -56,7 +52,7 @@ def _parity_households(probe_amount: int) -> list[LiveHousehold]: "first-adult": {"age": {YEAR: 38}}, "second-adult": { "age": {YEAR: 36}, - "employment_income": {YEAR: 15_000 + probe_amount}, + "employment_income": {YEAR: 15_000 + run_amount}, }, "first-child": {"age": {YEAR: 7}}, "second-child": {"age": {YEAR: 3}}, @@ -87,7 +83,7 @@ def _parity_households(probe_amount: int) -> list[LiveHousehold]: people={ "adult": { "age": {YEAR: 30}, - "employment_income": {YEAR: 28_000 + probe_amount}, + "employment_income": {YEAR: 28_000 + run_amount}, "income_tax": CALCULATED, "national_insurance": CALCULATED, } @@ -101,7 +97,7 @@ def _parity_households(probe_amount: int) -> list[LiveHousehold]: "first-adult": {"age": {YEAR: 34}}, "second-adult": { "age": {YEAR: 33}, - "employment_income": {YEAR: 9_000 + probe_amount}, + "employment_income": {YEAR: 9_000 + run_amount}, }, "child": {"age": {YEAR: 4}}, }, @@ -136,7 +132,7 @@ def test_a_v2_household_calculation_matches_api_v1( ): (household,) = [ candidate - for candidate in _parity_households(_probe_amount(integration_probe_id)) + for candidate in _parity_households(probe_amount(integration_probe_id)) if candidate.name == household_name ] household_id = store_household(api_client, household) diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 6ab92cfc6..e45e6cd62 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -51,7 +51,7 @@ def __init__(self, countries: FakeCountries | None = None) -> None: countries=countries or FakeCountries(), households=self.households, policies=FakePolicies(), - catalogs=FakeCatalogs(), + catalog_reader_factory=FakeCatalogs, simulation=self.simulation, ) dependencies = NativeRouteDependencies( diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index 30242c867..135ad18ae 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -52,16 +52,22 @@ def __init__(self, countries: FakeCountries | None = None) -> None: self.countries = countries or FakeCountries() self.households = FakeHouseholds() self.policies = FakePolicies() - self.catalogs = FakeCatalogs() + self.catalog_readers: list[FakeCatalogs] = [] self.simulation = FakeSimulation() self.service = V2HouseholdCalculationService( countries=self.countries, households=self.households, policies=self.policies, - catalogs=self.catalogs, + catalog_reader_factory=self.open_catalog_reader, simulation=self.simulation, ) + def open_catalog_reader(self) -> FakeCatalogs: + """Open and keep a new catalog reader.""" + reader = FakeCatalogs() + self.catalog_readers.append(reader) + return reader + def calculation_input(**changes: object) -> HouseholdCalculationInput: """Return a calculation of the stored US household under the stored reform.""" @@ -105,7 +111,8 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: assert calculation.calculation_id == CALCULATION_ID assert calculation.status == "running" assert calculation.result is None - assert parts.catalogs.closed + (catalog_reader,) = parts.catalog_readers + assert catalog_reader.closed def test_current_law_and_no_saved_choice_send_neither() -> None: @@ -253,3 +260,14 @@ def test_an_unreachable_simulation_service_is_reported() -> None: with pytest.raises(SimulationServiceUnavailableError): parts.service.submit_calculation(calculation_input()) + + +def test_a_refused_submission_and_a_read_open_no_catalog_reader() -> None: + parts = Parts() + parts.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + with pytest.raises(HouseholdNotFoundError): + parts.service.submit_calculation(calculation_input(household_id=uuid4())) + parts.service.get_calculation(CALCULATION_ID) + + assert parts.catalog_readers == [] From d9101539cb97907decd5d4fc7969da13769cb790 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:25:42 +0400 Subject: [PATCH 099/118] Leave SPM availability unrecorded until the service serves the version Catalog publication recorded {"available": false} for a US version the simulation service's /versions did not serve yet, and a recorded value is never replaced, so the version could never offer SPM settings. Such a version now stays unrecorded and a later publication run records it. The stage 9 runbook lists SIMULATION_ENTRYPOINT_URL for the publication command, and the catalog cache note says how long an earlier answer may be reused. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 17 ++-- .../data/v2/catalog/spm_availability.py | 46 +++++++--- .../fastapi_routes/v2/metadata/common.py | 5 +- .../unit/v2/test_catalog_spm_availability.py | 89 +++++++++++++++++-- 4 files changed, 131 insertions(+), 26 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index d02cbd0b4..ac447d1b1 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -42,8 +42,10 @@ version offers SPM settings for a household: `available`, and when it does, `defaults` (the certified selection). Catalog publication records it from the simulation service's public `GET /versions`: a US version offers SPM when the service lists a canonical capability with pinned defaults for its -PolicyEngine.py version, and other countries never do. A version whose -availability is not recorded yet offers none. The result names the +PolicyEngine.py version, and other countries never do. A US version the +service does not serve yet stays unrecorded until a later publication run, and +a version whose availability is not recorded yet offers none; a response read +before then, and reused for up to a day, keeps saying so. The result names the selected PolicyEngine.py version, so a client that starts without a version learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated @@ -192,10 +194,15 @@ must execute in this order: Alembic chain, confirms all heads are current, and detects ungenerated model changes. 3. Remove the migration URL from the command environment. -4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity and +4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity, `HUGGING_FACE_TOKEN` (from the organization secret - `PE_UK_PRIVATE_HF_READ_TOKEN`), then run - `uv run python scripts/publish_v2_metadata_catalog.py`. Before publishing a + `PE_UK_PRIVATE_HF_READ_TOKEN`) and `SIMULATION_ENTRYPOINT_URL` (from the + environment secret of the same name), then run + `uv run python scripts/publish_v2_metadata_catalog.py`. After publishing, the + command reads the simulation service's public `GET /versions` to record each + version's SPM availability, and fails if the URL is unset or the answer + cannot be read; the catalog itself is already published by then, and the + next run records the availability. Before publishing a catalog version that is not stored yet, the command reads the certified data artifacts its regions' filters use and stops if one cannot select a region. It stops before reading anything if a UK version needs checking and diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index 673759971..2b5f28a2b 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -9,7 +9,11 @@ Publication records it after the catalog itself, on each version that has none yet, so versions published before it existed get it on the next run. A -bundle pins its SPM release, so a recorded value never changes. +bundle pins its SPM release, so a recorded value never changes. A US version +the service does not serve yet (its ``policyengine`` routes do not name it) +is left unrecorded rather than recorded as unavailable, and the next +publication run tries again; ``/v2/model-catalog`` offers no SPM settings for +it meanwhile. """ from __future__ import annotations @@ -44,17 +48,31 @@ MODEL_VERSIONS = TaxBenefitModelVersion.__table__ +def _served(policyengine_version: str, versions: dict[str, Any]) -> bool: + """Return whether the service's ``policyengine`` routes name the version.""" + routes = versions.get("policyengine") + if not isinstance(routes, dict): + return False + return policyengine_version != "latest" and ( + policyengine_version in routes or routes.get("latest") == policyengine_version + ) + + def spm_availability( country_id: str, policyengine_version: str, versions: object -) -> dict[str, Any]: +) -> dict[str, Any] | None: """Return ``{available, settings_schema, defaults}`` for one catalog version. Only a US version whose PolicyEngine.py version has a canonical capability - with pinned defaults (forecast artifact and scenario) offers SPM; anything - else is ``{"available": False}``. + with pinned defaults (forecast artifact and scenario) offers SPM; any other + country, and a US version the service serves without one, is + ``{"available": False}``. A US version the service does not serve yet is + None: not known. """ - if country_id != SPM_COUNTRY_ID or not isinstance(versions, dict): + if country_id != SPM_COUNTRY_ID: return dict(UNAVAILABLE) + if not isinstance(versions, dict) or not _served(policyengine_version, versions): + return None capabilities = versions.get("spm_capabilities") capability = ( capabilities.get(policyengine_version) @@ -141,6 +159,8 @@ def record_published_spm_availability( ) -> None: """Record each published country's SPM availability for the catalog version. + A US version the service does not serve yet stays unrecorded. + Raise V2ConfigurationError without ``SIMULATION_ENTRYPOINT_URL``, and CatalogPublicationError when the service's versions cannot be read. """ @@ -149,11 +169,13 @@ def record_published_spm_availability( ) with engine.begin() as connection: for country in catalog.countries: - record_spm_availability( - connection, - model_name=country.model.name, - version=country.model_version.version, - availability=spm_availability( - country.country_id, catalog.policyengine_version, versions - ), + availability = spm_availability( + country.country_id, catalog.policyengine_version, versions ) + if availability is not None: + record_spm_availability( + connection, + model_name=country.model.name, + version=country.model_version.version, + availability=availability, + ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index a2dab29ee..4f00dbf07 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -140,7 +140,10 @@ def policyengine_version(self) -> str: A published catalog version never changes, so such a response stays valid until its code's response format changes; the ETag lets a client revalidate -it cheaply after that. +it cheaply after that. The one exception is a version's SPM availability in +``/v2/model-catalog``, recorded once after publication: until a later +publication run records it, a response may say the version offers no SPM +settings, and a reused response can keep saying so for up to this long. """ RUNNING_VERSION_MAX_AGE_SECONDS = 300 """How long a response for the running catalog version may be reused. diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index b07f3c6d8..e07e63833 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -2,6 +2,10 @@ from __future__ import annotations +from contextlib import contextmanager +from types import SimpleNamespace +from typing import TYPE_CHECKING + import httpx import pytest @@ -10,6 +14,9 @@ from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.spm import SPMSelection +if TYPE_CHECKING: + from collections.abc import Iterator + POLICYENGINE_VERSION = "6.10.0" DEFAULTS = { "forecast_content_sha256": "3d" * 32, @@ -19,8 +26,9 @@ "county_vintage": "2020", "as_of": None, } +ROUTES = {"latest": POLICYENGINE_VERSION, POLICYENGINE_VERSION: "policyengine-v6-10-0"} VERSIONS = { - "policyengine": {"latest": POLICYENGINE_VERSION}, + "policyengine": ROUTES, "spm_capabilities": { POLICYENGINE_VERSION: { "contract_version": "canonical-spm-v1", @@ -46,38 +54,38 @@ def test_a_us_version_with_a_certified_capability_offers_spm() -> None: ("country_id", "policyengine_version", "versions"), [ ("uk", POLICYENGINE_VERSION, VERSIONS), - ("us", "6.11.0", VERSIONS), - ("us", POLICYENGINE_VERSION, {}), - ("us", POLICYENGINE_VERSION, {"spm_capabilities": []}), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES}), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES, "spm_capabilities": []}), ( "us", POLICYENGINE_VERSION, { + "policyengine": ROUTES, "spm_capabilities": { POLICYENGINE_VERSION: { "contract_version": "canonical-spm-v0", "defaults": DEFAULTS, } - } + }, }, ), ( "us", POLICYENGINE_VERSION, { + "policyengine": ROUTES, "spm_capabilities": { POLICYENGINE_VERSION: { "contract_version": "canonical-spm-v1", "defaults": {**DEFAULTS, "scenario": None}, } - } + }, }, ), ], ids=[ "uk", - "unlisted-version", - "no-capabilities", + "no-capability", "malformed-capabilities", "other-contract", "unpinned-defaults", @@ -91,6 +99,71 @@ def test_anything_else_offers_no_spm( ) == {"available": False} +@pytest.mark.parametrize( + "versions", + [ + {**VERSIONS, "policyengine": {"latest": "6.11.0", "6.11.0": "app"}}, + {"spm_capabilities": VERSIONS["spm_capabilities"]}, + [], + ], + ids=["version-not-served", "no-route-map", "not-an-object"], +) +def test_a_us_version_the_service_does_not_serve_is_not_known_yet( + versions: object, +) -> None: + assert ( + spm_availability.spm_availability("us", POLICYENGINE_VERSION, versions) is None + ) + + +class RecordingEngine: + """An engine whose transactions are only counted.""" + + def __init__(self) -> None: + """Start with no transaction.""" + self.transactions = 0 + + @contextmanager + def begin(self) -> Iterator[str]: + """Count one transaction and hand out a placeholder connection.""" + self.transactions += 1 + yield "connection" + + +def test_publication_records_only_what_is_known( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[tuple[str, dict]] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + (model_name, dict(availability)) + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id=country_id, + model=SimpleNamespace(name=f"policyengine-{country_id}"), + model_version=SimpleNamespace(version="1.0.0"), + ) + for country_id in ("uk", "us") + ], + ) + not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=lambda _url: not_served, + ) + + assert recorded == [("policyengine-uk", {"available": False})] + + def test_the_versions_come_from_the_simulation_services_public_route() -> None: requests: list[httpx.Request] = [] From f6f5f0c22a29bfb73ccc453ade1c77e5743c70dd Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 22:27:48 +0400 Subject: [PATCH 100/118] Compare reforms and added variables with API v1 in the staging parity set The parity set gains a US household under a stored reform that also adds missing variables (compared with /calculate-full) and a UK household under a stored reform, so the policy conversion and the added variables are checked against API v1 too. The memory check's description now says what it asks for. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 9 ++- .../live_household_calculations.py | 70 +++++++++++++++++-- .../test_live_v2_household_calculations.py | 3 +- .../test_live_v2_household_parity.py | 42 +++++++++-- 4 files changed, 108 insertions(+), 16 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index c43665420..2a350cef6 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -317,9 +317,12 @@ database: within 10 seconds and finished within 300 seconds; and one household varied across 401 points of employment income, which must finish. The measured times appear in the job log as `HouseholdCalculationMeasurement` warnings. -- `test_live_v2_household_parity.py`: two US and two UK households, one of - each with axes, calculated through `/v2/household-calculations` and through - API v1's `/calculate`; every calculated value and warning must match. +- `test_live_v2_household_parity.py`: three US and three UK households, one + of each with axes and one of each under a reform stored through + `/v2/policies` (the US one also adding missing variables), calculated + through `/v2/household-calculations` and through API v1's `/calculate` + (`/calculate-full` when adding missing variables); every calculated value + and warning must match. `STAGING_HOUSEHOLD_PARALLEL_COUNT`, `STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, `STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS` and diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index b3dc8f936..a640e32a6 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -25,6 +25,8 @@ TRANSIENT_STATUS_CODES = {500, 502, 503, 504} FINISHED_STATUSES = {"succeeded", "failed"} YEAR = "2026" +REFORM_START = f"{YEAR}-01-01" +OPEN_END = "2100-12-31" COLLECTIONS = { "us": { "household": "households", @@ -43,7 +45,10 @@ class LiveHousehold: ``people`` and ``groups`` map variable names to values by period; a null value asks for the variable to be calculated. ``axes`` vary inputs across - points, as in a ``/calculate`` situation. + points, as in a ``/calculate`` situation. ``reform`` sets each named + parameter to a value from the start of ``YEAR`` with no end, through a + policy stored with ``/v2/policies``. ``add_missing_variables`` calculates + every variable the household lacks, as ``/calculate-full`` does. """ name: str @@ -51,6 +56,8 @@ class LiveHousehold: people: dict[str, dict[str, Any]] groups: dict[str, dict[str, Any]] = field(default_factory=dict) axes: list[list[dict[str, Any]]] | None = None + reform: dict[str, int | float] | None = None + add_missing_variables: bool = False def document(self) -> dict[str, Any]: """Return the household as ``/v2/households`` stores it.""" @@ -117,16 +124,59 @@ def store_household(client: httpx.Client, household: LiveHousehold) -> str: return response.json()["result"]["item"]["id"] +def _one_catalog_item( + client: httpx.Client, resource: str, country_id: str, **filters: str +) -> dict[str, Any]: + response = client.get( + f"/v2/{resource}", params={"country_id": country_id, "limit": 1, **filters} + ) + assert response.status_code == 200, response.text[:500] + (item,) = response.json()["result"]["items"] + return item + + +def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: + """Store the household's reform through ``/v2/policies``; return its ID.""" + if household.reform is None: + return None + country_id = household.country_id + model = _one_catalog_item(client, "tax-benefit-models", country_id) + parameter_values = [ + { + "parameter_id": _one_catalog_item( + client, "parameters", country_id, name=parameter_name + )["id"], + "value": value, + "start_date": f"{REFORM_START}T00:00:00Z", + } + for parameter_name, value in household.reform.items() + ] + response = client.post( + "/v2/policies", + params={"country_id": country_id}, + json={ + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "parameter_values": parameter_values, + }, + ) + assert response.status_code in {200, 201}, response.text[:500] + return response.json()["result"]["item"]["id"] + + def calculate( client: httpx.Client, household: LiveHousehold, household_id: str, + policy_id: str | None = None, ) -> FinishedCalculation: """Submit the stored household's calculation and poll until it finishes.""" started = time.monotonic() body: dict[str, Any] = { "country_id": household.country_id, "household_id": household_id, + "policy_id": policy_id, + "add_missing_variables": household.add_missing_variables, } if household.axes is not None: body["axes"] = household.axes @@ -156,11 +206,19 @@ def calculate( def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: - """Return API v1's ``/calculate`` answer for the same household.""" - response = client.post( - f"/{household.country_id}/calculate", - json={"household": household.situation()}, - ) + """Return API v1's answer for the same household and reform. + + ``/calculate-full`` answers when the household adds missing variables, + ``/calculate`` otherwise. + """ + route = "calculate-full" if household.add_missing_variables else "calculate" + request: dict[str, Any] = {"household": household.situation()} + if household.reform is not None: + request["policy"] = { + parameter_name: {f"{REFORM_START}.{OPEN_END}": value} + for parameter_name, value in household.reform.items() + } + response = client.post(f"/{household.country_id}/{route}", json=request) assert response.status_code == 200, response.text payload = response.json() assert payload["status"] == "ok", payload diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py index 278575621..c5a62fdb6 100644 --- a/tests/integration/test_live_v2_household_calculations.py +++ b/tests/integration/test_live_v2_household_calculations.py @@ -10,7 +10,8 @@ ``STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`` and finished within ``STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS``; - a memory check: a household varied across 401 points of employment income, - every variable calculated at each point, which must finish without failing. + with its net income calculated at each point, which must finish without + failing; a household function that runs out of memory fails it. The measured times are reported as a warning, so the job log keeps them. """ diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index f310add83..4b9cc7be9 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -1,10 +1,12 @@ """Live staging parity of v2 household calculations with API v1's ``/calculate``. -Each household is stored through ``/v2/households``, calculated through -``/v2/household-calculations`` (the v2 household function on Modal) and, as -the same situation, through API v1's in-process ``/calculate`` on the same -candidate. Every calculated value must match, axes included. A run-specific -input makes each household new, so no earlier calculation is reused. +Each household is stored through ``/v2/households`` (and its reform through +``/v2/policies``), calculated through ``/v2/household-calculations`` (the v2 +household function on Modal) and, as the same situation and policy, through +API v1's in-process ``/calculate`` (``/calculate-full`` when it adds missing +variables) on the same candidate. Every calculated value must match, axes, +reforms and added variables included. A run-specific input makes each +household new, so no earlier calculation is reused. """ from __future__ import annotations @@ -19,6 +21,7 @@ household_differences, probe_amount, store_household, + store_reform, ) CALCULATED = {YEAR: None} @@ -77,6 +80,19 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: ] ], ), + LiveHousehold( + name="us-single-reform-added-variables", + country_id="us", + people={ + "adult": { + "age": {YEAR: 29}, + "employment_income": {YEAR: 61_000 + run_amount}, + } + }, + groups={"household": {"state_name": {YEAR: "NY"}}}, + reform={"gov.irs.deductions.standard.amount.SINGLE": 20_000}, + add_missing_variables=True, + ), LiveHousehold( name="uk-single", country_id="uk", @@ -90,6 +106,19 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: }, groups={"household": {"household_net_income": CALCULATED}}, ), + LiveHousehold( + name="uk-single-reform", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 41}, + "employment_income": {YEAR: 36_000 + run_amount}, + "income_tax": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + reform={"gov.hmrc.income_tax.allowances.personal_allowance.amount": 15_000}, + ), LiveHousehold( name="uk-couple-child-axes", country_id="uk", @@ -136,8 +165,9 @@ def test_a_v2_household_calculation_matches_api_v1( if candidate.name == household_name ] household_id = store_household(api_client, household) + policy_id = store_reform(api_client, household) - finished = calculate(api_client, household, household_id) + finished = calculate(api_client, household, household_id, policy_id) v1_answer = calculate_with_v1(api_client, household) assert finished.item["status"] == "succeeded", finished.item["error"] From 0a3851f9554ebf48e951029083db3e184f01e356 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 23:38:56 +0400 Subject: [PATCH 101/118] Keep the household's key order to and from the simulation service The generated client's models wrote the household with its groups before its people and each group's members after its values, in the request and in the answer. The household function lists warnings and calculated values in the household's key order, as API v1 does, so the order could differ from API v1's. The submission and the answers are now checked against the generated models but sent and returned as built and decoded. Co-Authored-By: Claude Opus 5.5 --- .../simulation_client.py | 96 +++++++++++-------- ...household_calculation_simulation_client.py | 23 +++++ 2 files changed, 77 insertions(+), 42 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 498d23614..4d482adba 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -1,11 +1,17 @@ """The simulation service's household calculation routes, through its client. Calls go to the Cloud Run Simulation Entrypoint (``SIMULATION_ENTRYPOINT_URL``), -the only simulation service with these routes, with the gateway credentials -the Public API already uses (``GATEWAY_AUTH_*``) and the request's -correlation headers. Answers are read through the client's generated models: -a 400 is a refusal, whose reason the service gives; a 422 lists what the -service finds invalid in the household or its options; anything else, +the only simulation service with these routes, through the generated client's +HTTP client, with the gateway credentials the Public API already uses +(``GATEWAY_AUTH_*``) and the request's correlation headers. + +The submission is checked against the generated submission model and sent as +built, and each answer is checked against the generated model for its status +and returned as decoded, both in their original key order: the household +function lists warnings and calculated values in the household's key order, +as API v1 does, and the generated models write their keys in their own +order. A 400 is a refusal, whose reason the service gives; a 422 lists what +the service finds invalid in the household or its options; anything else, including a 503, no answer, an answer of another shape, a gateway token that cannot be fetched or missing configuration, means the service is unavailable. @@ -16,14 +22,11 @@ from __future__ import annotations from functools import lru_cache -from typing import TYPE_CHECKING, Any, TypeVar +from http import HTTPStatus +from typing import TYPE_CHECKING, Any import httpx from policyengine_api_simulation_client import AuthenticatedClient -from policyengine_api_simulation_client.api.default import ( - get_household_calculation, - submit_household_calculation, -) from policyengine_api_simulation_client.models.household_calculation_response import ( HouseholdCalculationResponse, ) @@ -62,31 +65,43 @@ from collections.abc import Callable from uuid import UUID - from policyengine_api_simulation_client.types import Response - -ParsedT = TypeVar("ParsedT") +CALCULATIONS_PATH = "/v2/household-calculations" REQUEST_TIMEOUT_SECONDS = 30.0 CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" -def _call(send: Callable[[], Response[ParsedT]]) -> Response[ParsedT]: - """Send one request; raise SimulationServiceUnavailableError if it cannot be read. - - The generated routes decode the body for each status they document, so a - body that is not JSON or not of the documented shape raises while - decoding. - """ +def _send(send: Callable[[], httpx.Response]) -> httpx.Response: + """Send one request; raise SimulationServiceUnavailableError if none answers.""" try: return send() except (httpx.HTTPError, GatewayAuthError) as error: msg = "The simulation service could not be reached." raise SimulationServiceUnavailableError(msg) from error - except (ValueError, KeyError, TypeError) as error: + + +def _decoded(response: httpx.Response, model: Any) -> tuple[Any, Any]: # noqa: ANN401 # a generated model class and its instance + """Return the decoded body and the generated model read from it. + + Raise SimulationServiceUnavailableError when the body is not JSON or not + of the model's shape. + """ + try: + body = response.json() + return body, model.from_dict(body) + except (ValueError, KeyError, TypeError, AttributeError) as error: msg = "The simulation service's answer could not be read." raise SimulationServiceUnavailableError(msg) from error +def _calculation(response: httpx.Response) -> dict[str, Any]: + body, _calculation_model = _decoded(response, HouseholdCalculationResponse) + if not isinstance(body, dict): + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) + return body + + def _invalid_request_reason(error: HTTPValidationError) -> str: """Return each invalid field and why, without the request's ``body`` prefix.""" reasons = [] @@ -165,32 +180,29 @@ def from_environment(cls) -> SimulationHouseholdClient: def submit(self, submission: dict[str, Any]) -> dict[str, Any]: """Submit a calculation and return the service's calculation answer.""" - body = HouseholdCalculationSubmission.from_dict(submission) - response = _call( - lambda: submit_household_calculation.sync_detailed( - client=self._client, body=body - ) - ) - answer = response.parsed - if isinstance(answer, HouseholdCalculationResponse): - return answer.to_dict() - if isinstance(answer, HTTPValidationError): - raise SimulationServiceInvalidRequestError(_invalid_request_reason(answer)) - if isinstance(answer, HouseholdRouteError) and response.status_code == 400: # noqa: PLR2004 # the service's refusal status - raise SimulationServiceRefusedError(answer.detail) + HouseholdCalculationSubmission.from_dict(submission) + http_client = self._client.get_httpx_client() + response = _send(lambda: http_client.post(CALCULATIONS_PATH, json=submission)) + if response.status_code == HTTPStatus.ACCEPTED: + return _calculation(response) + if response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: + _body, invalid = _decoded(response, HTTPValidationError) + raise SimulationServiceInvalidRequestError(_invalid_request_reason(invalid)) + if response.status_code == HTTPStatus.BAD_REQUEST: + _body, refusal = _decoded(response, HouseholdRouteError) + raise SimulationServiceRefusedError(refusal.detail) raise _unavailable(response.status_code) def read(self, calculation_id: UUID) -> dict[str, Any] | None: """Return the service's calculation answer, or None when it has none.""" - response = _call( - lambda: get_household_calculation.sync_detailed( - calculation_id, client=self._client - ) + http_client = self._client.get_httpx_client() + response = _send( + lambda: http_client.get(f"{CALCULATIONS_PATH}/{calculation_id}") ) - answer = response.parsed - if isinstance(answer, HouseholdCalculationResponse): - return answer.to_dict() - if isinstance(answer, HouseholdRouteError) and response.status_code == 404: # noqa: PLR2004 # the service's unknown-calculation status + if response.status_code == HTTPStatus.OK: + return _calculation(response) + if response.status_code == HTTPStatus.NOT_FOUND: + _decoded(response, HouseholdRouteError) return None raise _unavailable(response.status_code) diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py index e99eb96ca..747b629e9 100644 --- a/tests/unit/v2/test_household_calculation_simulation_client.py +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -237,3 +237,26 @@ def auth_flow(self, request: httpx.Request): # httpx's generator protocol client.submit(SUBMISSION) with pytest.raises(SimulationServiceUnavailableError): client.read(CALCULATION_ID) + + +def test_the_household_keeps_its_key_order_both_ways() -> None: + requests: list[httpx.Request] = [] + answer = succeeded_calculation() + client = client_answering(200, answer, requests) + submit_client = client_answering(202, answer, requests) + + submitted = submit_client.submit(SUBMISSION) + read = client.read(CALCULATION_ID) + + sent_household = json.loads(requests[0].content)["household"] + assert list(sent_household) == list(SITUATION) + assert list(sent_household["households"]["household-1"]) == list( + SITUATION["households"]["household-1"] + ) + for calculation in (submitted, read): + assert calculation is not None + returned_household = calculation["result"]["household"] + assert list(returned_household) == list(answer["result"]["household"]) + assert list(returned_household["households"]["household-1"]) == list( + answer["result"]["household"]["households"]["household-1"] + ) From d82e798067dd27702fd37619d44b9d902500ac51 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 23:41:40 +0400 Subject: [PATCH 102/118] Build the simulation service client only once a request reaches it The default service factory built the simulation client before any check, so with the simulation service unconfigured a country without household calculations, or a household that does not exist, got 503 instead of 422 or 404. The client is now built only when a submission or read is sent. Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/dependencies.py | 2 +- .../v2/household_calculations/services.py | 12 ++++-- .../v2/test_household_calculation_routes.py | 2 +- .../v2/test_household_calculation_service.py | 40 ++++++++++++++++++- 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 9521876ae..2b9466f5a 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -899,7 +899,7 @@ def _default_v2_household_calculation_service_factory() -> ( households=_default_v2_household_service_factory(), policies=_default_v2_policy_service_factory(), catalog_reader_factory=_default_v2_metadata_reader_factory, - simulation=process_simulation_household_client(), + simulation_factory=process_simulation_household_client, ) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 4c5d7fca8..9250d0606 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -54,18 +54,22 @@ def __init__( households: HouseholdSource, policies: PolicySource, catalog_reader_factory: Callable[[], CatalogSource], - simulation: SimulationHouseholdCalculations, + simulation_factory: Callable[[], SimulationHouseholdCalculations], ) -> None: """Read through these services for one request. ``catalog_reader_factory`` opens a catalog reader, with its own database session, only once a submission needs the catalog version. + ``simulation_factory`` returns the simulation service's client only + once a request is sent to it, so a request the Public API refuses + gets its own answer even when the simulation service is not + configured. """ self._countries = countries self._households = households self._policies = policies self._catalog_reader_factory = catalog_reader_factory - self._simulation = simulation + self._simulation_factory = simulation_factory def submit_calculation( self, calculation: HouseholdCalculationInput @@ -132,12 +136,12 @@ def submit_calculation( if spm is not None: submission["spm"] = spm return HouseholdCalculationRead.from_simulation( - self._simulation.submit(submission) + self._simulation_factory().submit(submission) ) def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: """Return the calculation, or raise HouseholdCalculationNotFoundError.""" - calculation = self._simulation.read(calculation_id) + calculation = self._simulation_factory().read(calculation_id) if calculation is None: msg = f"Household calculation {calculation_id} not found." raise HouseholdCalculationNotFoundError(msg) diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index e45e6cd62..56ad0b595 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -52,7 +52,7 @@ def __init__(self, countries: FakeCountries | None = None) -> None: households=self.households, policies=FakePolicies(), catalog_reader_factory=FakeCatalogs, - simulation=self.simulation, + simulation_factory=lambda: self.simulation, ) dependencies = NativeRouteDependencies( readiness_probe=lambda: True, diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index 135ad18ae..aee458b1d 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -59,7 +59,7 @@ def __init__(self, countries: FakeCountries | None = None) -> None: households=self.households, policies=self.policies, catalog_reader_factory=self.open_catalog_reader, - simulation=self.simulation, + simulation_factory=lambda: self.simulation, ) def open_catalog_reader(self) -> FakeCatalogs: @@ -271,3 +271,41 @@ def test_a_refused_submission_and_a_read_open_no_catalog_reader() -> None: parts.service.get_calculation(CALCULATION_ID) assert parts.catalog_readers == [] + + +def _unconfigured_simulation() -> FakeSimulation: + message = "The simulation service is not configured." + raise SimulationServiceUnavailableError(message) + + +def test_requests_refused_by_the_public_api_need_no_simulation_service() -> None: + parts = Parts( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + unconfigured = V2HouseholdCalculationService( + countries=parts.countries, + households=parts.households, + policies=parts.policies, + catalog_reader_factory=parts.open_catalog_reader, + simulation_factory=_unconfigured_simulation, + ) + available_country = V2HouseholdCalculationService( + countries=FakeCountries(), + households=parts.households, + policies=parts.policies, + catalog_reader_factory=parts.open_catalog_reader, + simulation_factory=_unconfigured_simulation, + ) + + with pytest.raises(HouseholdCalculationCountryError): + unconfigured.submit_calculation(calculation_input()) + with pytest.raises(HouseholdNotFoundError): + available_country.submit_calculation(calculation_input(household_id=uuid4())) + with pytest.raises(SimulationServiceUnavailableError): + available_country.submit_calculation(calculation_input()) From 7ed16f2e3fb2f09dd6cc07e88aee84f0464d0b70 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 23:44:32 +0400 Subject: [PATCH 103/118] Vary the household's default year when an axis names no period An axis without a period varied the country model's default period, the current year, rather than the year the household's values name. Such an axis is now sent with the household's default year as its period. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 3 +- .../v2/household_calculations/services.py | 34 +++++++++++++------ .../v2/household_calculations/types.py | 2 +- .../v2/test_household_calculation_routes.py | 4 ++- .../v2/test_household_calculation_service.py | 14 +++++++- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 2a350cef6..f4e5ea674 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -262,7 +262,8 @@ for immediate rollback. `POST /v2/household-calculations` calculates a household stored through `/v2/households`, under a policy stored through `/v2/policies` or, without `policy_id`, under current law, for the household's `default_year`. The body -may add `axes` and `add_missing_variables`. An identical calculation that +may add `axes` (an axis without a `period` varies the household's +`default_year`) and `add_missing_variables`. An identical calculation that succeeded in the last 28 days is returned again; callers cannot force a new one. The Public API checks the country's `countries` row allows household calculations diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 9250d0606..228f7461b 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -44,6 +44,28 @@ ) +def _axes_in_year( + calculation: HouseholdCalculationInput, year: int +) -> list[list[dict[str, Any]]] | None: + """Return the axes as sent, each without a period varying ``year``. + + The country model would otherwise vary its own default period, the + current year, rather than the year the household's values name. + """ + if calculation.axes is None: + return None + sent_axes = [] + for group in calculation.axes: + sent_group = [] + for axis in group: + sent_axis = axis.model_dump(mode="json", exclude_unset=True) + if sent_axis.get("period") is None: + sent_axis["period"] = str(year) + sent_group.append(sent_axis) + sent_axes.append(sent_group) + return sent_axes + + class V2HouseholdCalculationService: """Submit household calculations to the simulation service and read them.""" @@ -114,17 +136,7 @@ def submit_calculation( "parameters": parameters, }, "options": { - "axes": ( - None - if calculation.axes is None - else [ - [ - axis.model_dump(mode="json", exclude_unset=True) - for axis in group - ] - for group in calculation.axes - ] - ), + "axes": _axes_in_year(calculation, year), "add_missing_variables": calculation.add_missing_variables, }, # A calculation is reused for 28 days after it succeeded; callers diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index 78d50f8ed..f8e33e626 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -39,7 +39,7 @@ class HouseholdCalculationAxis(StrictCalculationModel): """One input varied across points. The variable ``name`` takes ``count`` evenly spaced values from ``min`` to - ``max`` in ``period`` (the household's default period when omitted), for + ``max`` in ``period`` (the household's ``default_year`` when omitted), for the entity member at position ``index`` (the first when omitted). """ diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 56ad0b595..aca622d83 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -106,7 +106,9 @@ def test_a_calculation_with_axes_sends_them() -> None: response = app.client.post(PATH, json=body(axes=axes)) assert response.status_code == 202 - assert app.simulation.submissions[0]["options"]["axes"] == axes + assert app.simulation.submissions[0]["options"]["axes"] == [ + [{**axes[0][0], "period": "2026"}] + ] def test_a_succeeded_us_calculation_reads_back_its_spm_choice() -> None: diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index aee458b1d..246e119e1 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -104,7 +104,10 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: } }, }, - "options": {"axes": AXES, "add_missing_variables": True}, + "options": { + "axes": [[{**AXES[0][0], "period": "2026"}]], + "add_missing_variables": True, + }, "force": False, "trigger": "initial", } @@ -309,3 +312,12 @@ def test_requests_refused_by_the_public_api_need_no_simulation_service() -> None available_country.submit_calculation(calculation_input(household_id=uuid4())) with pytest.raises(SimulationServiceUnavailableError): available_country.submit_calculation(calculation_input()) + + +def test_an_axis_keeps_its_own_period() -> None: + parts = Parts() + axes = [[{**AXES[0][0], "period": "2025-03"}]] + + parts.service.submit_calculation(calculation_input(axes=axes)) + + assert parts.simulation.submissions[0]["options"]["axes"] == axes From 538499de164b8a09762e8c44ec7ac6e2c619dfca Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Tue, 27 Oct 2026 23:57:07 +0400 Subject: [PATCH 104/118] Register the household calculation routes with the household route group /v2/household-calculations requests were logged with the route group "unknown"; they now belong to the household group. The contract registry records both routes as a typed v2 workflow, and the generated migration contract documents include them. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/migration-contracts.md | 16 ++++-- docs/generated/migration_contracts.json | 51 ++++++++++++++++++- policyengine_api/migration_registry.py | 1 + tests/contract/registry.py | 44 ++++++++++++++++ .../test_app_v2_workflow_contracts.py | 2 + .../unit/test_migration_contract_artifacts.py | 5 +- tests/unit/test_migration_flags.py | 5 ++ 7 files changed, 117 insertions(+), 7 deletions(-) diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index 31614d7d4..0398c7a94 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 17 | -| request count | 93 | +| workflow count | 18 | +| request count | 95 | | db entity count | 6 | | sim flow count | 3 | @@ -20,7 +20,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | -| `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | +| `household` | `household`, `households`, `household-calculations`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations`, `user-simulations` | `simulation` | `economy` | | `report` | `report`, `user-reports`, `shared-reports` | `report` | `report` | @@ -100,6 +100,16 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/households/{household_id}?country_id=us` | 200 | `household` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.default_year`, `result.item.household_data`, `result.item.created_at`, `result.item.updated_at` | | `GET` | `/v2/households?country_id=us` | 200 | `household` | none | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +### `household_calculations_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Household calculations on v2 + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `POST` | `/v2/household-calculations` | 202 | `household` | none | `status`, `message`, `result.calculation_id`, `result.country_id`, `result.household_id`, `result.policy_id`, `result.year`, `result.catalog_version_id`, `result.status`, `result.created_at` | +| `GET` | `/v2/household-calculations/{calculation_id}` | 200 | `household` | none | `status`, `message`, `result.calculation_id`, `result.status`, `result.completed_at`, `result.result.household`, `result.result.warnings`, `result.result.spm_config`, `result.result.spm_provenance`, `result.result.policyengine_version`, `result.error` | + ### `user_household_associations_v2` - Current contract: `typed_v2_resources` diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 928b956b3..22de0e6a6 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 93, + "request_count": 95, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 17 + "workflow_count": 18 }, "route_groups": [ { @@ -61,6 +61,7 @@ "path_segments": [ "household", "households", + "household-calculations", "user-household", "user-households", "calculate", @@ -538,6 +539,52 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Household calculations on v2", + "name": "household_calculations_v2", + "requests": [ + { + "expected_status": 202, + "headers": [], + "method": "POST", + "path": "/v2/household-calculations", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.calculation_id", + "result.country_id", + "result.household_id", + "result.policy_id", + "result.year", + "result.catalog_version_id", + "result.status", + "result.created_at" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/household-calculations/{calculation_id}", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.calculation_id", + "result.status", + "result.completed_at", + "result.result.household", + "result.result.warnings", + "result.result.spm_config", + "result.result.spm_provenance", + "result.result.policyengine_version", + "result.error" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "PR 12A: User Login and v1-Anchored User Associations", diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index 74bcba926..f46c5b8aa 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -60,6 +60,7 @@ class RouteGroupConfig: path_segments=( "household", "households", + "household-calculations", "user-household", "user-households", "calculate", diff --git a/tests/contract/registry.py b/tests/contract/registry.py index ac187802d..987c9dee8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -355,6 +355,50 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="household_calculations_v2", + current_contract="typed_v2_resources", + future_owner_pr="Household calculations on v2", + requests=( + ContractRequest( + method="POST", + path="/v2/household-calculations", + expected_status=202, + stable_response_fields=( + "status", + "message", + "result.calculation_id", + "result.country_id", + "result.household_id", + "result.policy_id", + "result.year", + "result.catalog_version_id", + "result.status", + "result.created_at", + ), + route_group="household", + ), + ContractRequest( + method="GET", + path="/v2/household-calculations/{calculation_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.calculation_id", + "result.status", + "result.completed_at", + "result.result.household", + "result.result.warnings", + "result.result.spm_config", + "result.result.spm_provenance", + "result.result.policyengine_version", + "result.error", + ), + route_group="household", + ), + ), + ), WorkflowContract( name="user_household_associations_v2", current_contract="typed_v2_resources", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index f74640678..a4580d9c7 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -14,6 +14,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", @@ -37,6 +38,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "policy_resources_v2", "user_policy_associations_v2", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index 026d0950f..90eec1780 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -13,8 +13,8 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 17, - "request_count": 93, + "workflow_count": 18, + "request_count": 95, "db_entity_count": 6, "sim_flow_count": 3, } @@ -25,6 +25,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index e110da879..4ca5dd68f 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -181,6 +181,11 @@ def test_explicit_migration_context_rejects_invalid_database_sources( "policy", ), ("/v2/user-households", "household"), + ("/v2/household-calculations", "household"), + ( + "/v2/household-calculations/00000000-0000-0000-0000-000000000001", + "household", + ), ("/v2/user-simulations", "simulation"), ("/v2/user-reports/00000000-0000-0000-0000-000000000001", "report"), ("/v2/me", "user_profile"), From bdbab60e44cd758fe14d3ebb0316ab3d92dde3e6 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:00:27 +0400 Subject: [PATCH 105/118] Check the simulation service URL first, and never fail on its versions Catalog publication now stops before extraction when SIMULATION_ENTRYPOINT_URL is unset, instead of after the catalog is written. When the simulation service's /versions answer cannot be read, publication logs a warning and leaves SPM availability unrecorded for the next run, so a slow or unreachable service no longer stops a release. The cache note and the stage 9 guide say that an instance keeps an earlier answer until it restarts. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-9-v2-metadata.md | 16 +++++---- .../data/v2/catalog/initialization.py | 5 ++- .../data/v2/catalog/spm_availability.py | 26 +++++++++----- .../fastapi_routes/v2/metadata/common.py | 9 +++-- tests/unit/v2/test_catalog_initialization.py | 16 +++++++++ .../unit/v2/test_catalog_spm_availability.py | 36 +++++++++++++++++++ 6 files changed, 89 insertions(+), 19 deletions(-) diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index ac447d1b1..de87f336f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -44,8 +44,10 @@ simulation service's public `GET /versions`: a US version offers SPM when the service lists a canonical capability with pinned defaults for its PolicyEngine.py version, and other countries never do. A US version the service does not serve yet stays unrecorded until a later publication run, and -a version whose availability is not recorded yet offers none; a response read -before then, and reused for up to a day, keeps saying so. The result names the +a version whose availability is not recorded yet offers none. An API instance +that read the version before it was recorded keeps that answer until it +restarts, normally at the deployment after the next publication run, and a +client may reuse it for up to a day after that. The result names the selected PolicyEngine.py version, so a client that starts without a version learns the running one and requests every other part for that exact version. The query uses the shared `CatalogQuery` model: unknown and repeated @@ -198,11 +200,11 @@ must execute in this order: `HUGGING_FACE_TOKEN` (from the organization secret `PE_UK_PRIVATE_HF_READ_TOKEN`) and `SIMULATION_ENTRYPOINT_URL` (from the environment secret of the same name), then run - `uv run python scripts/publish_v2_metadata_catalog.py`. After publishing, the - command reads the simulation service's public `GET /versions` to record each - version's SPM availability, and fails if the URL is unset or the answer - cannot be read; the catalog itself is already published by then, and the - next run records the availability. Before publishing a + `uv run python scripts/publish_v2_metadata_catalog.py`. The command stops + before extraction if `SIMULATION_ENTRYPOINT_URL` is unset. After publishing, + it reads the simulation service's public `GET /versions` to record each + version's SPM availability; if the answer cannot be read, it logs a warning, + records none, and the next run records them. Before publishing a catalog version that is not stored yet, the command reads the certified data artifacts its regions' filters use and stops if one cannot select a region. It stops before reading anything if a UK version needs checking and diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 036b055d9..6c974f121 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -30,6 +30,7 @@ ) from policyengine_api.data.v2.catalog.spm_availability import ( record_published_spm_availability, + simulation_entrypoint_url, ) from policyengine_api.data.v2.settings import ( V2ConfigurationError, @@ -130,10 +131,12 @@ def initialize_catalog( ) -> PublicationEvidence: """Load only row-write settings, extract fully, verify, then publish explicitly. - After publishing, record each version's SPM availability. + After publishing, record each version's SPM availability. The simulation + service's URL, which that needs, is checked before anything else. """ selected_steps = steps or PublicationSteps() settings = load_v2_data_write_database_settings(environ) + simulation_entrypoint_url(os.environ if environ is None else environ) catalog = extractor() engine = engine_builder(settings) try: diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index 2b5f28a2b..ab9c3f1c4 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -13,11 +13,13 @@ the service does not serve yet (its ``policyengine`` routes do not name it) is left unrecorded rather than recorded as unavailable, and the next publication run tries again; ``/v2/model-catalog`` offers no SPM settings for -it meanwhile. +it meanwhile. So is every version when the service's versions cannot be +read: recording never fails the publication. """ from __future__ import annotations +import logging import os from typing import TYPE_CHECKING, Any @@ -39,6 +41,7 @@ from policyengine_api.data.v2.catalog.records import NormalizedCatalog +LOGGER = logging.getLogger(__name__) SIMULATION_ENTRYPOINT_URL = "SIMULATION_ENTRYPOINT_URL" SPM_COUNTRY_ID = "us" SPM_CONTRACT_VERSION = "canonical-spm-v1" @@ -159,14 +162,21 @@ def record_published_spm_availability( ) -> None: """Record each published country's SPM availability for the catalog version. - A US version the service does not serve yet stays unrecorded. - - Raise V2ConfigurationError without ``SIMULATION_ENTRYPOINT_URL``, and - CatalogPublicationError when the service's versions cannot be read. + A US version the service does not serve yet stays unrecorded, and so does + every version when the service's versions cannot be read; the next + publication run tries again. Raise V2ConfigurationError without + ``SIMULATION_ENTRYPOINT_URL``. """ - versions = fetch_versions( - simulation_entrypoint_url(os.environ if environ is None else environ) - ) + url = simulation_entrypoint_url(os.environ if environ is None else environ) + try: + versions = fetch_versions(url) + except CatalogPublicationError as error: + LOGGER.warning( + "SPM availability was not recorded; the next publication run " + "tries again: %s", + error, + ) + return with engine.begin() as connection: for country in catalog.countries: availability = spm_availability( diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index 4f00dbf07..5d36bf50d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -141,9 +141,12 @@ def policyengine_version(self) -> str: A published catalog version never changes, so such a response stays valid until its code's response format changes; the ETag lets a client revalidate it cheaply after that. The one exception is a version's SPM availability in -``/v2/model-catalog``, recorded once after publication: until a later -publication run records it, a response may say the version offers no SPM -settings, and a reused response can keep saying so for up to this long. +``/v2/model-catalog``, recorded once after publication: until a publication +run records it, a response says the version offers no SPM settings. A +process keeps the selected catalog rows and built responses it has read for +its whole life, so it keeps saying so until it restarts (normally at the next +deployment, which follows the next publication run), and a client may reuse +such a response for up to this long after that. """ RUNNING_VERSION_MAX_AGE_SECONDS = 300 """How long a response for the running catalog version may be reused. diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index aee1342b5..22f01b72b 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -34,6 +34,7 @@ V2_SUPABASE_ENVIRONMENT: "test-foundation", "V2_RUNTIME_DATABASE_URL": "invalid-runtime-value", "V2_MIGRATION_DATABASE_URL": "invalid-migration-value", + "SIMULATION_ENTRYPOINT_URL": "https://simulation.example", } @@ -308,3 +309,18 @@ def verify_without_token(): assert "HUGGING_FACE_TOKEN must be set to read the certified uk data" in ( failure.err ) + + +def test_a_missing_simulation_service_url_stops_before_extraction() -> None: + def refuse_extraction(): + msg = "extraction must not start" + raise AssertionError(msg) + + environment = { + name: value + for name, value in ENVIRONMENT.items() + if name != "SIMULATION_ENTRYPOINT_URL" + } + + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + initialization.initialize_catalog(environment, extractor=refuse_extraction) diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py index e07e63833..c8cd1ac1b 100644 --- a/tests/unit/v2/test_catalog_spm_availability.py +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -205,3 +205,39 @@ def test_recording_needs_the_simulation_service_url() -> None: ) == "https://simulation.example" ) + + +def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[str] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + model_name + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id="us", + model=SimpleNamespace(name="policyengine-us"), + model_version=SimpleNamespace(version="1.0.0"), + ) + ], + ) + + def unreadable(_url: str) -> object: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=unreadable, + ) + + assert recorded == [] From b54f7eff9ed0506cde77b7cc78cc3e3fef7f0f8c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:01:29 +0400 Subject: [PATCH 106/118] Send a live check's calculation again after a 5xx answer The simulation service answers 503 with Retry-After while its database or version manifest cannot be read. The live staging checks retried such answers when polling but not when submitting, so one short outage failed the job that staging promotion waits on. Submissions are now sent again, after the Retry-After wait, until the calculation's deadline. Co-Authored-By: Claude Opus 5.5 --- .../live_household_calculations.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index a640e32a6..58d0cf113 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -164,6 +164,29 @@ def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: return response.json()["result"]["item"]["id"] +def _retry_wait_seconds(response: httpx.Response) -> float: + retry_after = response.headers.get("Retry-After", "") + return float(retry_after) if retry_after.isdigit() else POLL_INTERVAL_SECONDS + + +def _submit( + client: httpx.Client, body: dict[str, Any], deadline: float +) -> httpx.Response: + """Post the calculation, sending it again after a 5xx answer until the deadline. + + The simulation service answers 503 with ``Retry-After`` while its database + or version manifest cannot be read; such an outage is not a failure of + the calculation. + """ + while True: + response = client.post(CALCULATION_PATH, json=body) + if response.status_code not in TRANSIENT_STATUS_CODES: + return response + if time.monotonic() >= deadline: + return response + time.sleep(_retry_wait_seconds(response)) + + def calculate( client: httpx.Client, household: LiveHousehold, @@ -180,11 +203,11 @@ def calculate( } if household.axes is not None: body["axes"] = household.axes - response = client.post(CALCULATION_PATH, json=body) + deadline = started + CALCULATION_TIMEOUT_SECONDS + response = _submit(client, body, deadline) submit_seconds = time.monotonic() - started assert response.status_code == 202, response.text item = response.json()["result"] - deadline = started + CALCULATION_TIMEOUT_SECONDS while item["status"] not in FINISHED_STATUSES: if time.monotonic() >= deadline: msg = ( From 4108a807ccd0ac699229ead50c37640b6061edee Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:21:09 +0400 Subject: [PATCH 107/118] Log household calculations with the sources and compute they use Requests to /v2/household-calculations were logged with the household route group's v1 database flags and its configured compute backend. They now log a Supabase read for a submission, no database read for a poll, no write, and the Cloud Run Simulation Entrypoint's v2 workers as the compute, whatever those flags select. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/migration_logging.py | 20 ++++++++ .../routes/test_migration_context_logging.py | 49 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 18f5ac7a9..be94d982c 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -61,6 +61,15 @@ V2_USER_RESOURCE_SEGMENTS = frozenset( {"user-policies", "user-households", "user-simulations", "user-reports", "me"} ) +# /v2/household-calculations reads the stored household, policy and catalog +# from Supabase when submitting, stores nothing itself, and always computes on +# the Cloud Run Simulation Entrypoint's v2 workers, whatever the household +# route group's flags select. +V2_HOUSEHOLD_CALCULATION_SEGMENT = "household-calculations" +V2_HOUSEHOLD_CALCULATION_COMPUTE = { + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", +} V2_RESOURCE_METHODS = frozenset({"GET", "POST", "PATCH", "DELETE"}) # A v2 resource path has at least the "v2" segment and the resource segment. V2_RESOURCE_PATH_MINIMUM_SEGMENTS = 2 @@ -247,6 +256,17 @@ def _migration_context( route_impl: RouteImplementation | None, ) -> dict[str, str | None]: route_group = infer_route_group(path) + if _v2_resource_segment(path) == V2_HOUSEHOLD_CALCULATION_SEGMENT: + context = get_migration_log_context( + route_group, + route_impl=route_impl, + explicit_db_sources=ExplicitDatabaseSources( + db_read_source="supabase" if method == "POST" else None + ), + ) + if "migration_flag_error" not in context: + context.update(V2_HOUSEHOLD_CALCULATION_COMPUTE) + return context is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) is_v2_supabase_resource = _is_v2_supabase_resource(method, path) uses_explicit_v2_source = is_v2_metadata_read or is_v2_supabase_resource diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 641c8c34b..2be8e2f4c 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -717,3 +717,52 @@ def test_compute_routes_in_the_same_groups_keep_their_compute_labels(): assert context["route_group"] == "simulation" assert context["sim_flow"] == "economy" assert context["sim_compute"] is not None + + +def test_household_calculations_log_their_actual_sources_and_compute(monkeypatch): + monkeypatch.setenv("DB_READ_HOUSEHOLD", "invalid-for-native-v2") + monkeypatch.setenv("DB_WRITE_HOUSEHOLD", "invalid-for-native-v2") + monkeypatch.setenv("SIM_ENTRYPOINT", "old_gateway_direct") + monkeypatch.delenv("SIM_COMPUTE_HOUSEHOLD", raising=False) + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + for method, path in ( + ("POST", "/v2/household-calculations"), + ( + "GET", + "/v2/household-calculations/00000000-0000-0000-0000-000000000001", + ), + ): + log_migration_request( + ServedRequest( + request_id=f"request-{method}", + method=method, + path=path, + status_code=202 if method == "POST" else 200, + started_at=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + + submit_context, read_context = ( + call.args[0]["migration"] for call in mock_logger.log_struct.call_args_list + ) + computed_on_v2 = { + "route_group": "household", + "route_impl": "fastapi_native", + "sim_flow": "household", + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", + } + assert submit_context == { + **submit_context, + **computed_on_v2, + "db_write": None, + "db_read": "supabase", + } + assert read_context == { + **read_context, + **computed_on_v2, + "db_write": None, + "db_read": None, + } From e3b5f8dc41f8a71e110840ca8dce38d7d0fadfce Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:24:15 +0400 Subject: [PATCH 108/118] Drop the unused generated client wrapper and correct two descriptions The household client held its HTTP client inside the generated AuthenticatedClient with an empty token but never called a generated route; it now keeps the HTTP client itself. Its description says the household's key order is the stored household's as the v2 database returns it. The SPM availability description and changelog entry say a version gets its record on the next publication run of its own PolicyEngine.py version, and that only an unreadable versions answer never fails publication. Co-Authored-By: Claude Opus 5.5 --- changelog.d/catalog-spm-availability.added.md | 2 +- .../data/v2/catalog/spm_availability.py | 10 +++-- .../simulation_client.py | 41 ++++++++----------- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/changelog.d/catalog-spm-availability.added.md b/changelog.d/catalog-spm-availability.added.md index bdf508050..c22bc0f26 100644 --- a/changelog.d/catalog-spm-availability.added.md +++ b/changelog.d/catalog-spm-availability.added.md @@ -1 +1 @@ -Each published catalog version now records whether it offers SPM settings, with their schema and certified defaults, from the simulation service's public versions answer; the catalog publication step reads `SIMULATION_ENTRYPOINT_URL`, and versions published earlier get it on the next publication run. +Each published catalog version now records whether it offers SPM settings, with their schema and certified defaults, from the simulation service's public versions answer; the catalog publication step reads `SIMULATION_ENTRYPOINT_URL`, and a version published earlier gets it on the next publication run of its PolicyEngine.py version. diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py index ab9c3f1c4..398cb2a41 100644 --- a/policyengine_api/data/v2/catalog/spm_availability.py +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -7,14 +7,16 @@ version offers SPM when the service reports a canonical capability for its PolicyEngine.py version; other countries never do. -Publication records it after the catalog itself, on each version that has -none yet, so versions published before it existed get it on the next run. A -bundle pins its SPM release, so a recorded value never changes. A US version +Publication records it after the catalog itself, for the installed +PolicyEngine.py version's catalog versions that have none yet, so a version +published before this existed gets it on the next publication run of that +PolicyEngine.py version; versions of other PolicyEngine.py versions keep none. +A bundle pins its SPM release, so a recorded value never changes. A US version the service does not serve yet (its ``policyengine`` routes do not name it) is left unrecorded rather than recorded as unavailable, and the next publication run tries again; ``/v2/model-catalog`` offers no SPM settings for it meanwhile. So is every version when the service's versions cannot be -read: recording never fails the publication. +read: an unreadable answer never fails the publication. """ from __future__ import annotations diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 4d482adba..dd2c4ee6a 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -1,16 +1,18 @@ -"""The simulation service's household calculation routes, through its client. +"""The simulation service's household calculation routes, checked by its models. Calls go to the Cloud Run Simulation Entrypoint (``SIMULATION_ENTRYPOINT_URL``), -the only simulation service with these routes, through the generated client's -HTTP client, with the gateway credentials the Public API already uses -(``GATEWAY_AUTH_*``) and the request's correlation headers. - -The submission is checked against the generated submission model and sent as -built, and each answer is checked against the generated model for its status -and returned as decoded, both in their original key order: the household -function lists warnings and calculated values in the household's key order, -as API v1 does, and the generated models write their keys in their own -order. A 400 is a refusal, whose reason the service gives; a 422 lists what +the only simulation service with these routes, over HTTP with the gateway +credentials the Public API already uses (``GATEWAY_AUTH_*``) and the request's +correlation headers. + +The submission is checked against the generated client's submission model and +sent as built, and each answer is checked against the generated model for its +status and returned as decoded, both in their original key order: the +household function lists warnings and calculated values in the household's +key order, and the generated models write their keys in an order of their +own. That order is the stored household's as the v2 database returns it, +which orders each record's keys itself, so it can differ from the order a +caller sent. A 400 is a refusal, whose reason the service gives; a 422 lists what the service finds invalid in the household or its options; anything else, including a 503, no answer, an answer of another shape, a gateway token that cannot be fetched or missing configuration, means the service is unavailable. @@ -26,7 +28,6 @@ from typing import TYPE_CHECKING, Any import httpx -from policyengine_api_simulation_client import AuthenticatedClient from policyengine_api_simulation_client.models.household_calculation_response import ( HouseholdCalculationResponse, ) @@ -148,13 +149,7 @@ def __init__( ) instrument_httpx(http_client, get_runtime()) self._base_url = base_url.rstrip("/") - # The generated routes take an AuthenticatedClient. Its fixed token is - # sent only from an httpx client it builds itself; the one set here - # authenticates through ``httpx_args`` (GatewayBearerAuth, which - # fetches and renews the gateway token), so the empty token is unused. - self._client = AuthenticatedClient( - base_url=base_url, token="" - ).set_httpx_client(http_client) + self._http_client = http_client @property def base_url(self) -> str: @@ -181,8 +176,9 @@ def from_environment(cls) -> SimulationHouseholdClient: def submit(self, submission: dict[str, Any]) -> dict[str, Any]: """Submit a calculation and return the service's calculation answer.""" HouseholdCalculationSubmission.from_dict(submission) - http_client = self._client.get_httpx_client() - response = _send(lambda: http_client.post(CALCULATIONS_PATH, json=submission)) + response = _send( + lambda: self._http_client.post(CALCULATIONS_PATH, json=submission) + ) if response.status_code == HTTPStatus.ACCEPTED: return _calculation(response) if response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: @@ -195,9 +191,8 @@ def submit(self, submission: dict[str, Any]) -> dict[str, Any]: def read(self, calculation_id: UUID) -> dict[str, Any] | None: """Return the service's calculation answer, or None when it has none.""" - http_client = self._client.get_httpx_client() response = _send( - lambda: http_client.get(f"{CALCULATIONS_PATH}/{calculation_id}") + lambda: self._http_client.get(f"{CALCULATIONS_PATH}/{calculation_id}") ) if response.status_code == HTTPStatus.OK: return _calculation(response) From 243c2d9cba7d02491bf9c7aa46ced5333accc684 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:25:54 +0400 Subject: [PATCH 109/118] Retry only short outages in the live checks, and compare warnings in any order The live household checks sent a request again after any 5xx answer until the calculation's 600-second deadline, so a lasting failure kept each check waiting until the job's time limit and lost the failing answer. They now retry 502, 503 and 504 for at most 90 seconds in a row and fail at once, with the answer's body, on anything else. The parity set compares warnings regardless of order, since the v2 database orders stored keys itself, and one US household now has a deprecated input so that both calculations warn. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 11 +++- .../live_household_calculations.py | 56 +++++++++++-------- .../test_live_v2_household_parity.py | 8 ++- 3 files changed, 49 insertions(+), 26 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index f4e5ea674..222d87e31 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -325,10 +325,17 @@ database: (`/calculate-full` when adding missing variables); every calculated value and warning must match. +Each request is sent again while it gets 502, 503 or 504 for at most 90 +seconds in a row; any other failure, or a longer outage, fails the check with +the answer's body. Warnings are compared regardless of order, because the v2 +database orders each stored record's keys itself; one US household has a +deprecated input, so both calculations warn. + `STAGING_HOUSEHOLD_PARALLEL_COUNT`, `STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, -`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS` and +`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS`, `STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS` (600 seconds per calculation) -change those limits; none is set in the workflow. The checks see neither the +and `STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS` change those limits; none is +set in the workflow. The checks see neither the number of Simulation Entrypoint instances that served the calculations nor the household functions' memory use: a calculation that runs out of memory fails, and so fails the check. Both modules fail until the staging Simulation diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 58d0cf113..580715718 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -10,9 +10,12 @@ import os import time from dataclasses import dataclass, field +from functools import partial from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from collections.abc import Callable + import httpx CALCULATION_PATH = "/v2/household-calculations" @@ -22,7 +25,15 @@ POLL_INTERVAL_SECONDS = float( os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") ) -TRANSIENT_STATUS_CODES = {500, 502, 503, 504} +# A 502, 503 or 504 can be a short outage, such as the simulation service +# failing to read its database or version manifest for a moment; each request +# is sent again while such answers last at most TRANSIENT_LIMIT_SECONDS in a +# row. Any other failure, and a longer outage, fails the check at once with +# the answer's body. +TRANSIENT_STATUS_CODES = {502, 503, 504} +TRANSIENT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") +) FINISHED_STATUSES = {"succeeded", "failed"} YEAR = "2026" REFORM_START = f"{YEAR}-01-01" @@ -49,6 +60,8 @@ class LiveHousehold: parameter to a value from the start of ``YEAR`` with no end, through a policy stored with ``/v2/policies``. ``add_missing_variables`` calculates every variable the household lacks, as ``/calculate-full`` does. + ``expects_warnings`` says the household has an input the country model + ignores with a warning. """ name: str @@ -58,6 +71,7 @@ class LiveHousehold: axes: list[list[dict[str, Any]]] | None = None reform: dict[str, int | float] | None = None add_missing_variables: bool = False + expects_warnings: bool = False def document(self) -> dict[str, Any]: """Return the household as ``/v2/households`` stores it.""" @@ -164,27 +178,21 @@ def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: return response.json()["result"]["item"]["id"] -def _retry_wait_seconds(response: httpx.Response) -> float: - retry_after = response.headers.get("Retry-After", "") - return float(retry_after) if retry_after.isdigit() else POLL_INTERVAL_SECONDS - - -def _submit( - client: httpx.Client, body: dict[str, Any], deadline: float -) -> httpx.Response: - """Post the calculation, sending it again after a 5xx answer until the deadline. - - The simulation service answers 503 with ``Retry-After`` while its database - or version manifest cannot be read; such an outage is not a failure of - the calculation. - """ +def _send_through_short_outages(send: Callable[[], httpx.Response]) -> httpx.Response: + """Send the request again while it gets 502, 503 or 504 for a short time.""" + outage_started: float | None = None while True: - response = client.post(CALCULATION_PATH, json=body) + response = send() if response.status_code not in TRANSIENT_STATUS_CODES: return response - if time.monotonic() >= deadline: - return response - time.sleep(_retry_wait_seconds(response)) + outage_started = outage_started or time.monotonic() + if time.monotonic() - outage_started >= TRANSIENT_LIMIT_SECONDS: + msg = ( + f"HTTP {response.status_code} for {TRANSIENT_LIMIT_SECONDS:.0f} s: " + f"{response.text[:500]}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) def calculate( @@ -204,7 +212,9 @@ def calculate( if household.axes is not None: body["axes"] = household.axes deadline = started + CALCULATION_TIMEOUT_SECONDS - response = _submit(client, body, deadline) + response = _send_through_short_outages( + lambda: client.post(CALCULATION_PATH, json=body) + ) submit_seconds = time.monotonic() - started assert response.status_code == 202, response.text item = response.json()["result"] @@ -216,9 +226,9 @@ def calculate( ) raise AssertionError(msg) time.sleep(POLL_INTERVAL_SECONDS) - polled = client.get(f"{CALCULATION_PATH}/{item['calculation_id']}") - if polled.status_code in TRANSIENT_STATUS_CODES: - continue + polled = _send_through_short_outages( + partial(client.get, f"{CALCULATION_PATH}/{item['calculation_id']}") + ) assert polled.status_code == 200, polled.text item = polled.json()["result"] return FinishedCalculation( diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index 4b9cc7be9..758482aa8 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -36,6 +36,8 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: "adult": { "age": {YEAR: 35}, "employment_income": {YEAR: 42_000 + run_amount}, + # Deprecated: dropped with a warning by both calculations. + "medical_out_of_pocket_expenses": {YEAR: 1_000}, "income_tax": CALCULATED, } }, @@ -47,6 +49,7 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, "spm_unit": {"snap": CALCULATED}, }, + expects_warnings=True, ), LiveHousehold( name="us-couple-two-children-texas-axes", @@ -174,7 +177,10 @@ def test_a_v2_household_calculation_matches_api_v1( result = finished.item["result"] differences = household_differences(v1_answer["result"], result["household"]) assert differences == [], "\n".join(differences[:50]) - assert result["warnings"] == v1_answer.get("warnings", []) + # The v2 database orders each stored record's keys itself, so warnings, + # listed in the household's key order, may come in another order. + assert sorted(result["warnings"]) == sorted(v1_answer.get("warnings", [])) + assert bool(result["warnings"]) == household.expects_warnings if household.country_id == "us": assert result["spm_config"] is not None assert result["spm_provenance"] is not None From ba761d42ba112257721bfd801ed31a94235f55ba Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:41:56 +0400 Subject: [PATCH 110/118] Log household calculations without checking flags they do not use An invalid SIM_ENTRYPOINT or SIM_COMPUTE_HOUSEHOLD, which household calculations never read, reduced their migration log record to a flag error. The record is now built from what the route uses, checking only the route implementation. The guide and the live check helper say which requests are sent again after a 502, 503 or 504, and that failures the Public API answers with 503 take about 90 seconds to fail. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 9 ++-- policyengine_api/migration_logging.py | 44 +++++++++++++------ .../simulation_client.py | 14 +++--- .../live_household_calculations.py | 12 ++--- .../routes/test_migration_context_logging.py | 27 ++++++++++++ 5 files changed, 77 insertions(+), 29 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 222d87e31..55f9b75c9 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -325,9 +325,12 @@ database: (`/calculate-full` when adding missing variables); every calculated value and warning must match. -Each request is sent again while it gets 502, 503 or 504 for at most 90 -seconds in a row; any other failure, or a longer outage, fails the check with -the answer's body. Warnings are compared regardless of order, because the v2 +A calculation's submission and each poll are sent again while they get 502, +503 or 504 for at most 90 seconds in a row; any other failure, or a longer +run of such answers, fails the check with the answer's body. The Public API +also answers 503 for failures that do not pass, such as a simulation service +without these routes, so those take about 90 seconds to fail. Every other +request in the checks fails at once. Warnings are compared regardless of order, because the v2 database orders each stored record's keys itself; one US household has a deprecated input, so both calculations warn. diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index be94d982c..e4b2e22ac 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -13,10 +13,13 @@ from policyengine_api.gcp_logging import logger from policyengine_api.migration_flags import ( ExplicitDatabaseSources, + MigrationContext, RouteImplementation, get_migration_log_context, + get_route_impl, infer_route_group, ) +from policyengine_api.migration_registry import ROUTE_GROUP_CONFIG_BY_NAME from policyengine_api.observability.identifiers import ( OBSERVABILITY_ID_HEADER, normalize_observability_id, @@ -64,12 +67,11 @@ # /v2/household-calculations reads the stored household, policy and catalog # from Supabase when submitting, stores nothing itself, and always computes on # the Cloud Run Simulation Entrypoint's v2 workers, whatever the household -# route group's flags select. +# route group's database, entrypoint and compute flags select; it reads none +# of them, so none of them is checked for its log record. V2_HOUSEHOLD_CALCULATION_SEGMENT = "household-calculations" -V2_HOUSEHOLD_CALCULATION_COMPUTE = { - "sim_entrypoint": "cloud_run_simulation_entrypoint", - "sim_compute": "v2_primary", -} +V2_HOUSEHOLD_CALCULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" +V2_HOUSEHOLD_CALCULATION_COMPUTE = "v2_primary" V2_RESOURCE_METHODS = frozenset({"GET", "POST", "PATCH", "DELETE"}) # A v2 resource path has at least the "v2" segment and the resource segment. V2_RESOURCE_PATH_MINIMUM_SEGMENTS = 2 @@ -249,6 +251,27 @@ def log_migration_request(served_request: ServedRequest) -> None: ) +def _household_calculation_context( + route_group: str, *, method: str, route_impl: RouteImplementation | None +) -> dict[str, str | None]: + """Return a household calculation's log record from what it actually uses.""" + try: + selected_route_impl = route_impl or get_route_impl(route_group) + except ValueError as error: + return {"route_group": route_group, "migration_flag_error": str(error)} + route_config = ROUTE_GROUP_CONFIG_BY_NAME[route_group] + return MigrationContext( + route_group=route_group, + route_impl=selected_route_impl, + db_entity=route_config.db_entity, + db_write=None, + db_read="supabase" if method == "POST" else None, + sim_flow=route_config.sim_flow, + sim_entrypoint=V2_HOUSEHOLD_CALCULATION_ENTRYPOINT, + sim_compute=V2_HOUSEHOLD_CALCULATION_COMPUTE, + ).to_log_dict() + + def _migration_context( *, method: str, @@ -257,16 +280,9 @@ def _migration_context( ) -> dict[str, str | None]: route_group = infer_route_group(path) if _v2_resource_segment(path) == V2_HOUSEHOLD_CALCULATION_SEGMENT: - context = get_migration_log_context( - route_group, - route_impl=route_impl, - explicit_db_sources=ExplicitDatabaseSources( - db_read_source="supabase" if method == "POST" else None - ), + return _household_calculation_context( + route_group, method=method, route_impl=route_impl ) - if "migration_flag_error" not in context: - context.update(V2_HOUSEHOLD_CALCULATION_COMPUTE) - return context is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) is_v2_supabase_resource = _is_v2_supabase_resource(method, path) uses_explicit_v2_source = is_v2_metadata_read or is_v2_supabase_resource diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index dd2c4ee6a..0f2d158fc 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -7,13 +7,15 @@ The submission is checked against the generated client's submission model and sent as built, and each answer is checked against the generated model for its -status and returned as decoded, both in their original key order: the +status and returned as decoded, both in their original key order, because +the generated models write their keys in an order of their own and the household function lists warnings and calculated values in the household's -key order, and the generated models write their keys in an order of their -own. That order is the stored household's as the v2 database returns it, -which orders each record's keys itself, so it can differ from the order a -caller sent. A 400 is a refusal, whose reason the service gives; a 422 lists what -the service finds invalid in the household or its options; anything else, +key order. The household's key order is the stored household's as the v2 +database returns it, which orders each record's keys itself, so it can +differ from the order a caller sent. + +A 400 is a refusal, whose reason the service gives; a 422 lists what the +service finds invalid in the household or its options; anything else, including a 503, no answer, an answer of another shape, a gateway token that cannot be fetched or missing configuration, means the service is unavailable. diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py index 580715718..960891496 100644 --- a/tests/fixtures/integration/live_household_calculations.py +++ b/tests/fixtures/integration/live_household_calculations.py @@ -26,10 +26,10 @@ os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") ) # A 502, 503 or 504 can be a short outage, such as the simulation service -# failing to read its database or version manifest for a moment; each request -# is sent again while such answers last at most TRANSIENT_LIMIT_SECONDS in a -# row. Any other failure, and a longer outage, fails the check at once with -# the answer's body. +# failing to read its database or version manifest for a moment; a +# calculation's submission and each poll are sent again while such answers +# last at most TRANSIENT_LIMIT_SECONDS in a row. Any other failure, and a +# longer run of such answers, fails the check with the answer's body. TRANSIENT_STATUS_CODES = {502, 503, 504} TRANSIENT_LIMIT_SECONDS = float( os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") @@ -60,8 +60,8 @@ class LiveHousehold: parameter to a value from the start of ``YEAR`` with no end, through a policy stored with ``/v2/policies``. ``add_missing_variables`` calculates every variable the household lacks, as ``/calculate-full`` does. - ``expects_warnings`` says the household has an input the country model - ignores with a warning. + ``expects_warnings`` says the household has a deprecated input, which the + calculation drops with a warning before the country model runs. """ name: str diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 2be8e2f4c..efc3d27bf 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -766,3 +766,30 @@ def test_household_calculations_log_their_actual_sources_and_compute(monkeypatch "db_write": None, "db_read": None, } + + +def test_household_calculations_ignore_compute_flags_they_do_not_use(monkeypatch): + monkeypatch.setenv("SIM_ENTRYPOINT", "invalid-unused") + monkeypatch.setenv("SIM_COMPUTE_HOUSEHOLD", "invalid-unused") + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + ServedRequest( + request_id="request-submit", + method="POST", + path="/v2/household-calculations", + status_code=202, + started_at=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + + context = mock_logger.log_struct.call_args.args[0]["migration"] + assert "migration_flag_error" not in context + assert context == { + **context, + "route_group": "household", + "db_read": "supabase", + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", + } From d695268a68b2b5ad2444eddae06b9d65169343c0 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 00:46:22 +0400 Subject: [PATCH 111/118] Type the simulation answer reader without a suppression The helper that reads a simulation service answer took a generated model class as Any, with an ANN401 suppression; it now takes the model's from_dict reader, typed by the model it returns. Co-Authored-By: Claude Opus 5.5 --- .../simulation_client.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 0f2d158fc..6045e8281 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -27,7 +27,7 @@ from functools import lru_cache from http import HTTPStatus -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypeVar import httpx from policyengine_api_simulation_client.models.household_calculation_response import ( @@ -65,7 +65,7 @@ ) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Mapping from uuid import UUID CALCULATIONS_PATH = "/v2/household-calculations" @@ -83,7 +83,12 @@ def _send(send: Callable[[], httpx.Response]) -> httpx.Response: raise SimulationServiceUnavailableError(msg) from error -def _decoded(response: httpx.Response, model: Any) -> tuple[Any, Any]: # noqa: ANN401 # a generated model class and its instance +ModelT = TypeVar("ModelT") + + +def _decoded( + response: httpx.Response, read_model: Callable[[Mapping[str, Any]], ModelT] +) -> tuple[object, ModelT]: """Return the decoded body and the generated model read from it. Raise SimulationServiceUnavailableError when the body is not JSON or not @@ -91,14 +96,16 @@ def _decoded(response: httpx.Response, model: Any) -> tuple[Any, Any]: # noqa: """ try: body = response.json() - return body, model.from_dict(body) + return body, read_model(body) except (ValueError, KeyError, TypeError, AttributeError) as error: msg = "The simulation service's answer could not be read." raise SimulationServiceUnavailableError(msg) from error def _calculation(response: httpx.Response) -> dict[str, Any]: - body, _calculation_model = _decoded(response, HouseholdCalculationResponse) + body, _calculation_model = _decoded( + response, HouseholdCalculationResponse.from_dict + ) if not isinstance(body, dict): msg = "The simulation service's answer could not be read." raise SimulationServiceUnavailableError(msg) @@ -184,10 +191,10 @@ def submit(self, submission: dict[str, Any]) -> dict[str, Any]: if response.status_code == HTTPStatus.ACCEPTED: return _calculation(response) if response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: - _body, invalid = _decoded(response, HTTPValidationError) + _body, invalid = _decoded(response, HTTPValidationError.from_dict) raise SimulationServiceInvalidRequestError(_invalid_request_reason(invalid)) if response.status_code == HTTPStatus.BAD_REQUEST: - _body, refusal = _decoded(response, HouseholdRouteError) + _body, refusal = _decoded(response, HouseholdRouteError.from_dict) raise SimulationServiceRefusedError(refusal.detail) raise _unavailable(response.status_code) @@ -199,7 +206,7 @@ def read(self, calculation_id: UUID) -> dict[str, Any] | None: if response.status_code == HTTPStatus.OK: return _calculation(response) if response.status_code == HTTPStatus.NOT_FOUND: - _decoded(response, HouseholdRouteError) + _decoded(response, HouseholdRouteError.from_dict) return None raise _unavailable(response.status_code) From 220cf1e9401bf5cb2b63824a7779064d6dafe81d Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 16:52:36 +0400 Subject: [PATCH 112/118] Send the simulation service household and policy IDs, not their values The simulation service now reads the stored household, the policy's parameter values and the catalog version from the Public API's tables itself. A household calculation's submission therefore carries only the country, the household's and policy's IDs, the options and the running catalog version's PolicyEngine.py version. The Public API still checks the country allows household calculations and that the household and policy exist in it, and fills an axis's missing period with the household's default year. The conversion of stored households and policies, which moved into the simulation service, is removed here. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 16 +- .../v2/household_calculations/services.py | 45 ++--- .../household_calculations/transformations.py | 83 --------- .../v2/test_household_calculation_service.py | 39 ++-- ...household_calculation_simulation_client.py | 21 +-- ...t_household_calculation_transformations.py | 172 ------------------ 6 files changed, 42 insertions(+), 334 deletions(-) delete mode 100644 policyengine_api/services/v2/household_calculations/transformations.py delete mode 100644 tests/unit/v2/test_household_calculation_transformations.py diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 55f9b75c9..0a33fa8bb 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -267,12 +267,16 @@ may add `axes` (an axis without a `period` varies the household's succeeded in the last 28 days is returned again; callers cannot force a new one. The Public API checks the country's `countries` row allows household calculations -(422 otherwise; a household-only country is accepted), reads the household and -policy through the same services as their routes (404 when either is not -stored in that country), selects the country's running catalog version, and -sends the household in the situation shape, its saved SPM choice and the -policy's parameter values to the simulation service, which cannot read these -tables. The answer is 202 with the calculation's `calculation_id` and +(422 otherwise; a household-only country is accepted), checks the household +and policy exist through the same services as their routes (404 when either +is not stored in that country), selects the country's running catalog +version, and sends the simulation service the household's and policy's IDs, +the options and that version's PolicyEngine.py version. The simulation +service reads the household, the policy's parameter values and the catalog +version from these tables itself, through its own `SELECT` grant, and +refuses a policy changing a parameter the catalog version lacks, which the +Public API answers with 503. The answer is 202 with the calculation's +`calculation_id` and `status`. Poll `GET /v2/household-calculations/{calculation_id}` until `status` is `succeeded`, with `result`, or `failed`, with `error`. A US result carries `spm_config` and `spm_provenance`. diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 228f7461b..41dcae0fa 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -1,13 +1,15 @@ """Calculate stored households through the simulation service, and read them. Submitting a calculation checks the country's ``countries`` row allows -household calculations (a household-only country does), reads the stored -household and policy through the same services as ``/v2/households`` and -``/v2/policies`` (so a household or policy of another country, or none, is -not found), selects the country's running catalog version, and sends the -simulation service everything it needs: it cannot read these tables. The -simulation service records every calculation, so one not saved to an -account is still stored, as a simulation run with no report. +household calculations (a household-only country does), checks the stored +household and policy exist in that country through the same services as +``/v2/households`` and ``/v2/policies`` (so one of another country, or none, +is not found), selects the country's running catalog version, and sends the +simulation service their IDs and that version's PolicyEngine.py version. +The simulation service reads the household, the policy's parameter values +and the catalog version from the Public API's tables itself, and calculates +the household's default year. It records every calculation, so one not +saved to an account is still stored, as a simulation run with no report. Reading a calculation asks the simulation service for it as it stands. """ @@ -16,11 +18,6 @@ from typing import TYPE_CHECKING, Any -from policyengine_api.services.v2.household_calculations.transformations import ( - household_situation, - policy_parameters, - saved_spm_choice, -) from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationRead, ) @@ -107,12 +104,9 @@ def submit_calculation( household = self._households.get_household( country_id=country_id, household_id=calculation.household_id ) - parameters: dict[str, dict[str, Any]] = {} if calculation.policy_id is not None: - parameters = policy_parameters( - self._policies.get_policy( - country_id=country_id, policy_id=calculation.policy_id - ) + self._policies.get_policy( + country_id=country_id, policy_id=calculation.policy_id ) year = calculation_year(household.default_year) catalog_reader = self._catalog_reader_factory() @@ -122,19 +116,11 @@ def submit_calculation( catalog_reader.close() submission: dict[str, Any] = { "country": country_id, - "catalog_version_id": str(catalog.model_version.id), "policyengine_version": catalog.policyengine_version, - "year": year, "household_id": str(household.id), - "household": household_situation(country_id, household.household_data), - "policy": { - "policy_id": ( - None - if calculation.policy_id is None - else str(calculation.policy_id) - ), - "parameters": parameters, - }, + "policy_id": ( + None if calculation.policy_id is None else str(calculation.policy_id) + ), "options": { "axes": _axes_in_year(calculation, year), "add_missing_variables": calculation.add_missing_variables, @@ -144,9 +130,6 @@ def submit_calculation( "force": False, "trigger": "initial", } - spm = saved_spm_choice(household.household_data) - if spm is not None: - submission["spm"] = spm return HouseholdCalculationRead.from_simulation( self._simulation_factory().submit(submission) ) diff --git a/policyengine_api/services/v2/household_calculations/transformations.py b/policyengine_api/services/v2/household_calculations/transformations.py deleted file mode 100644 index 51de6eb10..000000000 --- a/policyengine_api/services/v2/household_calculations/transformations.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Stored households and policies, in the shapes the Simulation Entrypoint takes. - -A stored v2 household lists its people and each group entity as records, -each with an ``id`` and its ``values``; each person names, for every group -collection, the ``id`` of the group it belongs to. The Simulation Entrypoint's -household functions take the Public API's situation shape instead: ``people`` -maps each person to their values, and each group collection, by its plural, -maps each group to its ``members`` and values. Records keep their ``id`` as -their name, so a calculated value maps back to its record. - -A v2 policy stores each parameter value with its start date and an optional -end date; the household functions take the Public API's policy JSON, each -parameter's values keyed by ``YYYY-MM-DD.YYYY-MM-DD`` period ranges, where an -open end is ``2100-12-31``. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -from policyengine_api.services.v2.households.transformations import ( - LEGACY_COLLECTION_NAMES, -) -from policyengine_api.services.v2.households.validators import ( - SUPPORTED_ENTITY_COLLECTIONS, -) - -if TYPE_CHECKING: - from collections.abc import Mapping - - from policyengine_api.services.v2.policies.types import PolicyRead - -# Each country's stored group collections, as the households service accepts -# them, and their plural in the situation. -GROUP_COLLECTIONS: dict[str, dict[str, str]] = { - country_id: { - collection: LEGACY_COLLECTION_NAMES[collection] for collection in collections - } - for country_id, collections in SUPPORTED_ENTITY_COLLECTIONS.items() -} -OPEN_END_DATE = "2100-12-31" - - -def household_situation( - country_id: str, household_data: Mapping[str, Any] -) -> dict[str, Any]: - """Return a stored household's people and groups in the situation shape. - - The saved SPM choice is not part of the situation; ``saved_spm_choice`` - reads it. - """ - people = household_data["people"] - situation: dict[str, Any] = { - "people": {person["id"]: dict(person["values"]) for person in people} - } - for collection, plural in GROUP_COLLECTIONS[country_id].items(): - situation[plural] = { - group["id"]: { - "members": [ - person["id"] - for person in people - if person["memberships"].get(collection) == group["id"] - ], - **group["values"], - } - for group in household_data.get(collection, []) - } - return situation - - -def saved_spm_choice(household_data: Mapping[str, Any]) -> object: - """Return the household's saved SPM selection, or None when it has none.""" - return household_data.get("spm") - - -def policy_parameters(policy: PolicyRead) -> dict[str, dict[str, Any]]: - """Return the policy's values by parameter and period range.""" - parameters: dict[str, dict[str, Any]] = {} - for value in policy.parameter_values: - end = OPEN_END_DATE if value.end_date is None else value.end_date.date() - period = f"{value.start_date.date().isoformat()}.{end}" - parameters.setdefault(value.parameter_name, {})[period] = value.value - return parameters diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index 246e119e1..331adb3c5 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -23,12 +23,9 @@ from policyengine_api.services.v2.policies.validators import PolicyNotFoundError from tests.unit.v2.household_calculation_support import ( CALCULATION_ID, - CATALOG_VERSION_ID, HOUSEHOLD_ID, POLICY_ID, POLICYENGINE_VERSION, - SITUATION, - SPM_CHOICE, SPM_CONFIG, SPM_PROVENANCE, FakeCatalogs, @@ -80,7 +77,8 @@ def calculation_input(**changes: object) -> HouseholdCalculationInput: return HouseholdCalculationInput.model_validate(values) -def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: +def test_a_calculation_sends_the_household_and_policy_ids_and_the_version() -> None: + """The simulation service reads the household, policy and catalog version.""" parts = Parts() calculation = parts.service.submit_calculation( @@ -90,20 +88,9 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: (submission,) = parts.simulation.submissions assert submission == { "country": "us", - "catalog_version_id": str(CATALOG_VERSION_ID), "policyengine_version": POLICYENGINE_VERSION, - "year": 2026, "household_id": str(HOUSEHOLD_ID), - "household": SITUATION, - "spm": SPM_CHOICE, - "policy": { - "policy_id": str(POLICY_ID), - "parameters": { - "gov.irs.credits.ctc.amount.base[0].amount": { - "2026-01-01.2100-12-31": 3000 - } - }, - }, + "policy_id": str(POLICY_ID), "options": { "axes": [[{**AXES[0][0], "period": "2026"}]], "add_missing_variables": True, @@ -118,31 +105,27 @@ def test_a_calculation_sends_the_stored_household_policy_and_catalog() -> None: assert catalog_reader.closed -def test_current_law_and_no_saved_choice_send_neither() -> None: +def test_current_law_sends_no_policy() -> None: parts = Parts() - stored = household_read() - data = {key: value for key, value in stored.household_data.items() if key != "spm"} - parts.households.households[("us", HOUSEHOLD_ID)] = stored.__class__( - **{**stored.__dict__, "household_data": data} - ) parts.service.submit_calculation(calculation_input(policy_id=None)) (submission,) = parts.simulation.submissions - assert "spm" not in submission - assert submission["policy"] == {"policy_id": None, "parameters": {}} + assert submission["policy_id"] is None assert submission["options"] == {"axes": None, "add_missing_variables": False} -def test_the_year_is_the_households_default_year() -> None: +def test_an_axis_without_a_period_varies_the_households_default_year() -> None: parts = Parts() parts.households.households[("us", HOUSEHOLD_ID)] = household_read( default_year=2027 ) - parts.service.submit_calculation(calculation_input()) + parts.service.submit_calculation(calculation_input(axes=AXES)) - assert parts.simulation.submissions[0]["year"] == 2027 + assert parts.simulation.submissions[0]["options"]["axes"] == [ + [{**AXES[0][0], "period": "2027"}] + ] def test_a_household_without_a_default_year_is_refused() -> None: @@ -202,7 +185,7 @@ def test_a_household_only_country_is_accepted() -> None: (submission,) = parts.simulation.submissions assert submission["country"] == "uk" - assert set(submission["household"]) == {"people", "households", "benunits"} + assert submission["policy_id"] is None def test_an_unknown_household_or_policy_sends_nothing() -> None: diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py index 747b629e9..c1624f643 100644 --- a/tests/unit/v2/test_household_calculation_simulation_client.py +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -22,10 +22,8 @@ ) from tests.unit.v2.household_calculation_support import ( CALCULATION_ID, - CATALOG_VERSION_ID, HOUSEHOLD_ID, POLICYENGINE_VERSION, - SITUATION, simulation_calculation, succeeded_calculation, ) @@ -33,14 +31,13 @@ BASE_URL = "https://simulation.example" SUBMISSION = { "country": "us", - "catalog_version_id": str(CATALOG_VERSION_ID), "policyengine_version": POLICYENGINE_VERSION, - "year": 2026, "household_id": str(HOUSEHOLD_ID), - "household": SITUATION, - "spm": {"scenario": "ce_trend"}, - "policy": {"policy_id": None, "parameters": {}}, - "options": {"axes": None, "add_missing_variables": False}, + "policy_id": None, + "options": { + "axes": [[{"name": "employment_income", "count": 3, "min": 0, "max": 1}]], + "add_missing_variables": False, + }, "force": False, "trigger": "initial", } @@ -239,7 +236,7 @@ def auth_flow(self, request: httpx.Request): # httpx's generator protocol client.read(CALCULATION_ID) -def test_the_household_keeps_its_key_order_both_ways() -> None: +def test_the_household_keeps_its_key_order_as_answered() -> None: requests: list[httpx.Request] = [] answer = succeeded_calculation() client = client_answering(200, answer, requests) @@ -248,11 +245,7 @@ def test_the_household_keeps_its_key_order_both_ways() -> None: submitted = submit_client.submit(SUBMISSION) read = client.read(CALCULATION_ID) - sent_household = json.loads(requests[0].content)["household"] - assert list(sent_household) == list(SITUATION) - assert list(sent_household["households"]["household-1"]) == list( - SITUATION["households"]["household-1"] - ) + assert json.loads(requests[0].content) == SUBMISSION for calculation in (submitted, read): assert calculation is not None returned_household = calculation["result"]["household"] diff --git a/tests/unit/v2/test_household_calculation_transformations.py b/tests/unit/v2/test_household_calculation_transformations.py deleted file mode 100644 index 6efd6e8f7..000000000 --- a/tests/unit/v2/test_household_calculation_transformations.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Stored households and policies, as the Simulation Entrypoint takes them.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from uuid import uuid4 - -from policyengine_api.services.v2.household_calculations.transformations import ( - household_situation, - policy_parameters, - saved_spm_choice, -) -from policyengine_api.services.v2.policies.types import ( - PolicyParameterValueRead, - PolicyRead, -) - -US_HOUSEHOLD = { - "people": [ - { - "id": "person-1", - "source_name": "you", - "values": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, - "memberships": { - "household": "household-1", - "family": "family-1", - "tax_unit": "tax_unit-1", - "spm_unit": "spm_unit-1", - "marital_unit": "marital_unit-1", - }, - }, - { - "id": "person-2", - "values": {"age": {"2026": 8}}, - "memberships": { - "household": "household-1", - "family": "family-1", - "tax_unit": "tax_unit-1", - "spm_unit": "spm_unit-1", - "marital_unit": "marital_unit-2", - }, - }, - ], - "household": [{"id": "household-1", "values": {"state_name": {"2026": "CA"}}}], - "family": [{"id": "family-1", "values": {}}], - "tax_unit": [{"id": "tax_unit-1", "values": {"eitc": {"2026": None}}}], - "spm_unit": [{"id": "spm_unit-1", "values": {}}], - "marital_unit": [ - {"id": "marital_unit-1", "values": {}}, - {"id": "marital_unit-2", "values": {}}, - ], - "spm": {"scenario": "ce_trend", "geography_kind": "national"}, -} -UK_HOUSEHOLD = { - "people": [ - { - "id": "person-1", - "values": {"age": {"2026": 30}}, - "memberships": {"household": "household-1", "benunit": "benunit-1"}, - } - ], - "household": [{"id": "household-1", "values": {"region": {"2026": "WALES"}}}], - "benunit": [{"id": "benunit-1", "values": {"universal_credit": {"2026": None}}}], -} - - -def test_a_us_household_becomes_a_situation_keyed_by_record_id() -> None: - assert household_situation("us", US_HOUSEHOLD) == { - "people": { - "person-1": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, - "person-2": {"age": {"2026": 8}}, - }, - "households": { - "household-1": { - "members": ["person-1", "person-2"], - "state_name": {"2026": "CA"}, - } - }, - "families": {"family-1": {"members": ["person-1", "person-2"]}}, - "tax_units": { - "tax_unit-1": {"members": ["person-1", "person-2"], "eitc": {"2026": None}} - }, - "spm_units": {"spm_unit-1": {"members": ["person-1", "person-2"]}}, - "marital_units": { - "marital_unit-1": {"members": ["person-1"]}, - "marital_unit-2": {"members": ["person-2"]}, - }, - } - - -def test_a_uk_household_has_households_and_benefit_units() -> None: - assert household_situation("uk", UK_HOUSEHOLD) == { - "people": {"person-1": {"age": {"2026": 30}}}, - "households": { - "household-1": {"members": ["person-1"], "region": {"2026": "WALES"}} - }, - "benunits": { - "benunit-1": { - "members": ["person-1"], - "universal_credit": {"2026": None}, - } - }, - } - - -def test_a_group_without_members_keeps_an_empty_member_list() -> None: - household = {**UK_HOUSEHOLD, "benunit": [*UK_HOUSEHOLD["benunit"]]} - household["benunit"].append({"id": "benunit-2", "values": {}}) - - situation = household_situation("uk", household) - - assert situation["benunits"]["benunit-2"] == {"members": []} - - -def test_the_saved_spm_choice_is_the_documents_selection_or_none() -> None: - assert saved_spm_choice(US_HOUSEHOLD) == { - "scenario": "ce_trend", - "geography_kind": "national", - } - assert saved_spm_choice(UK_HOUSEHOLD) is None - - -def _value(*, name: str, value: object, start: datetime, end: datetime | None): - return PolicyParameterValueRead( - id=uuid4(), - parameter_id=uuid4(), - parameter_name=name, - value=value, - start_date=start, - end_date=end, - ) - - -def test_a_policy_becomes_parameter_values_by_period_range() -> None: - policy = PolicyRead( - id=uuid4(), - country_id="us", - tax_benefit_model_id=uuid4(), - tax_benefit_model_version_id=uuid4(), - created_at=datetime(2026, 1, 1, tzinfo=UTC), - updated_at=datetime(2026, 1, 1, tzinfo=UTC), - parameter_values=( - _value( - name="gov.irs.credits.ctc.amount.base[0].amount", - value=3000, - start=datetime(2026, 1, 1, tzinfo=UTC), - end=None, - ), - _value( - name="gov.irs.credits.ctc.refundable.fully_refundable", - value=True, - start=datetime(2026, 1, 1, tzinfo=UTC), - end=datetime(2027, 12, 31, tzinfo=UTC), - ), - _value( - name="gov.irs.credits.ctc.amount.base[0].amount", - value=3500, - start=datetime(2028, 1, 1, tzinfo=UTC), - end=None, - ), - ), - ) - - assert policy_parameters(policy) == { - "gov.irs.credits.ctc.amount.base[0].amount": { - "2026-01-01.2100-12-31": 3000, - "2028-01-01.2100-12-31": 3500, - }, - "gov.irs.credits.ctc.refundable.fully_refundable": { - "2026-01-01.2027-12-31": True - }, - } From 78219f38f33e88d793fc9f77a4429bf686b11366 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 17:10:30 +0400 Subject: [PATCH 113/118] Send axes as given, now that the household function fills a missing period The household function now takes the run's year, the household's default year, from the simulation service and uses it for an axis without a period, so the Public API no longer fills that period itself. The stage 11 guide says the household's default year is also the year of the variables add_missing_variables adds. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 11 +++-- .../v2/household_calculations/services.py | 40 ++++++++----------- .../v2/test_household_calculation_routes.py | 4 +- .../v2/test_household_calculation_service.py | 12 ++---- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 0a33fa8bb..29bc2688a 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -281,12 +281,11 @@ Public API answers with 503. The answer is 202 with the calculation's `status` is `succeeded`, with `result`, or `failed`, with `error`. A US result carries `spm_config` and `spm_provenance`. -With `add_missing_variables`, the v2 household function adds each missing -variable for the year it would choose for `/calculate-full`: the first period -of the age of a person named `you`, else the current calendar year (UTC). The -Public API names people by their record IDs, so for a household whose -`default_year` is not the current year the added variables are requested for -the current year until the household function takes the calculation's year. +The simulation service sends the household function the household's +`default_year` as the calculation's year: the year of the variables +`add_missing_variables` adds, and of values and axes the household gives +without a period. (`/calculate-full` takes that year from the first age +period of a person named `you`, or else the current year.) The simulation service stores every calculation as a simulation run, so a calculation no account saved is still stored, with no report. Neither route diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index 41dcae0fa..cadc5ab0f 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -41,28 +41,6 @@ ) -def _axes_in_year( - calculation: HouseholdCalculationInput, year: int -) -> list[list[dict[str, Any]]] | None: - """Return the axes as sent, each without a period varying ``year``. - - The country model would otherwise vary its own default period, the - current year, rather than the year the household's values name. - """ - if calculation.axes is None: - return None - sent_axes = [] - for group in calculation.axes: - sent_group = [] - for axis in group: - sent_axis = axis.model_dump(mode="json", exclude_unset=True) - if sent_axis.get("period") is None: - sent_axis["period"] = str(year) - sent_group.append(sent_axis) - sent_axes.append(sent_group) - return sent_axes - - class V2HouseholdCalculationService: """Submit household calculations to the simulation service and read them.""" @@ -108,7 +86,9 @@ def submit_calculation( self._policies.get_policy( country_id=country_id, policy_id=calculation.policy_id ) - year = calculation_year(household.default_year) + # The simulation service calculates the household's default year; + # one without it is refused here, with 422. + calculation_year(household.default_year) catalog_reader = self._catalog_reader_factory() try: catalog = catalog_reader.get_model_by_country(country_id) @@ -122,7 +102,19 @@ def submit_calculation( None if calculation.policy_id is None else str(calculation.policy_id) ), "options": { - "axes": _axes_in_year(calculation, year), + # The household function varies the run's year, the + # household's default year, along an axis without a period. + "axes": ( + None + if calculation.axes is None + else [ + [ + axis.model_dump(mode="json", exclude_unset=True) + for axis in group + ] + for group in calculation.axes + ] + ), "add_missing_variables": calculation.add_missing_variables, }, # A calculation is reused for 28 days after it succeeded; callers diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index aca622d83..56ad0b595 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -106,9 +106,7 @@ def test_a_calculation_with_axes_sends_them() -> None: response = app.client.post(PATH, json=body(axes=axes)) assert response.status_code == 202 - assert app.simulation.submissions[0]["options"]["axes"] == [ - [{**axes[0][0], "period": "2026"}] - ] + assert app.simulation.submissions[0]["options"]["axes"] == axes def test_a_succeeded_us_calculation_reads_back_its_spm_choice() -> None: diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index 331adb3c5..ab45ac526 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -92,7 +92,7 @@ def test_a_calculation_sends_the_household_and_policy_ids_and_the_version() -> N "household_id": str(HOUSEHOLD_ID), "policy_id": str(POLICY_ID), "options": { - "axes": [[{**AXES[0][0], "period": "2026"}]], + "axes": AXES, "add_missing_variables": True, }, "force": False, @@ -115,17 +115,13 @@ def test_current_law_sends_no_policy() -> None: assert submission["options"] == {"axes": None, "add_missing_variables": False} -def test_an_axis_without_a_period_varies_the_households_default_year() -> None: +def test_an_axis_without_a_period_is_sent_as_given() -> None: + """The simulation service varies the household's default year for it.""" parts = Parts() - parts.households.households[("us", HOUSEHOLD_ID)] = household_read( - default_year=2027 - ) parts.service.submit_calculation(calculation_input(axes=AXES)) - assert parts.simulation.submissions[0]["options"]["axes"] == [ - [{**AXES[0][0], "period": "2027"}] - ] + assert parts.simulation.submissions[0]["options"]["axes"] == AXES def test_a_household_without_a_default_year_is_refused() -> None: From 25427397c515383514d6bd8ca965cfd977c0e32b Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 17:49:54 +0400 Subject: [PATCH 114/118] Pass the simulation service's 422 refusals through with their reasons The simulation service now refuses the caller's data with 422 and its reason as text: a policy changing a parameter the catalog version lacks, or a stored policy value or household the household functions cannot take, such as a group no person belongs to. The Public API read only a list of refused fields, so such a refusal became 503. It now reads the service's refusal model and answers 422 with the reason, or with each refused field. The service's other refusals (400), for references the Public API has already checked, stay 503. The route's declared 422 answer, the adapter's description and the stage 11 guide say so. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 19 ++++---- .../household_calculations/response_models.py | 6 ++- .../simulation_client.py | 44 ++++++++++++------- .../v2/household_calculations/types.py | 7 ++- .../v2/test_household_calculation_routes.py | 32 +++++++++++--- ...household_calculation_simulation_client.py | 26 ++++++++--- 6 files changed, 94 insertions(+), 40 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index 29bc2688a..e9968a26e 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -273,11 +273,8 @@ is not stored in that country), selects the country's running catalog version, and sends the simulation service the household's and policy's IDs, the options and that version's PolicyEngine.py version. The simulation service reads the household, the policy's parameter values and the catalog -version from these tables itself, through its own `SELECT` grant, and -refuses a policy changing a parameter the catalog version lacks, which the -Public API answers with 503. The answer is 202 with the calculation's -`calculation_id` and -`status`. Poll `GET /v2/household-calculations/{calculation_id}` until +version from these tables itself, through its own `SELECT` grant. The +answer is 202 with the calculation's `calculation_id` and `status`. Poll `GET /v2/household-calculations/{calculation_id}` until `status` is `succeeded`, with `result`, or `failed`, with `error`. A US result carries `spm_config` and `spm_provenance`. @@ -305,9 +302,15 @@ Without the URL, or with gateway auth partly configured, both routes answer 503. They also answer 503 when the simulation service cannot be reached, its gateway token cannot be fetched, it answers with an error or with a body of another shape, or it refuses the calculation (for example when no v2 worker is -deployed for the catalog version's PolicyEngine.py version). A household or -options the simulation service finds invalid, such as a group no person -belongs to, get 422 with each invalid field and why. +deployed for the catalog version's PolicyEngine.py version, or when it +cannot find a household, policy or catalog version the Public API has just +found, which means the two services disagree). The simulation service checks +every stored household and policy value against the household functions' +contract before it writes anything, and the caller's data it refuses gets +422 with its reason: a policy changing a parameter the catalog version +lacks, a stored policy value it cannot take (such as an object), or a stored +household it cannot take (such as a group no person belongs to, which +`/v2/households` accepts). ### Staging checks diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index d21b25968..20cf7cc0f 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -95,8 +95,10 @@ class HouseholdCalculationResponse(StrictHouseholdCalculationAPIModel): "description": ( "The request does not match the schema, the country does not offer " "household calculations, the household has no default year, or the " - "simulation service finds the household or its options invalid " - "(the message lists why)." + "simulation service refuses the caller's data, with its reason in " + "the message: a policy changing a parameter the catalog version " + "lacks, a stored policy value or household it cannot calculate " + "(such as a group no person belongs to), or an invalid option." ), }, 500: { diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 6045e8281..097747d79 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -14,8 +14,13 @@ database returns it, which orders each record's keys itself, so it can differ from the order a caller sent. -A 400 is a refusal, whose reason the service gives; a 422 lists what the -service finds invalid in the household or its options; anything else, +A 400 is a refusal, whose reason the service gives (a household, policy or +catalog version it cannot find, or a household without a default year, +which the Public API checks first, or no household function for the +version); a 422 is the caller's data the service refuses, with its reason +as text (a parameter the catalog version lacks, or a stored policy value or +household the household functions cannot take) or one entry per request +field it refuses; anything else, including a 503, no answer, an answer of another shape, a gateway token that cannot be fetched or missing configuration, means the service is unavailable. @@ -36,12 +41,12 @@ from policyengine_api_simulation_client.models.household_calculation_submission import ( HouseholdCalculationSubmission, ) +from policyengine_api_simulation_client.models.household_refusal_response import ( + HouseholdRefusalResponse, +) from policyengine_api_simulation_client.models.household_route_error import ( HouseholdRouteError, ) -from policyengine_api_simulation_client.models.http_validation_error import ( - HTTPValidationError, -) from policyengine_observability import ( # type: ignore[import-untyped] # the package has no type information instrument_httpx, ) @@ -112,17 +117,22 @@ def _calculation(response: httpx.Response) -> dict[str, Any]: return body -def _invalid_request_reason(error: HTTPValidationError) -> str: - """Return each invalid field and why, without the request's ``body`` prefix.""" - reasons = [] - for item in error.detail if isinstance(error.detail, list) else []: - location = [str(part) for part in item.loc] - if location[:1] == ["body"]: - location = location[1:] - reasons.append(f"{'.'.join(location)}: {item.msg}") - return "The simulation service cannot calculate this household: " + ( - "; ".join(reasons) or "the request is invalid" - ) +def _invalid_request_reason(refusal: HouseholdRefusalResponse) -> str: + """Return the service's reason, or each refused field and why. + + A field's location is given without the request's ``body`` prefix. + """ + if isinstance(refusal.detail, str): + reason = refusal.detail + else: + reasons = [] + for item in refusal.detail: + location = [str(part) for part in item.loc] + if location[:1] == ["body"]: + location = location[1:] + reasons.append(f"{'.'.join(location)}: {item.msg}") + reason = "; ".join(reasons) or "the request is invalid" + return f"The simulation service cannot calculate this household: {reason}" def _unavailable(status_code: int) -> SimulationServiceUnavailableError: @@ -191,7 +201,7 @@ def submit(self, submission: dict[str, Any]) -> dict[str, Any]: if response.status_code == HTTPStatus.ACCEPTED: return _calculation(response) if response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: - _body, invalid = _decoded(response, HTTPValidationError.from_dict) + _body, invalid = _decoded(response, HouseholdRefusalResponse.from_dict) raise SimulationServiceInvalidRequestError(_invalid_request_reason(invalid)) if response.status_code == HTTPStatus.BAD_REQUEST: _body, refusal = _decoded(response, HouseholdRouteError.from_dict) diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py index f8e33e626..d0768b4b7 100644 --- a/policyengine_api/services/v2/household_calculations/types.py +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -177,9 +177,12 @@ class SimulationServiceRefusedError(RuntimeError): class SimulationServiceInvalidRequestError(ValueError): - """The simulation service finds the household or its options invalid. + """The simulation service refuses the caller's data. - The message lists each invalid field and why, as the service reports it. + Such as a policy changing a parameter the catalog version lacks, a stored + policy value or household the household functions cannot take, or an + invalid option. The message gives the service's reason, or each refused + field and why. """ diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 56ad0b595..667cf175a 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -4,6 +4,7 @@ from uuid import uuid4 +import httpx import pytest from fastapi.testclient import TestClient from flask import Flask @@ -18,8 +19,10 @@ from policyengine_api.services.v2.household_calculations.services import ( V2HouseholdCalculationService, ) +from policyengine_api.services.v2.household_calculations.simulation_client import ( + SimulationHouseholdClient, +) from policyengine_api.services.v2.household_calculations.types import ( - SimulationServiceInvalidRequestError, SimulationServiceRefusedError, SimulationServiceUnavailableError, ) @@ -248,17 +251,34 @@ def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None assert app.simulation.submissions == [] -def test_a_household_the_simulation_service_finds_invalid_gets_422_with_why() -> None: +def test_a_stored_household_with_a_group_no_person_belongs_to_gets_422() -> None: + """The simulation service's 422 reaches the caller with its reason.""" + refusal = ( + f"the household {HOUSEHOLD_ID} cannot be calculated: " + "benunits.benunit-2.members: List should have at least 1 item after " + "validation" + ) + + def answer(request: httpx.Request) -> httpx.Response: + del request + return httpx.Response(422, json={"detail": refusal}) + app = App() - app.simulation.error = SimulationServiceInvalidRequestError( - "The simulation service cannot calculate this household: " - "household.households.household-1.members: List should have at least 1 item" + app.service = V2HouseholdCalculationService( + countries=FakeCountries(), + households=app.households, + policies=FakePolicies(), + catalog_reader_factory=FakeCatalogs, + simulation_factory=lambda: SimulationHouseholdClient( + "https://simulation.example", + httpx_args={"transport": httpx.MockTransport(answer)}, + ), ) response = app.client.post(PATH, json=body()) assert response.status_code == 422 - assert "members" in response.json()["message"] + assert refusal in response.json()["message"] def test_axis_groups_within_the_point_limit_are_accepted() -> None: diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py index c1624f643..d132e953b 100644 --- a/tests/unit/v2/test_household_calculation_simulation_client.py +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -165,15 +165,31 @@ def test_the_configured_service_is_called_without_auth_when_none_is_set( assert client.base_url == BASE_URL -def test_a_household_the_service_finds_invalid_is_reported_with_its_reasons() -> None: +STORED_HOUSEHOLD_REFUSAL = ( + "the household 00000000-0000-4000-8000-0000000000a1 cannot be calculated: " + "benunits.benunit-2.members: List should have at least 1 item after validation" +) + + +def test_caller_data_the_service_refuses_is_reported_with_its_reason() -> None: + """The service refuses a stored household or policy value with 422 and text.""" + client = client_answering(422, {"detail": STORED_HOUSEHOLD_REFUSAL}, []) + + with pytest.raises( + SimulationServiceInvalidRequestError, match=r"benunit-2\.members" + ): + client.submit(SUBMISSION) + + +def test_request_fields_the_service_refuses_are_reported_one_by_one() -> None: client = client_answering( 422, { "detail": [ { - "loc": ["body", "household", "households", "h", "members"], - "msg": "List should have at least 1 item after validation", - "type": "too_short", + "loc": ["body", "options", "axes", 0, 0, "count"], + "msg": "Input should be greater than or equal to 1", + "type": "greater_than_equal", } ] }, @@ -182,7 +198,7 @@ def test_a_household_the_service_finds_invalid_is_reported_with_its_reasons() -> with pytest.raises( SimulationServiceInvalidRequestError, - match=r"household\.households\.h\.members: List should have at least 1 item", + match=r"options\.axes\.0\.0\.count: Input should be greater", ): client.submit(SUBMISSION) From 21ad2051e7a1f60b8770acb6ca1b53e5ac079d89 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 17:50:38 +0400 Subject: [PATCH 115/118] Name the parity household's person `you`, so both sides add one year's variables The staging parity household that adds missing variables named its person `adult`, so API v1's /calculate-full added them for the current calendar year while the v2 path adds them for the household's default year, and the two would differ from 2027. It now names its person `you`, whose age gives /calculate-full the household's year, and a test with the date fixed in 2027 checks both sides use the same year. Co-Authored-By: Claude Opus 5.5 --- .../test_live_v2_household_parity.py | 10 +++-- .../test_live_household_parity_households.py | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/unit/v2/test_live_household_parity_households.py diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py index 758482aa8..f6042f6ca 100644 --- a/tests/integration/test_live_v2_household_parity.py +++ b/tests/integration/test_live_v2_household_parity.py @@ -27,7 +27,7 @@ CALCULATED = {YEAR: None} -def _parity_households(run_amount: int) -> list[LiveHousehold]: +def parity_households(run_amount: int) -> list[LiveHousehold]: return [ LiveHousehold( name="us-single-california", @@ -86,8 +86,10 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: LiveHousehold( name="us-single-reform-added-variables", country_id="us", + # /calculate-full adds variables for the age year of a person + # named ``you``; the v2 path for the household's default year. people={ - "adult": { + "you": { "age": {YEAR: 29}, "employment_income": {YEAR: 61_000 + run_amount}, } @@ -155,7 +157,7 @@ def _parity_households(run_amount: int) -> list[LiveHousehold]: ] -PARITY_NAMES = [household.name for household in _parity_households(0)] +PARITY_NAMES = [household.name for household in parity_households(0)] @pytest.mark.parametrize("household_name", PARITY_NAMES) @@ -164,7 +166,7 @@ def test_a_v2_household_calculation_matches_api_v1( ): (household,) = [ candidate - for candidate in _parity_households(probe_amount(integration_probe_id)) + for candidate in parity_households(probe_amount(integration_probe_id)) if candidate.name == household_name ] household_id = store_household(api_client, household) diff --git a/tests/unit/v2/test_live_household_parity_households.py b/tests/unit/v2/test_live_household_parity_households.py new file mode 100644 index 000000000..3782c329d --- /dev/null +++ b/tests/unit/v2/test_live_household_parity_households.py @@ -0,0 +1,44 @@ +"""The staging parity set's households calculate the same year on both sides. + +API v1's ``/calculate-full`` adds missing variables for the first age period +of a person named ``you``, or else the current year; the v2 household +function adds them for the stored household's default year. A parity +household that adds missing variables must therefore name its person +``you``, or the two sides differ once the calendar year moves on. +""" + +from __future__ import annotations + +from datetime import date + +from policyengine_api.services import household_calculation_service +from tests.fixtures.integration.live_household_calculations import YEAR +from tests.integration.test_live_v2_household_parity import parity_households + + +class DateIn2027(date): + """Today is a day in 2027, a year after the households' year.""" + + @classmethod + def today(cls) -> DateIn2027: + """Return 2027-06-01.""" + return cls(2027, 6, 1) + + +def test_households_that_add_variables_use_one_year_on_both_sides( + monkeypatch, +) -> None: + monkeypatch.setattr(household_calculation_service, "date", DateIn2027) + adding = [ + household + for household in parity_households(run_amount=0) + if household.add_missing_variables + ] + + assert adding + for household in adding: + v1_year = household_calculation_service.get_household_year( + household.situation() + ) + # The v2 side calculates the stored household's default year, YEAR. + assert str(v1_year) == YEAR, household.name From 5184700273e2c05ad9e3af8d1254c29787607b55 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 28 Oct 2026 17:53:29 +0400 Subject: [PATCH 116/118] Check that axis periods are forwarded unchanged, and name the year check A test now sends an axis with an explicit period and one with a null period and checks both reach the simulation service unchanged, as they must now that the household function fills a missing period itself. The check that a household has a default year returns nothing, so it is named require_default_year. Co-Authored-By: Claude Opus 5.5 --- .../services/v2/household_calculations/services.py | 4 ++-- .../services/v2/household_calculations/validators.py | 8 +++++--- tests/unit/v2/test_household_calculation_service.py | 12 ++++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py index cadc5ab0f..7a20d30d6 100644 --- a/policyengine_api/services/v2/household_calculations/services.py +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -23,7 +23,7 @@ ) from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationNotFoundError, - calculation_year, + require_default_year, require_household_country, ) @@ -88,7 +88,7 @@ def submit_calculation( ) # The simulation service calculates the household's default year; # one without it is refused here, with 422. - calculation_year(household.default_year) + require_default_year(household.default_year) catalog_reader = self._catalog_reader_factory() try: catalog = catalog_reader.get_model_by_country(country_id) diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py index 5475e3328..9eb7dcd93 100644 --- a/policyengine_api/services/v2/household_calculations/validators.py +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -37,9 +37,11 @@ def require_household_country( raise HouseholdCalculationCountryError(msg) -def calculation_year(default_year: int | None) -> int: - """Return the household's default year, the year its values name.""" +def require_default_year(default_year: int | None) -> None: + """Raise HouseholdCalculationYearError unless the household has a default year. + + The simulation service calculates that year, the year its values name. + """ if default_year is None: msg = "This household has no default year to calculate." raise HouseholdCalculationYearError(msg) - return default_year diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index ab45ac526..f443cca0f 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -300,3 +300,15 @@ def test_an_axis_keeps_its_own_period() -> None: parts.service.submit_calculation(calculation_input(axes=axes)) assert parts.simulation.submissions[0]["options"]["axes"] == axes + + +@pytest.mark.parametrize( + "period", ["2025", None], ids=["explicit-period", "null-period"] +) +def test_an_axis_period_is_forwarded_unchanged(period) -> None: + parts = Parts() + axes = [[{**AXES[0][0], "period": period}]] + + parts.service.submit_calculation(calculation_input(axes=axes)) + + assert parts.simulation.submissions[0]["options"]["axes"] == axes From 109d62630c91cf460ad1ffc8d2e02f1d0d0b4391 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 29 Oct 2026 02:22:46 +0400 Subject: [PATCH 117/118] Answer 503 for a simulation refusal of another shape The generated refusal model keeps a 422's detail as it came when it is neither text nor a list of field errors, such as null, an object, a number, a list of strings or an entry without a type, and building the reason from it raised, so the route answered 500. Such a refusal now means the simulation service's answer could not be read, and the route answers 503 as documented. Co-Authored-By: Claude Opus 5.5 --- .../simulation_client.py | 22 ++++++++++--- .../v2/test_household_calculation_routes.py | 33 +++++++++++++++++++ ...household_calculation_simulation_client.py | 18 ++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py index 097747d79..6ff1689a7 100644 --- a/policyengine_api/services/v2/household_calculations/simulation_client.py +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -44,6 +44,9 @@ from policyengine_api_simulation_client.models.household_refusal_response import ( HouseholdRefusalResponse, ) +from policyengine_api_simulation_client.models.household_request_field_error import ( + HouseholdRequestFieldError, +) from policyengine_api_simulation_client.models.household_route_error import ( HouseholdRouteError, ) @@ -120,13 +123,24 @@ def _calculation(response: httpx.Response) -> dict[str, Any]: def _invalid_request_reason(refusal: HouseholdRefusalResponse) -> str: """Return the service's reason, or each refused field and why. - A field's location is given without the request's ``body`` prefix. + A field's location is given without the request's ``body`` prefix. Raise + SimulationServiceUnavailableError for a refusal of another shape: the + generated model keeps a ``detail`` it cannot read as text or fields. """ - if isinstance(refusal.detail, str): - reason = refusal.detail + detail: object = refusal.detail + if isinstance(detail, str): + reason = detail else: + fields = ( + [item for item in detail if isinstance(item, HouseholdRequestFieldError)] + if isinstance(detail, list) + else [] + ) + if not isinstance(detail, list) or len(fields) != len(detail): + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) reasons = [] - for item in refusal.detail: + for item in fields: location = [str(part) for part in item.loc] if location[:1] == ["body"]: location = location[1:] diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py index 667cf175a..2bbd63642 100644 --- a/tests/unit/v2/test_household_calculation_routes.py +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -294,3 +294,36 @@ def test_axis_groups_within_the_point_limit_are_accepted() -> None: response = app.client.post(PATH, json=body(axes=axes)) assert response.status_code == 202 + + +@pytest.mark.parametrize( + "refusal_body", + [ + {"detail": None}, + {"detail": {"reason": "x"}}, + {"detail": 7}, + {"detail": ["the household is refused"]}, + {"detail": [{"loc": ["body", "options"], "msg": "Field required"}]}, + ], + ids=["null", "object", "number", "list-of-strings", "entry-without-type"], +) +def test_a_simulation_refusal_of_another_shape_answers_503(refusal_body) -> None: + def answer(request: httpx.Request) -> httpx.Response: + del request + return httpx.Response(422, json=refusal_body) + + app = App() + app.service = V2HouseholdCalculationService( + countries=FakeCountries(), + households=app.households, + policies=FakePolicies(), + catalog_reader_factory=FakeCatalogs, + simulation_factory=lambda: SimulationHouseholdClient( + "https://simulation.example", + httpx_args={"transport": httpx.MockTransport(answer)}, + ), + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 503 diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py index d132e953b..8d84b8c8f 100644 --- a/tests/unit/v2/test_household_calculation_simulation_client.py +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -269,3 +269,21 @@ def test_the_household_keeps_its_key_order_as_answered() -> None: assert list(returned_household["households"]["household-1"]) == list( answer["result"]["household"]["households"]["household-1"] ) + + +@pytest.mark.parametrize( + "body", + [ + {"detail": None}, + {"detail": {"reason": "x"}}, + {"detail": 7}, + {"detail": ["the household is refused"]}, + {"detail": [{"loc": ["body", "options"], "msg": "Field required"}]}, + ], + ids=["null", "object", "number", "list-of-strings", "entry-without-type"], +) +def test_a_refusal_of_another_shape_is_unavailable(body) -> None: + client = client_answering(422, body, []) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + client.submit(SUBMISSION) From 4c515e2e346a51eaee65c5ee59467bd1c6d6baa6 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 29 Oct 2026 02:25:49 +0400 Subject: [PATCH 118/118] Describe the remaining 503s, the per-field 422, and test the sent default year The declared 503 answer now lists what is left for it (an unavailable dependency, an unreadable answer, a refused reference the Public API has already checked, or no v2 worker), and the stage 11 guide says a refused request field gets 422 with each field. The parity year test checks the default year the live check stores the household with, not a constant, and the axis-period tests are one parametrized test. Co-Authored-By: Claude Opus 5.5 --- docs/migration/stage-11-v2-households.md | 3 ++- .../household_calculations/response_models.py | 5 ++-- .../v2/test_household_calculation_service.py | 13 +++------- .../test_live_household_parity_households.py | 26 ++++++++++++++++--- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index e9968a26e..701673f0b 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -310,7 +310,8 @@ contract before it writes anything, and the caller's data it refuses gets 422 with its reason: a policy changing a parameter the catalog version lacks, a stored policy value it cannot take (such as an object), or a stored household it cannot take (such as a group no person belongs to, which -`/v2/households` accepts). +`/v2/households` accepts). A request field it refuses, such as an invalid +option, gets 422 with each refused field and why. ### Staging checks diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py index 20cf7cc0f..d1705fab7 100644 --- a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -109,8 +109,9 @@ class HouseholdCalculationResponse(StrictHouseholdCalculationAPIModel): "model": V2ErrorResponse, "description": ( "The simulation service, the country's catalog version or v2 " - "storage is unavailable, or the simulation service cannot " - "calculate this household." + "storage is unavailable; the simulation service's answer cannot " + "be read; or it refuses a reference the Public API has already " + "checked, or has no v2 worker for the catalog version." ), }, } diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py index f443cca0f..c87c820db 100644 --- a/tests/unit/v2/test_household_calculation_service.py +++ b/tests/unit/v2/test_household_calculation_service.py @@ -293,17 +293,10 @@ def test_requests_refused_by_the_public_api_need_no_simulation_service() -> None available_country.submit_calculation(calculation_input()) -def test_an_axis_keeps_its_own_period() -> None: - parts = Parts() - axes = [[{**AXES[0][0], "period": "2025-03"}]] - - parts.service.submit_calculation(calculation_input(axes=axes)) - - assert parts.simulation.submissions[0]["options"]["axes"] == axes - - @pytest.mark.parametrize( - "period", ["2025", None], ids=["explicit-period", "null-period"] + "period", + ["2025", "2025-03", None], + ids=["explicit-year", "explicit-month", "null-period"], ) def test_an_axis_period_is_forwarded_unchanged(period) -> None: parts = Parts() diff --git a/tests/unit/v2/test_live_household_parity_households.py b/tests/unit/v2/test_live_household_parity_households.py index 3782c329d..d66ebd6ea 100644 --- a/tests/unit/v2/test_live_household_parity_households.py +++ b/tests/unit/v2/test_live_household_parity_households.py @@ -9,10 +9,13 @@ from __future__ import annotations +import json from datetime import date +import httpx + from policyengine_api.services import household_calculation_service -from tests.fixtures.integration.live_household_calculations import YEAR +from tests.fixtures.integration.live_household_calculations import store_household from tests.integration.test_live_v2_household_parity import parity_households @@ -40,5 +43,22 @@ def test_households_that_add_variables_use_one_year_on_both_sides( v1_year = household_calculation_service.get_household_year( household.situation() ) - # The v2 side calculates the stored household's default year, YEAR. - assert str(v1_year) == YEAR, household.name + # The v2 side calculates the default year the household is stored with. + assert str(v1_year) == str(_stored_default_year(household)), household.name + + +def _stored_default_year(household) -> int: + """Return the default year the live check stores the household with.""" + sent: list[dict] = [] + + def answer(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response(201, json={"result": {"item": {"id": "stored"}}}) + + with httpx.Client( + base_url="https://public-api.example", + transport=httpx.MockTransport(answer), + ) as client: + store_household(client, household) + (request,) = sent + return request["default_year"]