diff --git a/.github/scripts/check_simulation_entrypoint_environment.sh b/.github/scripts/check_simulation_entrypoint_environment.sh new file mode 100755 index 000000000..7b6555949 --- /dev/null +++ b/.github/scripts/check_simulation_entrypoint_environment.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# Check that a deployment environment calls its own Simulation Entrypoint: +# staging must call the staging service and production the production +# service. The entrypoint names the Cloud Run revision that served a request +# in its x-policyengine-simulation-revision response header, and a Cloud Run +# revision name is its service name followed by a five-digit number and a +# three-character suffix. +# +# The old gateway (SIM_ENTRYPOINT=old_gateway_direct) is one shared Modal app +# that reports no environment, so it is not checked. +# +# Required env vars: DEPLOYMENT_ENVIRONMENT (staging or production), +# SIM_ENTRYPOINT, and the URL variable that SIM_ENTRYPOINT selects. +# Exit codes: 0 when the environments match, 1 when they do not or the +# response names no revision, 2 for an unknown DEPLOYMENT_ENVIRONMENT, and +# curl's own exit code when the request fails. + +set -euo pipefail + +source .github/scripts/simulation_entrypoint_env.sh + +deployment_environment="${DEPLOYMENT_ENVIRONMENT:-}" +case "${deployment_environment}" in + staging) + expected_service="policyengine-simulation-entry-staging" + ;; + production) + expected_service="policyengine-simulation-entry" + ;; + *) + printf 'DEPLOYMENT_ENVIRONMENT must be staging or production, not %q\n' \ + "${deployment_environment}" >&2 + exit 2 + ;; +esac + +if [[ "${SIM_ENTRYPOINT:-}" == "old_gateway_direct" ]]; then + printf 'SIM_ENTRYPOINT=old_gateway_direct: the shared old gateway reports no environment, so there is nothing to check\n' + exit 0 +fi + +simulation_url="$(simulation_entrypoint_selected_url "${SIM_ENTRYPOINT:-}")" +simulation_url="${simulation_url%/}" + +health_response="$( + curl --fail --silent --show-error --include \ + --connect-timeout 10 --max-time 60 \ + "${simulation_url}/health" +)" + +# With --include, curl prints every response's headers; the last revision +# header belongs to the final response. +served_revision="$( + awk ' + tolower($1) == "x-policyengine-simulation-revision:" { + gsub("\r", "", $2) + print $2 + } + ' <<<"${health_response}" | + tail -n 1 +)" + +if [[ -z "${served_revision}" ]]; then + printf '%s/health did not name a Simulation Entrypoint revision; the %s deployment must call the stable URL of %s\n' \ + "${simulation_url}" "${deployment_environment}" "${expected_service}" >&2 + exit 1 +fi + +if ! [[ "${served_revision}" =~ ^(.+)-[0-9]{5}-[a-z0-9]{3}$ ]]; then + printf '%s, served by %s, is not a Cloud Run revision name\n' \ + "${served_revision}" "${simulation_url}" >&2 + exit 1 +fi +served_service="${BASH_REMATCH[1]}" + +if [[ "${served_service}" != "${expected_service}" ]]; then + printf 'The %s deployment calls %s, which revision %s of %s serves; it must call %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" "${expected_service}" >&2 + exit 1 +fi + +printf 'The %s deployment calls %s, served by revision %s of %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" diff --git a/.github/scripts/run_live_staging_tests.sh b/.github/scripts/run_live_staging_tests.sh index 39bc7a3cb..d7b5f6731 100755 --- a/.github/scripts/run_live_staging_tests.sh +++ b/.github/scripts/run_live_staging_tests.sh @@ -8,4 +8,6 @@ python -m pytest \ tests/integration/test_live_budget_window_cache.py \ tests/integration/test_live_calculate.py \ tests/integration/test_live_economy.py \ + tests/integration/test_live_v2_household_calculations.py \ + tests/integration/test_live_v2_household_parity.py \ -v diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index af02524d3..d18fa4f53 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -73,11 +73,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: staging env: + DEPLOYMENT_ENVIRONMENT: staging SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to staging + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation @@ -339,7 +342,7 @@ jobs: parallel-live-simulation-tests-staging-cloud-run: name: Run live simulations against the staging candidate runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 needs: deploy-cloud-run-staging if: | (github.repository == 'PolicyEngine/policyengine-api') @@ -438,11 +441,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: production env: + DEPLOYMENT_ENVIRONMENT: production SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to production + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation diff --git a/.github/workflows/seed-v2-database.yml b/.github/workflows/seed-v2-database.yml index 634352775..723b0bcb2 100644 --- a/.github/workflows/seed-v2-database.yml +++ b/.github/workflows/seed-v2-database.yml @@ -66,3 +66,9 @@ jobs: run: uv run python scripts/publish_v2_metadata_catalog.py env: V2_DATA_WRITE_DATABASE_URL: ${{ secrets.V2_DATA_WRITE_DATABASE_URL }} + # Reads the certified UK data release, whose repository is private, + # to check each new catalog version's region filters. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + # Each version's SPM availability comes from this environment's + # simulation service's public versions answer. + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 3f402ae80..599726735 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -73,16 +73,16 @@ jobs: run: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles - name: Verify the managed v2 roles hold their declared privileges run: uv run python scripts/reconcile_v2_privileges.py verify - - name: Verify the installed PolicyEngine.py catalog interface - run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py + - name: Verify the installed PolicyEngine.py catalog interface and its parity with v1 metadata + run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py tests/integration/test_v2_catalog_metadata_parity.py env: RUN_V2_CATALOG_COMPATIBILITY: "1" - - name: Test v2 metadata publication and resource routes - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + - name: Test v2 metadata publication, resource routes and the countries list + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_catalog_spm_availability.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_user_identity_persistence.py tests/integration/test_v2_saved_policy_view.py - name: Test comparison-run persistence and privilege reconciliation as the managed roles run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_stage12_comparison_persistence.py tests/integration/test_v2_privilege_reconciliation.py - name: Test v2 household persistence and immediate v1 copying diff --git a/.gitignore b/.gitignore index d69167e9d..18fe946f3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,8 @@ gha-creds-*.json ## vscode settings /.vscode *.db.init.lock + +# coverage.py data and reports written by `make test` +.coverage +.coverage.* +coverage.xml diff --git a/changelog.d/catalog-spm-availability.added.md b/changelog.d/catalog-spm-availability.added.md new file mode 100644 index 000000000..c22bc0f26 --- /dev/null +++ b/changelog.d/catalog-spm-availability.added.md @@ -0,0 +1 @@ +Each published catalog version now records whether it offers SPM settings, with their schema and certified defaults, from the simulation service's public versions answer; the catalog publication step reads `SIMULATION_ENTRYPOINT_URL`, and a version published earlier gets it on the next publication run of its PolicyEngine.py version. diff --git a/changelog.d/model-catalog-spm-availability.added.md b/changelog.d/model-catalog-spm-availability.added.md new file mode 100644 index 000000000..b8515016b --- /dev/null +++ b/changelog.d/model-catalog-spm-availability.added.md @@ -0,0 +1 @@ +`GET /v2/model-catalog` now returns `spm`: whether the catalog version offers SPM settings for a household, with their JSON schema and certified defaults when it does. diff --git a/changelog.d/simulation-entrypoint-environment-check.added.md b/changelog.d/simulation-entrypoint-environment-check.added.md new file mode 100644 index 000000000..2e592c45b --- /dev/null +++ b/changelog.d/simulation-entrypoint-environment-check.added.md @@ -0,0 +1 @@ +Each release now checks, in its staging and production model-version alignment jobs, that the environment's simulation URL is served by that environment's own Simulation Entrypoint service, and fails otherwise; staging must call the staging simulation service, and production the production one. diff --git a/changelog.d/v2-countries-route.added.md b/changelog.d/v2-countries-route.added.md new file mode 100644 index 000000000..66b8d8e46 --- /dev/null +++ b/changelog.d/v2-countries-route.added.md @@ -0,0 +1 @@ +Added `GET /v2/countries`, which lists every row of the countries table: each served country's ID and whether it supports household and society-wide calculations. diff --git a/changelog.d/v2-countries-startup-check.added.md b/changelog.d/v2-countries-startup-check.added.md new file mode 100644 index 000000000..8e30ae192 --- /dev/null +++ b/changelog.d/v2-countries-startup-check.added.md @@ -0,0 +1 @@ +The Public API refuses to start when the v2 `countries` table and its country list in code name different countries, or when the table or the privilege to read it is missing. The check runs on Cloud Run and wherever a v2 runtime database is configured; when it cannot query the table, it logs why and startup continues. diff --git a/changelog.d/v2-countries-table.added.md b/changelog.d/v2-countries-table.added.md new file mode 100644 index 000000000..5c7e480ac --- /dev/null +++ b/changelog.d/v2-countries-table.added.md @@ -0,0 +1 @@ +Added a v2 `countries` table that records, for each country the Public API serves, whether it supports household and society-wide calculations. Its revision lists the US and the UK, both with both. Users' primary country and the country of every v2 household, policy and user association now reference this table instead of a fixed US-and-UK check. diff --git a/changelog.d/v2-household-calculations.added.md b/changelog.d/v2-household-calculations.added.md new file mode 100644 index 000000000..9a4a83ccc --- /dev/null +++ b/changelog.d/v2-household-calculations.added.md @@ -0,0 +1 @@ +`POST /v2/household-calculations` sends a stored household, under a stored policy or current law, with optional axes, to the simulation service for calculation, and `GET /v2/household-calculations/{calculation_id}` returns its status and, once it succeeded, the calculated household with its warnings and, in the US, the SPM selection applied and its receipt. Neither needs an account; a country whose `countries` row does not allow household calculations is refused with 422. diff --git a/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md new file mode 100644 index 000000000..03f5b55f3 --- /dev/null +++ b/changelog.d/v2plan-1f-115-catalog-metadata-fields.added.md @@ -0,0 +1 @@ +Catalog format 2 stores the metadata the app's v3 workspace reads, taken only from PolicyEngine.py's public records: each model version's entities and basic inputs; each variable's input and hidden-input flags, module, the parameter path its adds or subtracts names, its enum values' labels, its unit and its definition period; each parameter's economy and household flags. Bracket schedules are stored as parameter nodes with their stored label. A generated revision adds the columns as nullable, so catalog versions published earlier keep them empty and are compared without them when republished. An opt-in parity test, run by the v2 integration check, compares every new field for the US and the UK with /{country}/metadata. Each country's default dataset now comes from the installed PolicyEngine.py release's manifest; the Public API no longer fixes its own (populace_us_2024 and microcosm_uk_2024_25), and publication stops if the release names no default or does not certify it. diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md new file mode 100644 index 000000000..bbfc742cb --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.added.md @@ -0,0 +1,5 @@ +Every successful v2 metadata response carries a strong ETag (a hash of its body) and a public Cache-Control lifetime: a day when the request names its catalog with policyengine_version, five minutes for the running version, and never past the UTC midnight after the request for current parameter values (current=true). A request whose If-None-Match names the ETag gets 304 with no body. These responses and their 304s carry Vary: Origin, Accept-Encoding, each value once, and browser scripts can read the ETag. +GET /v2/model-catalog returns only the small parts of one catalog version (model, model version, entities, basic inputs, regions, logical input datasets and economy choices, which are empty lists for a catalog without regions) and names the selected PolicyEngine.py version, so a client can request every other part for that exact version. +GET /v2/model-catalog/variables returns every variable of a catalog version in one response, with the catalog format 2 fields (input and hidden-input flags, module, the parameter path an adds or subtracts list comes from, enum labels, unit and definition period), and the current-law values by date of each parameter such a list comes from. GET /v2/variables pages carry the same variable fields. +GET /v2/model-catalog/parameters fetches up to 100 parameters of a catalog version by name in one request, each with its economy and household flags and its current-law values by date, plus the stored labels of their ancestor nodes for breadcrumbs and the requested names the version lacks. /v2/parameters pages and details carry the economy and household flags. +Each API process keeps the serialized /v2/model-catalog and /v2/model-catalog/variables bodies and their ETags once built (up to eight route, country and version combinations), so only the first successful request per process reads them from the database; requests arriving during that read receive its outcome. diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md new file mode 100644 index 000000000..553c9f234 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.changed.md @@ -0,0 +1 @@ +Every v2 metadata route reads its query through the shared query models, so unknown or repeated parameters, a missing or unsupported country_id and a malformed policyengine_version return 422 before any read (unsupported countries and malformed versions returned 400 before, and unknown parameters were ignored). /v2/parameters accepts repeated name keys and /v2/parameter-values repeated parameter_name keys to page only those parameters. diff --git a/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md new file mode 100644 index 000000000..4c325f3a5 --- /dev/null +++ b/changelog.d/v2plan-1f-116-catalog-routes-on-demand.fixed.md @@ -0,0 +1 @@ +v2 metadata reads run in autocommit sessions on the shared pool, and each process keeps the catalog versions it has selected. A request that makes one read for a version the process has already selected, such as a page of parameters or one parameter's values, now sends two statements to the database (the pool's liveness check and the read) instead of five; each further read in a request adds one statement. diff --git a/changelog.d/v2plan-1f-121-repoint-saved-items.added.md b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md new file mode 100644 index 000000000..14b849f5b --- /dev/null +++ b/changelog.d/v2plan-1f-121-repoint-saved-items.added.md @@ -0,0 +1 @@ +PATCH /v2/user-policies/{id} accepts policy_id and PATCH /v2/user-households/{id} accepts household_id, so a saved policy or household can point at another v2 record in its country while keeping its ID, name and creation time. Supplied alone, the v2 reference clears the v1 reference (legacy_policy_id or legacy_household_id); supplied with it, both are stored, and the v2 reference must be the v2 copy of that v1 record when one exists. Unknown, other-country and conflicting records are refused as on creation, and saved policies maintained by the v1 saved-policy mirror still return 409. diff --git a/changelog.d/v2plan-1f-122-rename-user-policies.changed.md b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md new file mode 100644 index 000000000..9c9a77ad8 --- /dev/null +++ b/changelog.d/v2plan-1f-122-rename-user-policies.changed.md @@ -0,0 +1 @@ +The v2 table user_policies is renamed to user_policy_associations, matching the other user-association tables; its constraints and convention-named indexes take the new name. Saved policies, their v1 mappings and the reports that name them are kept. A temporary view named user_policies, with the runtime role's privileges, lets the release from before the rename keep reading and writing saved policies during the deploy and after a rollback; a later change removes it. The privilege reconciliation now finds, grants and verifies privileges on views. The route /v2/user-policies, the user_policy_id columns, legacy_user_policy_mappings and the v1 Cloud SQL table user_policies are unchanged. diff --git a/changelog.d/v2plan-1f-123-region-reference-files.changed.md b/changelog.d/v2plan-1f-123-region-reference-files.changed.md new file mode 100644 index 000000000..2ec0371e0 --- /dev/null +++ b/changelog.d/v2plan-1f-123-region-reference-files.changed.md @@ -0,0 +1 @@ +Catalog publication reads regions from the Public API's checked-in reference files, one per country, instead of PolicyEngine.py's region registry. Each region takes its dataset from the certified data release and a row filter from a scoping rule: state_fips or congressional_district_geoid for US states and districts, the UK region column for UK nations, and constituency_code_oa or la_code_oa on the local-area dataset the release routes UK constituencies and local authorities to. The UK now publishes 1,016 regions, with 650 constituencies and 361 local authorities (including Ards and North Down) by ONS code and Northern Ireland as country/northern_ireland; the national regions are "United States" and "United Kingdom". US places are listed in the reference file but left out of the catalog until they can run, and the publication evidence counts them. Before it publishes a catalog version that is not stored yet, the publication command reads the HDF5 metadata of the certified datasets the filters use, from Hugging Face at their pinned revisions, and stops if a filter's column is missing; the seed workflow's publication step receives the UK data read token as HUGGING_FACE_TOKEN for this. A catalog version that is already stored keeps the regions it was published with, so region file changes take effect in the next catalog version; republishing it logs the difference in region rows instead of failing, its evidence counts the stored regions and lists the kept version under kept_regions, and every other table must still match. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..0398c7a94 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 16 | -| request count | 89 | +| workflow count | 18 | +| request count | 95 | | db entity count | 6 | | sim flow count | 3 | @@ -18,9 +18,9 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | -| `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | +| `household` | `household`, `households`, `household-calculations`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations`, `user-simulations` | `simulation` | `economy` | | `report` | `report`, `user-reports`, `shared-reports` | `report` | `report` | @@ -100,6 +100,16 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/households/{household_id}?country_id=us` | 200 | `household` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.default_year`, `result.item.household_data`, `result.item.created_at`, `result.item.updated_at` | | `GET` | `/v2/households?country_id=us` | 200 | `household` | none | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +### `household_calculations_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Household calculations on v2 + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `POST` | `/v2/household-calculations` | 202 | `household` | none | `status`, `message`, `result.calculation_id`, `result.country_id`, `result.household_id`, `result.policy_id`, `result.year`, `result.catalog_version_id`, `result.status`, `result.created_at` | +| `GET` | `/v2/household-calculations/{calculation_id}` | 200 | `household` | none | `status`, `message`, `result.calculation_id`, `result.status`, `result.completed_at`, `result.result.household`, `result.result.warnings`, `result.result.spm_config`, `result.result.spm_provenance`, `result.result.policyengine_version`, `result.error` | + ### `user_household_associations_v2` - Current contract: `typed_v2_resources` @@ -190,6 +200,15 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/us/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | | `GET` | `/uk/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | +### `country_selection_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Countries table and one country list in code + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `GET` | `/v2/countries` | 200 | `metadata` | none | `status`, `message`, `result.items` | + ### `metadata_resources_v2_preview` - Current contract: `typed_v2_resources` @@ -215,6 +234,9 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/regions/by-code/state/ca?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.item` | | `GET` | `/v2/tax-benefit-models/by-country/us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.model`, `result.model_version` | | `GET` | `/v2/economy-options?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.current_law_id`, `result.region`, `result.time_period`, `result.datasets` | +| `GET` | `/v2/model-catalog?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.entities`, `result.basic_inputs`, `result.regions`, `result.datasets`, `result.economy_options`, `result.spm` | +| `GET` | `/v2/model-catalog/variables?country_id=us` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.variables`, `result.part_parameter_values` | +| `GET` | `/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1` | 200 | `metadata` | none | `status`, `message`, `result.policyengine_version`, `result.catalog_format`, `result.parameters`, `result.node_labels`, `result.missing_names` | ### `simulation_submit_poll` diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index d4a043d91..3b1926a1c 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -155,8 +155,10 @@ check the declarations: `apply` grants each declared privilege a role lacks. It never revokes: it reports privileges the file doesn't declare, and the projects' default -privileges, in the job summary. It grants nothing when a declared role or -table is missing. The roles themselves are provisioned outside the +privileges, in the job summary. It grants nothing when a declared role, +table or view is missing; a declared table must exist as a table and a +temporary view (`TEMPORARY_VIEW_TABLES`) as a view, so a relation of the other +kind counts as missing. The roles themselves are provisioned outside the repository; `--create-missing-roles` is accepted only for the disposable test database. @@ -179,6 +181,7 @@ When a pull request adds or changes a v2 table, or a connector's use of one: uv run python scripts/reconcile_v2_privileges.py apply --create-missing-roles uv run python scripts/reconcile_v2_privileges.py verify uv run pytest -q tests/integration/test_v2_user_identity_persistence.py \ + tests/integration/test_v2_saved_policy_view.py \ tests/integration/test_v2_stage12_comparison_persistence.py \ tests/integration/test_v2_privilege_reconciliation.py ``` @@ -294,3 +297,78 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### API v2 saved-policy table rename + +Revision `fa7fa1035444` was autogenerated after renaming the `UserPolicy` +model's table from `user_policies` to `user_policy_associations`, with its +constraints and convention-named indexes. Autogeneration cannot detect a +rename: it created an empty `user_policy_associations` table and dropped +`user_policies`, which would delete every saved policy, and its copies of the +three composite indexes collided with the old table's indexes of the same +names. Its only post-generation correction replaces that pair with +`op.rename_table` and, for each constraint the generated table named after +`user_policy_associations` (primary key, unique, three checks and two foreign +keys), drops it under its old name and creates it under the generated name +with the generated definition. The generated drops and re-creations of the +two convention-named indexes and of the three foreign keys that refer to the +table are kept, ordered around the rename. The downgrade is the exact reverse. +The v2 Postgres lifecycle test stores a saved policy with a v1 mapping and a +report that names it, and checks that the rows, the references, the +cascades, a table grant and every constraint and index name survive the +upgrade and the downgrade. + +The revision also holds the only approved hand-written exception in the v2 +chain, approved by a maintainer on 2026-10-22: after the rename it creates a +view named `user_policies` with every column of `user_policy_associations`, +and the downgrade drops the view before renaming the table back. The Public +API release still serving during a deploy, or after a rollback to it, names +the table `user_policies`; a single-table view is automatically updatable, so +that release keeps reading and writing saved policies through it. The +privilege manifest declares the view with the table's privileges +(`TEMPORARY_VIEW_TABLES` in `policyengine_api/data/v2/privileges.py`), and the +privilege reconciliation grants and verifies privileges on views as on tables. +Autogeneration does not reflect views, so `alembic check` and the head schema +validation ignore it. A later revision drops the view once no deployable +release names `user_policies`; until then, a revision cannot drop or change +the type of a `user_policy_associations` column the view shows without +dropping the view first. + +Tests: `test_alembic_v2.py` allows exactly these two statements and no other +`op.execute` in any v2 revision. The lifecycle test checks the view's columns +after the upgrade and its absence after the downgrade, and runs the previous +revision's inserts, reads, updates and deletes through it. +`test_v2_saved_policy_view.py` runs today's saved-policy services, the +sign-in merge, report creation and the Stage 10 saved-policy copy as the +runtime role with the table named `user_policies`, so every statement goes +through the view. + +### API v2 countries rows + +Revision `fb1a3e40d3d6` was autogenerated after adding the `countries` table +and replacing the US-and-UK checks on `users.primary_country` and on the +`country_id` of households, policies and the four user association tables +with foreign keys to it. It contains one hand-written operation: it inserts +the `uk` and `us` rows, both household and society-wide capable, between the +generated table creation and the generated foreign keys, which otherwise fail +on any database whose rows already name either country. The rows are exactly +the values the replaced checks allowed. The generated downgrade removes them +with the table, and its restored checks fail if any record names another +country. + +Inserting rows is a data operation, which this skill otherwise forbids. The +maintainers approved this insert as a one-off exception, because the new +foreign keys need the rows in the same revision. Unit tests pin the +operation, and the v2 Postgres lifecycle test covers an upgrade that keeps +existing rows, the downgrade, a downgrade refused by another country's +records, and the re-upgrade. + +The table must list exactly the countries in +`policyengine_api/supported_countries.py`: the Public API refuses to start +when they differ (`policyengine_api/startup_country_check.py`). Listing +another country later needs a revision that inserts its row, and the same +human migration decision. The deploy runs that revision before it starts +the new release, so instances of the previous release that start afterwards, +and a rollback to it, refuse to start until the revision is downgraded. The +module docstring of `supported_countries.py` lists the other checks a new +country must widen. diff --git a/docs/engineering/skills/v2-code-organization.md b/docs/engineering/skills/v2-code-organization.md index bbe997b65..b32fe2af1 100644 --- a/docs/engineering/skills/v2-code-organization.md +++ b/docs/engineering/skills/v2-code-organization.md @@ -21,9 +21,18 @@ user_policies/ metadata/ response_models.py common.py + query_dependencies.py + response_cache.py *_routes.py ``` +`common.py` also holds the route class that gives metadata responses their +caching headers. `query_dependencies.py` defines the typed FastAPI +dependencies for the shared query models once, for every metadata route +module to import. `response_cache.py` keeps serialized whole-version +responses (bodies and ETags) for the process; it is HTTP-level, so it lives +with the routes. + `errors.py` defines the strict error response and serialization function shared by every API v2 resource. Application-wide exception handling distinguishes API v2 from non-v2 requests but must not select error behavior by matching @@ -52,6 +61,13 @@ services/v2// deletes.py ``` +A resource may also add a module for process-wide state its service reuses +across requests, named for what it keeps, such as the metadata package's +`selected_catalogs.py` (each catalog version a request has selected, which +never changes once published). Such a module holds plain data, constructs no +SQL and owns no session; the composition code creates one instance per process +and injects it into the service. + Only add CRUD connector modules for operations the resource supports. Append a specific descriptor when one operation file would become too broad, such as `reads_variables.py` or `reads_parameter_tree.py`. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md index 1bc1c6134..295be325c 100644 --- a/docs/engineering/uk-private-hugging-face-credential.md +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -51,6 +51,14 @@ Cloud Run candidate resolver verifies that the exact deployed revision binds `HUGGING_FACE_TOKEN` to `pe-uk-private-hf-read-token:latest`, without printing the credential. +The v2 catalog publication step of `.github/workflows/seed-v2-database.yml` +also receives the organization secret as `HUGGING_FACE_TOKEN`, step-scoped. +Before it publishes a new catalog version, the step reads, through one +authenticated Hugging Face file system, the HDF5 metadata of the certified +datasets that version's region filters use: the UK national and local-area +datasets in the private UK data repository, and the public US dataset. It +downloads only the 1 MB blocks that hold that metadata, not whole files. + The simulation-entry Cloud Run service does not receive this credential. It submits requests to Modal and does not download model data. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..22de0e6a6 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 95, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 16 + "workflow_count": 18 }, "route_groups": [ { @@ -31,8 +31,10 @@ "name": "metadata", "path_segments": [ "metadata", + "countries", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", @@ -59,6 +61,7 @@ "path_segments": [ "household", "households", + "household-calculations", "user-household", "user-households", "calculate", @@ -536,6 +539,52 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Household calculations on v2", + "name": "household_calculations_v2", + "requests": [ + { + "expected_status": 202, + "headers": [], + "method": "POST", + "path": "/v2/household-calculations", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.calculation_id", + "result.country_id", + "result.household_id", + "result.policy_id", + "result.year", + "result.catalog_version_id", + "result.status", + "result.created_at" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/household-calculations/{calculation_id}", + "route_group": "household", + "stable_response_fields": [ + "status", + "message", + "result.calculation_id", + "result.status", + "result.completed_at", + "result.result.household", + "result.result.warnings", + "result.result.spm_config", + "result.result.spm_provenance", + "result.result.policyengine_version", + "result.error" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "PR 12A: User Login and v1-Anchored User Associations", @@ -1348,6 +1397,25 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Countries table and one country list in code", + "name": "country_selection_v2", + "requests": [ + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/countries", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.items" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "Later metadata read cutover and v2 route-prefix removal", @@ -1614,6 +1682,56 @@ "result.time_period", "result.datasets" ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options", + "result.spm" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/variables?country_id=us", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/model-catalog/parameters?country_id=us&name=gov.irs.income.bracket.rates.1", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names" + ] } ] }, diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 2d00bc6b6..69fb411e5 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -164,7 +164,7 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `api.catalog.initializer-not-at-startup` | Application startup modules never import the metadata catalog initializer. | mechanical | enforced | yes | pytest `tests/unit/v2/test_catalog_initialization.py` | `pr.yml:test` | **unproven** | | `api.startup.fail-closed-configuration` | Default Cloud SQL startup needs no Supabase configuration, and a selected v2 runtime without its database URL fails closed. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | -| `api.startup.no-import-side-effects` | Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | +| `api.startup.no-import-side-effects` | Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | | `api.startup.no-runtime-sqlite-modules` | Runtime SQLite modules are absent from the production package. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | ```mermaid @@ -294,7 +294,7 @@ flowchart LR | `api.quality.command-line-modules-ruff-exemptions` | Application modules that also run as command-line programs (if __name__ == "__main__") may print their output (T201); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of T201 for `policyengine_api/{data/v2/catalog/initialization,data/v2/household_migration_qualification,data/v2/policy_migration_qualification,data/v2/privilege_reconciliation,data/v2/user_association_migration_qualification,worker_spm_release}.py` | `pr.yml:lint` | — | | `api.quality.command-line-scripts-ruff-exemptions` | Command-line scripts under scripts/ may print their output, run other programs, fetch fixed URLs, stamp today's date and pick random samples (T201, S603, S607, S310, DTZ011, S311); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of DTZ011, S310, S311, S603, S607, T201 for `scripts/**` | `pr.yml:lint` | — | | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — | -| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | +| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `api.quality.ruff-lint` | All Python code passes ruff check with every ruff rule selected (ALL), with no ratchet, except the ignores in pyproject.toml: COM812 and ISC001, which conflict with ruff format, and D203 and D213, which conflict with the kept D211 and D212. | mechanical | enforced | yes | ruff check (ALL) | `pr.yml:lint` | — | | `api.quality.runtime-evaluated-models-resolve` | Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. | mechanical | enforced | yes | pytest `tests/unit/test_runtime_evaluated_annotations.py` | `pr.yml:test` | **unproven** | diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index a69209d24..4815630e6 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -259,6 +259,21 @@ not block a direct-Modal deployment. The legacy `SIMULATION_API_URL` secret is intentionally unsupported: its value is opaque, cannot be audited, and must not silently choose a deployment upstream. +Each environment calls its own Simulation Entrypoint. The `staging` secret +holds the stable URL of the `policyengine-simulation-entry-staging` Cloud Run +service, which calls the staging Modal environment; the `production` secret +holds `https://simulation.api.policyengine.org`, served by +`policyengine-simulation-entry`. In each release, the staging and production +model-version alignment jobs first run +`.github/scripts/check_simulation_entrypoint_environment.sh`. It reads the +`x-policyengine-simulation-revision` header from the selected URL's `/health` +response and fails the release unless the revision belongs to that +environment's service, so a release whose `staging` secret points at +production, or the reverse, stops before it deploys. Staging's live simulation +tests therefore run against the staging simulation service and staging Modal +workers. The old gateway is one shared Modal app that reports no environment, +so `old_gateway_direct` is not checked. + ## IAM and bootstrap constraints - The GitHub deploy service account holds `roles/run.developer`: it can deploy to diff --git a/docs/migration/stage-11-v2-households.md b/docs/migration/stage-11-v2-households.md index c9a68eb1c..701673f0b 100644 --- a/docs/migration/stage-11-v2-households.md +++ b/docs/migration/stage-11-v2-households.md @@ -256,3 +256,97 @@ Apply the reviewed schema revisions and application image only to independently verified production targets. Enable native v2 use and v1 create copying as separate operational decisions and retain the preceding application revision for immediate rollback. + +## Household Calculations + +`POST /v2/household-calculations` calculates a household stored through +`/v2/households`, under a policy stored through `/v2/policies` or, without +`policy_id`, under current law, for the household's `default_year`. The body +may add `axes` (an axis without a `period` varies the household's +`default_year`) and `add_missing_variables`. An identical calculation that +succeeded in the last 28 days is returned again; callers cannot force a new +one. The +Public API checks the country's `countries` row allows household calculations +(422 otherwise; a household-only country is accepted), checks the household +and policy exist through the same services as their routes (404 when either +is not stored in that country), selects the country's running catalog +version, and sends the simulation service the household's and policy's IDs, +the options and that version's PolicyEngine.py version. The simulation +service reads the household, the policy's parameter values and the catalog +version from these tables itself, through its own `SELECT` grant. The +answer is 202 with the calculation's `calculation_id` and `status`. Poll `GET /v2/household-calculations/{calculation_id}` until +`status` is `succeeded`, with `result`, or `failed`, with `error`. A US result +carries `spm_config` and `spm_provenance`. + +The simulation service sends the household function the household's +`default_year` as the calculation's year: the year of the variables +`add_missing_variables` adds, and of values and axes the household gives +without a period. (`/calculate-full` takes that year from the first age +period of a person named `you`, or else the current year.) + +The simulation service stores every calculation as a simulation run, so a +calculation no account saved is still stored, with no report. Neither route +needs an account. + +The routes always call the Cloud Run Simulation Entrypoint, whatever +`SIM_ENTRYPOINT` selects, because only it serves v2 calculations. They read: + +- `SIMULATION_ENTRYPOINT_URL`: the `staging` and `production` environment + secret of the same name, passed to the deployment job's `env:` and written + to the Cloud Run revision by `deploy_cloud_run_candidate.sh` when set. +- `GATEWAY_AUTH_ISSUER`, `GATEWAY_AUTH_AUDIENCE`, `GATEWAY_AUTH_CLIENT_ID` and + `GATEWAY_AUTH_CLIENT_SECRET_RESOURCE`: the organization secrets already used + for every simulation call, with `GATEWAY_AUTH_REQUIRED=1`. + +Without the URL, or with gateway auth partly configured, both routes answer +503. They also answer 503 when the simulation service cannot be reached, its +gateway token cannot be fetched, it answers with an error or with a body of +another shape, or it refuses the calculation (for example when no v2 worker is +deployed for the catalog version's PolicyEngine.py version, or when it +cannot find a household, policy or catalog version the Public API has just +found, which means the two services disagree). The simulation service checks +every stored household and policy value against the household functions' +contract before it writes anything, and the caller's data it refuses gets +422 with its reason: a policy changing a parameter the catalog version +lacks, a stored policy value it cannot take (such as an object), or a stored +household it cannot take (such as a group no person belongs to, which +`/v2/households` accepts). A request field it refuses, such as an invalid +option, gets 422 with each refused field and why. + +### Staging checks + +The parallel live staging job (`run_live_staging_tests.sh`) runs two modules +against each tagged staging candidate, through staging API v1, the staging +Simulation Entrypoint, the v2 household functions on Modal and the staging v2 +database: + +- `test_live_v2_household_calculations.py`: a submit, poll and result cycle + for a new US household; six calculations submitted at once, each accepted + within 10 seconds and finished within 300 seconds; and one household varied + across 401 points of employment income, which must finish. The measured + times appear in the job log as `HouseholdCalculationMeasurement` warnings. +- `test_live_v2_household_parity.py`: three US and three UK households, one + of each with axes and one of each under a reform stored through + `/v2/policies` (the US one also adding missing variables), calculated + through `/v2/household-calculations` and through API v1's `/calculate` + (`/calculate-full` when adding missing variables); every calculated value + and warning must match. + +A calculation's submission and each poll are sent again while they get 502, +503 or 504 for at most 90 seconds in a row; any other failure, or a longer +run of such answers, fails the check with the answer's body. The Public API +also answers 503 for failures that do not pass, such as a simulation service +without these routes, so those take about 90 seconds to fail. Every other +request in the checks fails at once. Warnings are compared regardless of order, because the v2 +database orders each stored record's keys itself; one US household has a +deprecated input, so both calculations warn. + +`STAGING_HOUSEHOLD_PARALLEL_COUNT`, `STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`, +`STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS`, +`STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS` (600 seconds per calculation) +and `STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS` change those limits; none is +set in the workflow. The checks see neither the +number of Simulation Entrypoint instances that served the calculations nor +the household functions' memory use: a calculation that runs out of memory +fails, and so fails the check. Both modules fail until the staging Simulation +Entrypoint serves `/v2/household-calculations`. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index c01657bc3..ed952e189 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -126,7 +126,7 @@ unauthenticated: a standard user's row is never locked. | State | Outcome | What happens | | --- | --- | --- | | The identity has no user yet; the browser's unauthenticated user exists | `converted` | The `user_identities` row (issuer, subject, verified email) is inserted pointing at the browser's `users.id`; the user becomes `standard` with `primary_country = country_id`. The ID is unchanged, so every association stays owned. | -| The identity already belongs to standard user S; the unauthenticated user exists | `merged` | Every row of `user_policies`, `user_household_associations`, `user_simulation_associations`, and `user_report_associations` owned by the unauthenticated user moves to S with its ID, contents, and timestamps unchanged; then the unauthenticated user is deleted. | +| The identity already belongs to standard user S; the unauthenticated user exists | `merged` | Every row of `user_policy_associations`, `user_household_associations`, `user_simulation_associations`, and `user_report_associations` owned by the unauthenticated user moves to S with its ID, contents, and timestamps unchanged; then the unauthenticated user is deleted. | | No unauthenticated user with that ID | `none` | The identity's user is resolved, or provisioned, as on any request. | | The ID names a standard user (already converted, or anyone else's) | `none` | That user is never touched; the identity's user is resolved as on any request. | @@ -183,9 +183,19 @@ Concurrency on Postgres: `legacy_policy_mappings` / `legacy_household_mappings` when a mapping exists. - Changing the legacy ID re-resolves the v2 reference, or clears it. + - `PATCH /v2/user-policies` also accepts `policy_id`, which must name a v2 + policy in the row's country. Supplied alone, it points the row at that + policy and clears `legacy_policy_id`, so an edited reform keeps its saved + ID, name and creation time. Supplied with `legacy_policy_id`, both are + stored, and `policy_id` must be the mirror copy of that v1 policy when + one exists. + - `PATCH /v2/user-households` accepts `household_id` the same way: it must + name a v2 household in the row's country; alone, it clears + `legacy_household_id`; with `legacy_household_id`, both are stored, and + it must be the copy of that v1 household when one exists. - `user-simulations` and `user-reports` accept only legacy IDs in 12A. -- **Stage 10 mirror rows.** A `user_policies` row with a - `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The +- **Stage 10 mirror rows.** A saved-policy row (`user_policy_associations`) + with a `legacy_user_policy_mappings` entry returns 409 on PATCH and DELETE. The mirror projection now also records `legacy_policy_id`. - **Request bounds.** Every `/v2/me*` and `/v2/user-*` write rejects a body over 256 KiB with 413. FastAPI reads and parses the whole JSON body before @@ -314,7 +324,16 @@ from the final SQLModel metadata. It has no post-generation corrections. "Rollback"). The models drop the `user_associations` relationships from v2 simulations and -reports. `V2_ALEMBIC_HEAD_REVISION` is `fbb7488ebe8a`. +reports. `V2_ALEMBIC_HEAD_REVISION` was `fbb7488ebe8a` when Stage 12A shipped. + +Revision `fa7fa1035444` later renamed `user_policies` to +`user_policy_associations`, with its constraints and its convention-named +indexes, matching the other association tables; the composite listing +indexes keep their `ix_user_policies_*` names. The route `/v2/user-policies`, +the `user_policy_id` columns and `legacy_user_policy_mappings` are unchanged. +The bullet points above describe revision `fbb7488ebe8a`, when the table was +still named `user_policies`. "Schema compatibility" below says what the +rename means for earlier Public API releases. ### Reshape safety @@ -337,15 +356,19 @@ and production and confirm both dormant tables are empty. ### Rollback -Roll forward whenever possible. The schema lets the previous Public API -revision run (see "Schema compatibility" below), but that revision serves -`/v2/user-policies` and `/v2/user-households` without authentication: any -caller can list a user's saved items by `user_id`, read, rename or delete one -by record ID, and create rows for any user. Rolling the application back -would therefore hand users' saved work to anyone. +Roll forward whenever possible. Since revision `fa7fa1035444` (see +"Schema compatibility" below), releases from before that revision use saved +policies through the temporary view `user_policies` until a later revision +drops it; after that, rolling the application back to such a release also +needs revision `fa7fa1035444` downgraded, which keeps every row (see below). +The release before Stage 12A also serves `/v2/user-policies` and `/v2/user-households` +without authentication: any caller can list a user's saved items by +`user_id`, read, rename or delete one by record ID, and create rows for any +user. Rolling the application back would therefore hand users' saved work to +anyone. - **Before any user-owned record exists, an application rollback is safe - without a database downgrade.** Signed-out saving needs no configuration, so + without a Stage 12A database downgrade.** Signed-out saving needs no configuration, so treat user-owned records as existing from the moment this revision serves traffic in an environment. - **Once any user-owned record exists, signed in or not, an application @@ -379,17 +402,36 @@ would therefore hand users' saved work to anyone. `/v2/user-policies` and `/v2/user-households` responses require a non-null `policy_id` / `household_id`, so any response that includes a legacy-only row (a v1 reference with no v2 copy) fails with 500. -- **A database downgrade is lossy.** It restores NOT NULL on `policy_id`, - `household_id`, `simulation_id`, `report_id`, `country`, and - `users.primary_country`, so it fails while legacy-only rows, any simulation - or report associations, or any unauthenticated user exist. Removing them - first deletes users' saved work. +- **Downgrading the rename alone is not lossy.** Downgrading revision + `fa7fa1035444` drops the temporary view and renames + `user_policy_associations` back to `user_policies` with every row, + reference and table grant. +- **A database downgrade of Stage 12A is lossy.** Downgrading revision + `fbb7488ebe8a` restores NOT NULL on `policy_id`, `household_id`, + `simulation_id`, `report_id`, `country`, and `users.primary_country`, so it + fails while legacy-only rows, any simulation or report associations, or any + unauthenticated user exist. Removing them first deletes users' saved work. ### Schema compatibility -The revision is additive (expand-only) for everything the previous Public API -revision maps: it drops no table or column, and it adds no required column -without a server default to any table that revision writes. The only new +**Since revision `fa7fa1035444`, earlier releases use saved policies through a +temporary view.** That revision renames `user_policies` to +`user_policy_associations` and creates a view named `user_policies` with every +column of the renamed table, the approved hand-written exception described in +`docs/engineering/skills/alembic-migrations.md` ("API v2 saved-policy table +rename"). A single-table view is automatically updatable, and the runtime role +holds on it what it holds on the table, so a release from before the rename +keeps reading and writing saved policies, during a deploy and after a +rollback: its `/v2/user-policies` routes, report creation that checks the saved +policies a report names, the sign-in merge, and the Stage 10 saved-policy +copy. `test_v2_saved_policy_view.py` runs those operations as the runtime role +through the view. Once a later revision drops the view, such a release fails +every statement on saved policies. + +The rest of this section describes revision `fbb7488ebe8a` alone. It was +additive (expand-only) for everything the previous Public API revision maps: +it drops no table or column, and it adds no required column without a server +default to any table that revision writes. The only new required columns are `country_id` on the simulation and report associations, which that revision maps but never inserts into, and `users.user_type`, whose server default makes every user the previous revision creates (its Stage 10 @@ -402,7 +444,9 @@ row with a NULL `primary_country` does not fail there. The lifecycle test `test_stage_12a_schema_serves_the_previous_revision` exercises this rather than only comparing columns. It reflects the schema at -the parent revision with SQLAlchemy and upgrades to head. Then, through the +the parent revision with SQLAlchemy and upgrades to head. Since the rename, +the parent's `user_policies` is head's temporary view of that name, so the +parent's statements on saved policies run through the view. Then, through the reflected parent tables, it does the following for every table the parent revision maps: diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8ee9dc4b4..de87f336f 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -27,6 +27,138 @@ package-derived `DatasetVersion` rows or simulation/report output datasets and does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset references. +## Loading a catalog version on demand + +`GET /v2/model-catalog?country_id=[&policyengine_version=]` +returns only the small parts of one catalog version: the model and model +version (with the current-law policy id and the years offered), the entities +(`key`, `plural`, `is_person`), the basic inputs, every region and logical +input dataset, and the economy choices. A catalog without regions, such as a +household-only country's, has empty region, dataset and economy-choice lists +rather than an error. A version published before catalog format 2 has a null +`catalog_format` and empty entities and basic inputs. `spm` says whether the +version offers SPM settings for a household: `available`, and when it does, +`settings_schema` (the JSON schema of a household's `spm` selection) and +`defaults` (the certified selection). Catalog publication records it from the +simulation service's public `GET /versions`: a US version offers SPM when the +service lists a canonical capability with pinned defaults for its +PolicyEngine.py version, and other countries never do. A US version the +service does not serve yet stays unrecorded until a later publication run, and +a version whose availability is not recorded yet offers none. An API instance +that read the version before it was recorded keeps that answer until it +restarts, normally at the deployment after the next publication run, and a +client may reuse it for up to a day after that. The result names the +selected PolicyEngine.py version, so a client that starts without a version +learns the running one and requests every other part for that exact version. +The query uses the shared `CatalogQuery` model: unknown and repeated +parameters are rejected. + +`GET /v2/model-catalog/variables` (same query) returns every variable of the +version in name order, in one response, with the catalog format 2 fields: +`is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path`, +`subtracts_parameter_path`, `enum_values` (each value's name and its label, +in the type the country package declares), `unit` (for example +`currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which +the household builder and breakdown read (currency formatting, per-period +labels and household validation). Where a variable's `adds` or `subtracts` list +comes from a parameter, `part_parameter_values` maps that +parameter's path to its current-law values in start-date order +(`start_date`, `end_date`, `value`), so a client picks the list for the year +it shows instead of the list resolved on 2025-01-01; a path whose parameter +the version lacks maps to an empty list. `GET /v2/variables` pages carry the +same variable fields. + +`GET /v2/model-catalog/parameters` takes the same query plus one or more +`name` keys (at most 100, each an exact parameter path) and returns those +parameters in name order, each with its `economy` and `household` flags and its +current-law values in start-date order, in one response: a client showing or +editing a reform loads all of its parameters at once. `node_labels` maps each +ancestor of a returned parameter that the version stores as a parameter node +to its stored label (null when it has none), for breadcrumbs; a bracket index +such as `rates[0]` counts as a child of its schedule `rates`. `missing_names` +lists requested names the version has no parameter for, so a reform saved +against an older version still opens. `GET /v2/parameters` pages and details +carry the same `economy` and `household` flags. + +`GET /v2/parameters` also accepts repeated `name` keys (at most 100) to page +only those exact parameters, and `GET /v2/parameter-values` repeated +`parameter_name` keys to page only those parameters' values. + +Every v2 metadata route reads its query through the shared query models in +`policyengine_api.query_parameters` (`CatalogQuery`, `CatalogPageQuery` and the +resource-specific models built on them): unknown parameters, repeated scalar +parameters, a missing or unsupported `country_id`, a malformed +`policyengine_version` and out-of-range `offset` or `limit` values are +rejected with HTTP 422 before any read. Before, the routes other than the +`/v2/model-catalog` ones ignored unknown parameters, and an unsupported country +or malformed version returned HTTP 400 from the read. A country given in the +path (`/v2/tax-benefit-models/by-country/{country_id}`) that is not supported +still returns HTTP 400. + +## Database round trips per request + +Metadata reads use sessions in autocommit mode on the shared v2 pool +(`get_v2_read_session_factory`), which only the metadata reader opens: each +read is one statement on rows a published catalog version never changes, so no +transaction is needed, and the database receives no `BEGIN` before the reads +and no `ROLLBACK` when the session closes. The sessions refuse to flush added +or changed rows (`ReadOnlySessionError`); a statement passed directly to +`execute` is not checked, so the metadata connectors keep to `SELECT`. Each process also keeps every catalog version a request has +selected (`SelectedCatalogStore`), since a published version never changes, so +later requests for the same country and version skip the selection query; +failed selections are not kept. Measured with statement logging in a local +database, a page of 500 parameters or one parameter's values took five +statements per request before (the pool's liveness check, `BEGIN`, the catalog +selection, the read and `ROLLBACK`) and two after, once the process has +selected that version (the liveness check and the read). A route that makes +several reads, such as `/v2/model-catalog` (regions and datasets), sends one +statement per read after the liveness check. The pool's liveness check +(`pool_pre_ping`) is unchanged. + +## Response caching + +Every successful metadata response carries a strong `ETag`, a hash of its +exact body, and `Cache-Control: public`. A request that names its catalog with +`policyengine_version` may be reused for a day (`max-age=86400`): a published +catalog version never changes, so such a response changes only with the +route's response format or with the day it is read on, and the `ETag` lets a +client revalidate after the day. A request without it selects the running +version, which changes on deploy, and may be reused for five minutes +(`max-age=300`). A response that depends on the request's UTC day, which is +`GET /v2/parameter-values?current=true` (the values in effect today), takes +the request's instant once: the read selects that day's values, and the +response may be reused only until the UTC midnight after that instant, +whichever lifetime would otherwise apply (`max-age=0` if its headers are +written after that midnight). A request whose `If-None-Match` names the current `ETag` (strong or +weak form, alone or in a list) gets HTTP 304 with no body. Error responses +carry neither header. Every cacheable response, and every 304, carries +`Vary: Origin, Accept-Encoding`, each value once: the API reflects every +origin in `Access-Control-Allow-Origin` and a request without `Origin` gets no +such header, so a shared cache must keep responses apart by `Origin`, and gzip +compresses bodies for clients that accept it. A 304 carries the same `Vary` as +its 200. The CORS and gzip middleware append to `Vary` without checking for +values already present, so the application's outermost middleware +(`UniqueVaryMiddleware`) lists each value once. `ETag` is +listed in `Access-Control-Expose-Headers`, so +browser scripts on other origins can read it (`Cache-Control` is readable +without that). + +Each API process also keeps the serialized bodies of `/v2/model-catalog` and +`/v2/model-catalog/variables` once built, each with its `ETag` computed once, +for up to eight route, country and version combinations (least recently used +dropped first; the 6.10.0 US variables body is 4,190,824 bytes uncompressed, +so eight entries hold about 12 MB for both countries' two routes at two +versions). A response is kept under the exact version it was read from, +so a request without `policyengine_version` (the process's running version, +which never changes while it runs) and one naming that version share it. Only +the first successful request per process reads the database. Requests that +arrive while that read is in progress wait for it and all receive its outcome, +the kept response or a copy of its error response, so one failing read is not +repeated by each of them; a failed read is not kept, so the first request after +it reads again. Measured locally against the full 6.10.0 US catalog, the +variables route takes about 160 ms when read and about 25 ms when kept, most of +it compression. + ## Configuration and credential boundaries Select the target with `V2_SUPABASE_PROJECT_REF` and @@ -64,8 +196,19 @@ must execute in this order: Alembic chain, confirms all heads are current, and detects ungenerated model changes. 3. Remove the migration URL from the command environment. -4. Supply only `V2_DATA_WRITE_DATABASE_URL` and the same target identity, then - run `uv run python scripts/publish_v2_metadata_catalog.py`. +4. Supply only `V2_DATA_WRITE_DATABASE_URL`, the same target identity, + `HUGGING_FACE_TOKEN` (from the organization secret + `PE_UK_PRIVATE_HF_READ_TOKEN`) and `SIMULATION_ENTRYPOINT_URL` (from the + environment secret of the same name), then run + `uv run python scripts/publish_v2_metadata_catalog.py`. The command stops + before extraction if `SIMULATION_ENTRYPOINT_URL` is unset. After publishing, + it reads the simulation service's public `GET /versions` to record each + version's SPM availability; if the answer cannot be read, it logs a warning, + records none, and the next run records them. Before publishing a + catalog version that is not stored yet, the command reads the certified data + artifacts its regions' filters use and stops if one cannot select a region. + It stops before reading anything if a UK version needs checking and + `HUGGING_FACE_TOKEN` is unset, because the UK data repository is private. 5. Require the command to finish successfully and retain its non-secret JSON evidence before creating the candidate revision. 6. Deploy the Cloud Run candidate with the target identity and @@ -93,6 +236,34 @@ metadata time-period options are stored on the model-version snapshot. PolicyEngine Core and country-package versions are derived diagnostic evidence and are not separately persisted. +Catalog format 2 adds the metadata the app's v3 workspace reads, taken only +from PolicyEngine.py's public records, which carry it from the release that +publishes entities, basic inputs and these flags: + +| Record | Fields | +| --- | --- | +| Model version | `catalog_format` (2), `entities` (`key`, `plural`, `is_person`, in the package's order), `basic_inputs` (the household builder's default fields) | +| Variable | `is_input_variable`, `hidden_input`, `module_name`, `adds_parameter_path` and `subtracts_parameter_path` (the parameter whose dated value lists the variables summed; `adds` and `subtracts` keep the list PolicyEngine.py resolved), `enum_values` (`value` and declared `label`, in order), `unit` (for example `currency-USD`) and `definition_period` (`year`, `month` or `eternity`), which the household builder and breakdown read | +| Parameter | `economy`, `household` | + +Bracket schedules are parameter nodes with their stored label, or none when the +package stores none. An empty basic-input list is stored as empty. Extraction +stops before reading any record if the installed PolicyEngine.py does not +publish these fields, and names each missing field: 6.7.5 and the locked 6.10.0 +publish only the variables' unit and definition period. It also stops if the +model has other than one person entity, names a basic input or a parameter path +that is not published, gives enum labels that do not match the variable's +possible values or are not JSON scalars, or gives a flag that is not a boolean. +An opt-in test (`tests/integration/test_v2_catalog_metadata_parity.py`, run by +the v2 integration check) compares every format 2 field for the US and the UK +with what `/{country}/metadata` builds. + +Publication never rewrites a stored version. A version published before +format 2 has a null `catalog_format` and null format 2 fields, and later +publications of the same PolicyEngine.py version compare it without those +fields. The first PolicyEngine.py version published after format 2 deploys +carries them. + Parameter extraction requires the PolicyEngine.py public history contract from 5.0.4 and later: values are ordered from oldest to newest, bounded `end_date` values are inclusive and fall one day before the following `start_date`, and @@ -101,20 +272,54 @@ does not reorder, deduplicate, or repair package output. A value superseded on the immediately following day therefore has equal start and end dates and is a valid one-day interval. -The US national region uses `populace_us_2024`. A US state, -congressional-district, or place region uses its PolicyEngine.py regional -alternative when one is available. Until a later reviewed change removes the -fallback, a subnational region without such an alternative uses -`populace_us_2024`. The command emits one summary warning containing only the -affected region types and counts. The UK national and country regions use -`microcosm_uk_2024_25`, the default certified by PolicyEngine.py 6.10.0; a -country filters that national file. UK constituency and local-authority -regions use the dataset PolicyEngine.py routes them to, -`microcosm_uk_2024_25_dense`, because the national file has no area codes. - -Review the fallback summary on every new PolicyEngine.py release. A changed -count may identify an upstream catalog change that needs a reviewed dataset -selection even when publication otherwise succeeds. +Regions come from the Public API's checked-in reference files, one per country +(`policyengine_api/data/v2/catalog/region_reference_files/`), not from +PolicyEngine.py's region registry. Each row lists a region's code, label, type, +parent, US state, the value that selects its households, and the v1 identifier +it replaces. Each country's default dataset is the one the installed +PolicyEngine.py release names in its manifest (`default_dataset`); the Public +API sets none of its own. Publication stops if the manifest names no default, +if the release does not certify it, or if the release routes national regions +to another dataset. A scoping rule per country and region type gives each +region its dataset from the certified data release and its filter: + +| Regions | Dataset | Filter | +| --- | --- | --- | +| US national | the release's default dataset (`populace_us_2024` in 6.10.0) | none | +| US states and congressional districts | the release's state and district dataset | `row_filter` on `state_fips` or `congressional_district_geoid` | +| UK national | the release's default dataset (`microcosm_uk_2024_25` in 6.10.0) | none | +| UK nations | the release's default dataset | `row_filter` on `region` | +| UK constituencies and local authorities | the release's local-area dataset (`microcosm_uk_2024_25_dense` in 6.10.0), which carries their codes | `row_filter` on `constituency_code_oa` or `la_code_oa`, value the ONS code | + +If the release names no dataset for a US state or district, those regions use +the default dataset and the command emits one summary warning with the affected +region types and counts. A UK constituency or local authority cannot fall back, +since the national file has no area codes: a release that routes no dataset to +their type stops publication. Region types without a rule, such as US places +until the data can select them, are left out and counted in the evidence's +`omitted_region_summaries`. + +Before a new catalog version is published, the command reads the HDF5 metadata +of each dataset the row filters use, with ranged requests at the pinned +revision, and stops if a filter's column is missing from its dataset's +household columns. It does not check that a column holds rows for every +region's value. + +Review the fallback and omitted-region summaries on every new PolicyEngine.py +release. A changed count may identify an upstream change to the certified data +release even when publication otherwise succeeds. + +A catalog version is identified by its country and PolicyEngine.py version and +never changes once published. When the command republishes a stored version +whose staged regions differ from the stored ones, it keeps the stored regions +unchanged, prints a warning with the country, the version and the counts of +staged region rows the stored version lacks and of stored rows not staged, and +continues. Its evidence then describes what is stored: the region count is the +stored one, `kept_regions` lists the country, version, stored and staged region +counts, and the fallback and omitted-region summaries leave that country out, +since they describe staged regions that were not published. Every other table +must still match exactly. Region reference changes therefore take effect in the +next catalog version that is published. ## Validation evidence and retry @@ -123,7 +328,9 @@ Successful JSON evidence contains: - the canonical PolicyEngine.py version; - the dependency versions derived from its packaged manifest; - catalog entity counts; -- US fallback counts; +- fallback counts and omitted-region counts by country and region type, and + the stored and staged region counts of any stored version whose regions were + kept; - elapsed publication time; and - a success outcome. @@ -142,8 +349,11 @@ confirms existing simulation and report record counts are unchanged. A failed attempt rolls back all catalog changes. Retry the same artifact only after correcting the external failure. A matching retry preserves identifiers, content, and row counts. Different normalized content under the same -PolicyEngine.py version fails for operator review; it is never silently -replaced. A later PolicyEngine.py version adds a catalog version without +PolicyEngine.py version fails for operator review, with an error naming the +first table that differs; it is never silently replaced. Regions are the +exception: they come from the Public API's reference files, so a stored version +keeps the regions it was published with (see "Source and regional dataset +behavior"). A later PolicyEngine.py version adds a catalog version without deleting or modifying earlier versioned rows, including their logical datasets, regions, model descriptions, current-law IDs, and time-period options. @@ -190,13 +400,24 @@ A successful response has HTTP 200, `status: "ok"`, `message: null`, and a typed `result`. Collection results contain `policyengine_version`, `items`, `offset`, `limit`, and `has_more`; detail results contain `policyengine_version` and `item`. A malformed or noncanonical explicit -version returns a typed HTTP 400 error. A valid explicit version that has not +version returns a typed HTTP 422 error. A valid explicit version that has not been published for the country returns a typed HTTP 404 error. An absent catalog for the candidate's installed default version returns a typed HTTP 503 service error. None of these outcomes reads or repairs v1. Unsupported countries and methods return typed client errors. +The same test loads each country's catalog in parts as a client starting +cold would: `GET /v2/model-catalog` for the running version, then +`/v2/model-catalog/variables` for that exact version twice (the second request +may be served by another instance, so its time is not necessarily a kept +response's) and once more with `If-None-Match`, which must return 304, then a +page of 500 parameters, one parameter's values (the two routes measured at +0.8–2.6 s and 0.5–0.65 s in production on 2026-10-21) and that parameter by +name. It appends each request's time and body size to the GitHub job summary, +even when an assertion fails, so every staging and production release records +these route times against its own catalog. + Request `GET /v2/openapi.json` and confirm that the public document contains every documented resource path and explicit component schema references for success and error responses. diff --git a/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py b/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py new file mode 100644 index 000000000..a8722ee1f --- /dev/null +++ b/migrations/v2/versions/aeb5f3e8f662_store_catalog_spm_availability.py @@ -0,0 +1,32 @@ +"""store catalog spm availability + +Revision ID: aeb5f3e8f662 +Revises: fb1a3e40d3d6 +Create Date: 2026-10-27 20:34:25.487436 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "aeb5f3e8f662" +down_revision: str | None = "fb1a3e40d3d6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "tax_benefit_model_versions", + sa.Column("spm_availability", sa.JSON(), nullable=True), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("tax_benefit_model_versions", "spm_availability") + # ### end Alembic commands ### diff --git a/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py new file mode 100644 index 000000000..fbaefb08f --- /dev/null +++ b/migrations/v2/versions/f5c1d9b4685b_store_catalog_format_2_metadata.py @@ -0,0 +1,94 @@ +"""store catalog format 2 metadata + +Revision ID: f5c1d9b4685b +Revises: fa7fa1035444 +Create Date: 2026-10-26 20:13:40.733924 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "f5c1d9b4685b" +down_revision: Union[str, None] = "fa7fa1035444" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("parameters", sa.Column("economy", sa.Boolean(), nullable=True)) + op.add_column("parameters", sa.Column("household", sa.Boolean(), nullable=True)) + op.add_column( + "tax_benefit_model_versions", + sa.Column("catalog_format", sa.Integer(), nullable=True), + ) + op.add_column( + "tax_benefit_model_versions", sa.Column("entities", sa.JSON(), nullable=True) + ) + op.add_column( + "tax_benefit_model_versions", + sa.Column("basic_inputs", sa.JSON(), nullable=True), + ) + op.add_column( + "variables", sa.Column("is_input_variable", sa.Boolean(), nullable=True) + ) + op.add_column("variables", sa.Column("hidden_input", sa.Boolean(), nullable=True)) + op.add_column( + "variables", + sa.Column( + "module_name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True + ), + ) + op.add_column( + "variables", + sa.Column( + "adds_parameter_path", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + ) + op.add_column( + "variables", + sa.Column( + "subtracts_parameter_path", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + ) + op.add_column("variables", sa.Column("enum_values", sa.JSON(), nullable=True)) + op.add_column( + "variables", + sa.Column("unit", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + ) + op.add_column( + "variables", + sa.Column( + "definition_period", + sqlmodel.sql.sqltypes.AutoString(length=32), + nullable=True, + ), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("variables", "definition_period") + op.drop_column("variables", "unit") + op.drop_column("variables", "enum_values") + op.drop_column("variables", "subtracts_parameter_path") + op.drop_column("variables", "adds_parameter_path") + op.drop_column("variables", "module_name") + op.drop_column("variables", "hidden_input") + op.drop_column("variables", "is_input_variable") + op.drop_column("tax_benefit_model_versions", "basic_inputs") + op.drop_column("tax_benefit_model_versions", "entities") + op.drop_column("tax_benefit_model_versions", "catalog_format") + op.drop_column("parameters", "household") + op.drop_column("parameters", "economy") + # ### end Alembic commands ### diff --git a/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py new file mode 100644 index 000000000..d9dea42da --- /dev/null +++ b/migrations/v2/versions/fa7fa1035444_rename_user_policies_to_user_policy_.py @@ -0,0 +1,306 @@ +"""rename user_policies to user_policy_associations + +Revision ID: fa7fa1035444 +Revises: fbb7488ebe8a +Create Date: 2026-10-22 17:07:03.720031 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +revision: str = "fa7fa1035444" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_legacy_user_policy_mappings_association_country"), + "legacy_user_policy_mappings", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_baseline_user_policy_id_use_7740"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_user_policy_id_user_policies"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") + op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") + # Post-generation correction: autogeneration cannot detect a rename. It + # created an empty user_policy_associations table and dropped user_policies, + # which would delete every saved policy (its three composite indexes would + # also have collided with the old table's indexes of the same names). The + # table is renamed in place instead, keeping its rows, grants and composite + # indexes. Each constraint the generated table named after + # user_policy_associations is dropped under its old name and created under + # the generated name with the generated definition. The foreign keys that + # refer to this table were dropped above, before its primary key and unique + # constraint. + op.rename_table("user_policies", "user_policy_associations") + op.drop_constraint("pk_user_policies", "user_policy_associations", type_="primary") + op.create_primary_key( + op.f("pk_user_policy_associations"), "user_policy_associations", ["id"] + ) + op.drop_constraint( + "uq_user_policies_id_country", "user_policy_associations", type_="unique" + ) + op.create_unique_constraint( + "uq_user_policy_associations_id_country", + "user_policy_associations", + ["id", "country_id"], + ) + op.drop_constraint( + "ck_user_policies_baseline", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_baseline"), + "user_policy_associations", + "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", + ) + op.drop_constraint( + "ck_user_policies_country", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + "country_id IN ('us', 'uk')", + ) + op.drop_constraint( + "ck_user_policies_policy_ref", "user_policy_associations", type_="check" + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_policy_ref"), + "user_policy_associations", + "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", + ) + op.drop_constraint( + "fk_user_policies_policy_country", + "user_policy_associations", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_user_policy_associations_policy_country", + "user_policy_associations", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + "fk_user_policies_user_id_users", + "user_policy_associations", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_user_policy_associations_user_id_users"), + "user_policy_associations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + op.f("ix_user_policy_associations_policy_id"), + "user_policy_associations", + ["policy_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_policy_associations_user_id"), + "user_policy_associations", + ["user_id"], + unique=False, + ) + op.create_foreign_key( + "fk_legacy_user_policy_mappings_association_country", + "legacy_user_policy_mappings", + "user_policy_associations", + ["user_policy_id", "country_id"], + ["id", "country_id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_user_policy_id_user_policy_associations"), + "user_report_associations", + "user_policy_associations", + ["user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f( + "fk_user_report_associations_baseline_user_policy_id_user_policy_associations" + ), + "user_report_associations", + "user_policy_associations", + ["baseline_user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + # Approved hand-written exception: a temporary view under the table's + # earlier name, so the Public API release still serving during the deploy, + # or after a rollback to it, keeps reading and writing saved policies. A + # maintainer approved this hand-written statement on 2026-10-22, and a later + # revision drops the view. A single-table view is automatically updatable; + # the privilege manifest grants on it what it grants on the table. + op.execute( + "CREATE VIEW user_policies AS SELECT id, created_at, updated_at, user_id, " + "policy_id, country_id, name, description, legacy_policy_id, baseline, " + "provenance FROM user_policy_associations" + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Approved hand-written exception: the upgrade's temporary view is dropped + # first, so the table can take its earlier name back. + op.execute("DROP VIEW user_policies") + op.drop_constraint( + op.f( + "fk_user_report_associations_baseline_user_policy_id_user_policy_associations" + ), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_user_policy_id_user_policy_associations"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + "fk_legacy_user_policy_mappings_association_country", + "legacy_user_policy_mappings", + type_="foreignkey", + ) + op.drop_index( + op.f("ix_user_policy_associations_user_id"), + table_name="user_policy_associations", + ) + op.drop_index( + op.f("ix_user_policy_associations_policy_id"), + table_name="user_policy_associations", + ) + # Post-generation correction: the reverse of the upgrade's rename. The + # generated downgrade created an empty user_policies table and dropped + # user_policy_associations, which would delete every saved policy. The + # table is renamed back in place, and each constraint is dropped under its + # new name and created under the name and definition it has at revision + # fbb7488ebe8a. + op.drop_constraint( + op.f("fk_user_policy_associations_user_id_users"), + "user_policy_associations", + type_="foreignkey", + ) + op.drop_constraint( + "fk_user_policy_associations_policy_country", + "user_policy_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_policy_ref"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_baseline"), + "user_policy_associations", + type_="check", + ) + op.drop_constraint( + "uq_user_policy_associations_id_country", + "user_policy_associations", + type_="unique", + ) + op.drop_constraint( + op.f("pk_user_policy_associations"), + "user_policy_associations", + type_="primary", + ) + op.rename_table("user_policy_associations", "user_policies") + op.create_primary_key(op.f("pk_user_policies"), "user_policies", ["id"]) + op.create_unique_constraint( + op.f("uq_user_policies_id_country"), "user_policies", ["id", "country_id"] + ) + op.create_check_constraint( + op.f("ck_user_policies_baseline"), + "user_policies", + "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), + "user_policies", + "country_id IN ('us', 'uk')", + ) + op.create_check_constraint( + op.f("ck_user_policies_policy_ref"), + "user_policies", + "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", + ) + op.create_foreign_key( + op.f("fk_user_policies_policy_country"), + "user_policies", + "policies", + ["policy_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + op.f("ix_user_policies_user_id"), "user_policies", ["user_id"], unique=False + ) + op.create_index( + op.f("ix_user_policies_policy_id"), + "user_policies", + ["policy_id"], + unique=False, + ) + op.create_foreign_key( + op.f("fk_user_report_associations_user_policy_id_user_policies"), + "user_report_associations", + "user_policies", + ["user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_baseline_user_policy_id_use_7740"), + "user_report_associations", + "user_policies", + ["baseline_user_policy_id"], + ["id"], + ondelete="SET NULL", + ) + op.create_foreign_key( + op.f("fk_legacy_user_policy_mappings_association_country"), + "legacy_user_policy_mappings", + "user_policies", + ["user_policy_id", "country_id"], + ["id", "country_id"], + ondelete="CASCADE", + ) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py new file mode 100644 index 000000000..1ea944f0f --- /dev/null +++ b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py @@ -0,0 +1,199 @@ +"""add countries table + +Revision ID: fb1a3e40d3d6 +Revises: f5c1d9b4685b +Create Date: 2026-10-22 02:16:50.126715 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "fb1a3e40d3d6" +down_revision: Union[str, None] = "f5c1d9b4685b" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + countries = op.create_table( + "countries", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False), + sa.Column("household_enabled", sa.Boolean(), nullable=False), + sa.Column("society_wide_enabled", sa.Boolean(), nullable=False), + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name=op.f("ck_countries_society_wide_requires_household"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_countries")), + ) + # Approved hand-written data operation: the one exception to generated-only + # revisions, decided by the maintainers for this table alone. The foreign + # keys below fail on any database whose rows already name the US or the UK + # unless both countries are listed first, so their rows are inserted here, + # in the same revision. They are the countries the replaced checks + # allowed, and both support household and society-wide calculations. The + # generated downgrade removes them with the table. Listing another + # country later needs its own revision and the same kind of decision. + op.bulk_insert( + countries, + [ + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, + ], + ) + op.create_foreign_key( + op.f("fk_households_country_id_countries"), + "households", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_households_country"), "households", type_="check") + op.create_foreign_key( + op.f("fk_policies_country_id_countries"), + "policies", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_policies_country"), "policies", type_="check") + op.create_foreign_key( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_policy_associations_country_id_countries"), + "user_policy_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_users_primary_country_countries"), + "users", + "countries", + ["primary_country"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_users_primary_country"), "users", type_="check") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_check_constraint( + op.f("ck_users_primary_country"), + "users", + "primary_country::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_users_primary_country_countries"), "users", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_policy_associations_country"), + "user_policy_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_policy_associations_country_id_countries"), + "user_policy_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_policies_country"), + "policies", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_policies_country_id_countries"), "policies", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_households_country"), + "households", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_households_country_id_countries"), "households", type_="foreignkey" + ) + op.drop_table("countries") + # ### end Alembic commands ### diff --git a/policyengine_api/asgi.py b/policyengine_api/asgi.py index 1f981c56a..fc5809cf1 100644 --- a/policyengine_api/asgi.py +++ b/policyengine_api/asgi.py @@ -10,6 +10,7 @@ from policyengine_api.observability import get_runtime from policyengine_api.readiness import mark_not_ready, mark_ready from policyengine_api.runtime_cache.client import close_runtime_cache_clients +from policyengine_api.startup_country_check import check_countries_table_at_startup from policyengine_api.warmup import run_startup_warmup @@ -24,6 +25,10 @@ def _close_runtime_resources() -> None: shutdown_callback=_close_runtime_resources, ) +# Refuse to start when the v2 countries table and the country list in code +# disagree (see policyengine_api.startup_country_check). +check_countries_table_at_startup() + # Warm the simulation machinery before serving (see policyengine_api.warmup). # POLICYENGINE_API_STARTUP_WARMUP=0 skips it. if os.environ.get("POLICYENGINE_API_STARTUP_WARMUP", "1") != "0": diff --git a/policyengine_api/asgi_factory.py b/policyengine_api/asgi_factory.py index 1b986188e..8f74b1bb2 100644 --- a/policyengine_api/asgi_factory.py +++ b/policyengine_api/asgi_factory.py @@ -53,6 +53,7 @@ current_observability_id, generate_request_id, ) +from policyengine_api.vary_headers import UniqueVaryMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Iterable @@ -471,7 +472,9 @@ def create_asgi_app( app.mount("/", WSGIMiddleware(wsgi_app)) # The public API already permits every web origin. Use the standard ASGI # implementation while preserving the existing reflected-origin response. - return CORSMiddleware( + # UniqueVaryMiddleware wraps it, because CORS and gzip each append to Vary + # values a route may already have named. + cors_app = CORSMiddleware( app=app, allow_origin_regex=".*", allow_methods=["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"], @@ -480,7 +483,10 @@ def create_asgi_app( REQUEST_ID_HEADER, OBSERVABILITY_ID_HEADER, "WWW-Authenticate", + # v2 metadata responses carry an ETag a client may revalidate with. + "ETag", ], allow_credentials=False, max_age=600, ) + return UniqueVaryMiddleware(cors_app) diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index d9b50fc76..1e97b6e3c 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -12,8 +12,9 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS -SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) +SUPPORTED_V2_COUNTRY_IDS = frozenset(SUPPORTED_COUNTRY_IDS) MAXIMUM_POLICYENGINE_VERSION_LENGTH = 128 diff --git a/policyengine_api/data/v2/catalog/certified_data_inspection.py b/policyengine_api/data/v2/catalog/certified_data_inspection.py new file mode 100644 index 000000000..58f0164a4 --- /dev/null +++ b/policyengine_api/data/v2/catalog/certified_data_inspection.py @@ -0,0 +1,140 @@ +"""Read household column names from certified HDF5 dataset files. + +The artifacts are read from Hugging Face at their pinned revisions through +ranged requests. Only the 1 MB blocks holding the HDF5 metadata and the +column-name arrays are downloaded, a small part of each file; the household +arrays themselves are never read. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, BinaryIO, Protocol, TypeVar +from urllib.parse import quote + +import h5py +from huggingface_hub import HfFileSystem + +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError + +if TYPE_CHECKING: + from collections.abc import Callable + from contextlib import AbstractContextManager + + from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +HOUSEHOLD_TABLE = "household" +# PyTables names the columns a table stores together "values_block_"; their +# real names live only in pickled attributes. +VALUE_BLOCK_PREFIX = "values_block_" +TABLE_INDEX_COLUMN = "index" + +_Result = TypeVar("_Result") + + +# HDF5 metadata is read in many small, scattered pieces; caching whole blocks of +# this size keeps the number of ranged requests low. +READ_BLOCK_BYTES = 1 << 20 + + +class ArtifactFileSystem(Protocol): + """Opens a file in a Hugging Face repository at a revision for reading.""" + + def open( + self, path: str, mode: str, *, block_size: int, cache_type: str + ) -> AbstractContextManager[BinaryIO]: + """Return a readable, seekable binary file that caches blocks it reads.""" + ... + + +def huggingface_path(artifact: CertifiedArtifact) -> str: + """Return the Hugging Face file system path of an artifact at its revision.""" + prefix = "datasets/" if artifact.repo_type == "dataset" else "" + revision = quote(artifact.revision, safe="") + return f"{prefix}{artifact.repo_id}@{revision}/{artifact.path}" + + +def _text(value: object) -> str: + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def _is_frame(group: h5py.Group) -> bool: + return "axis0" in group or "table" in group + + +def _frame_columns(group: h5py.Group) -> frozenset[str]: + """Return the column names of one pandas data frame stored in HDF5.""" + if "axis0" in group: + return frozenset(_text(value) for value in group["axis0"][()]) + names = group["table"].dtype.names or () + if any(name.startswith(VALUE_BLOCK_PREFIX) for name in names): + msg = ( + f"household table {group.name.lstrip('/')} stores columns in value " + "blocks, whose names cannot be read without unpickling" + ) + raise CertifiedDataReadError(msg) + return frozenset(name for name in names if name != TABLE_INDEX_COLUMN) + + +def household_columns_in_hdf5(h5: h5py.File) -> frozenset[str]: + """Return the household column names of a dataset file. + + A pandas file stores the household frame directly, or one frame per year + (then only columns every year has count). A policyengine-core file stores + one top-level entry per variable. + """ + if HOUSEHOLD_TABLE not in h5: + return frozenset(h5.keys()) + household = h5[HOUSEHOLD_TABLE] + if not isinstance(household, h5py.Group): + msg = "household entry is not an HDF5 group" + raise CertifiedDataReadError(msg) + if _is_frame(household): + return _frame_columns(household) + frames = [ + entry + for entry in household.values() + if isinstance(entry, h5py.Group) and _is_frame(entry) + ] + if not frames: + msg = "household group holds no data frame" + raise CertifiedDataReadError(msg) + return frozenset.intersection(*(_frame_columns(frame) for frame in frames)) + + +def _hugging_face_file_system(token: str | None) -> ArtifactFileSystem: + return HfFileSystem(token=token) + + +class HuggingFaceDataInspector: + """Reads certified artifacts from Hugging Face with an optional read token.""" + + def __init__( + self, + *, + token: str | None, + file_system_factory: Callable[[str | None], ArtifactFileSystem] = ( + _hugging_face_file_system + ), + ) -> None: + """Build the file system once; a token of None reads public data only.""" + self._file_system = file_system_factory(token) + + def _read( + self, + artifact: CertifiedArtifact, + read: Callable[[h5py.File], _Result], + ) -> _Result: + with ( + self._file_system.open( + huggingface_path(artifact), + "rb", + block_size=READ_BLOCK_BYTES, + cache_type="blockcache", + ) as handle, + h5py.File(handle, "r") as h5, + ): + return read(h5) + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + """Return the household column names of a dataset artifact.""" + return self._read(artifact, household_columns_in_hdf5) diff --git a/policyengine_api/data/v2/catalog/errors.py b/policyengine_api/data/v2/catalog/errors.py new file mode 100644 index 000000000..e33762b81 --- /dev/null +++ b/policyengine_api/data/v2/catalog/errors.py @@ -0,0 +1,14 @@ +"""Errors raised while catalog sources are read and normalized.""" + +from __future__ import annotations + + +class CatalogExtractionError(RuntimeError): + """Raised before database access when source metadata cannot be normalized.""" + + +class CertifiedDataReadError(RuntimeError): + """Raised when a certified artifact does not have the structure expected. + + Its message names only HDF5 entries and counts, so it is safe to print. + """ diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 75b0b213f..a46d042b1 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -2,70 +2,86 @@ from __future__ import annotations -import json import math import re -from collections import Counter from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from enum import Enum from importlib import metadata as importlib_metadata -from typing import TYPE_CHECKING, Any, NamedTuple, Protocol -from urllib.parse import urlsplit +from typing import TYPE_CHECKING, Any from uuid import NAMESPACE_URL, UUID, uuid5 +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.records import ( + CATALOG_FORMAT, CountryCatalog, DatasetRecord, + EntityRecord, + EnumValueRecord, FallbackSummary, ModelRecord, ModelVersionRecord, NormalizedCatalog, + OmittedRegionSummary, ParameterNodeRecord, ParameterRecord, ParameterValueRecord, RegionRecord, VariableRecord, ) +from policyengine_api.data.v2.catalog.region_references import ( + RegionReference, + load_region_references, +) +from policyengine_api.data.v2.catalog.region_scoping import ( + ScopedRegion, + filtered_artifacts, + scope_regions, +) +from policyengine_api.data.v2.models import Variable +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from policyengine_api.json_types import JSONValue -SUPPORTED_COUNTRIES = ("us", "uk") +SUPPORTED_COUNTRIES = SUPPORTED_COUNTRY_IDS REQUIRED_DEPENDENCIES = ( "policyengine-core", - "policyengine-us", - "policyengine-uk", + *(f"policyengine-{country_id}" for country_id in SUPPORTED_COUNTRY_IDS), ) -REVIEWED_DEFAULT_DATASETS = { - "us": "populace_us_2024", - "uk": "microcosm_uk_2024_25", + + +def _variable_column_length(column_name: str) -> int: + """Return the length limit of one text column of the v2 ``variables`` table.""" + length = getattr(Variable.__table__.c[column_name].type, "length", None) + if not isinstance(length, int): + msg = f"variables.{column_name} has no length limit" + raise TypeError(msg) + return length + + +# Catalog format 2's variable text fields, checked against the columns that +# store them, so the two limits cannot drift apart. +VARIABLE_TEXT_LIMITS: Mapping[str, int] = { + column_name: _variable_column_length(column_name) + for column_name in ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "unit", + "definition_period", + ) } CURRENT_LAW_IDS = {"us": 2, "uk": 1} METADATA_TIME_PERIODS = { "us": tuple(range(2035, 2021, -1)), "uk": tuple(range(2024, 2031)), } -SUPPORTED_REGION_TYPES = frozenset( - { - "national", - "country", - "state", - "congressional_district", - "constituency", - "local_authority", - "city", - "place", - } -) PLACEHOLDER_VERSIONS = frozenset({"", "0", "0.0", "0.0.0", "unknown"}) DATASET_YEAR_PATTERN = re.compile(r"(?:19|20)\d{2}") -class CatalogExtractionError(RuntimeError): - """Raised before database access when source metadata cannot be normalized.""" - - def _identifier(*parts: object) -> UUID: value = "/".join(str(part) for part in parts) return uuid5(NAMESPACE_URL, f"https://api.policyengine.org/v2/catalog/{value}") @@ -99,42 +115,18 @@ def _optional_text( return value -# The catalog's region filter_value column holds at most this many characters. -MAX_REGION_FILTER_VALUE_LENGTH = 255 - +def _optional_bool(value: object, *, field_name: str) -> bool | None: + if value is not None and not isinstance(value, bool): + msg = f"{field_name} must be a boolean or null" + raise CatalogExtractionError(msg) + return value -def _serialize_region_filter_value(value: object, *, region_code: str) -> str: - """Keep scalar text or encode flat membership values in the existing column. - Since PolicyEngine 6.2.7, the England filter names all nine English regions. The - catalog's existing text column can retain that complete list as a JSON - array without changing its schema or the representation of scalar filters. - """ - scalar_types = (str, int, float, bool) - if isinstance(value, scalar_types): - serialized = str(value) - elif ( - isinstance(value, (list, tuple)) - and value - and all(isinstance(item, scalar_types) for item in value) - ): - try: - serialized = json.dumps( - value, ensure_ascii=False, separators=(",", ":"), allow_nan=False - ) - except ValueError as error: - msg = f"region {region_code!r} has an unsupported filter value" - raise CatalogExtractionError(msg) from error - else: - msg = f"region {region_code!r} has an unsupported filter value" +def _required_bool(value: object, *, field_name: str) -> bool: + if not isinstance(value, bool): + msg = f"{field_name} must be a boolean" raise CatalogExtractionError(msg) - if len(serialized) > MAX_REGION_FILTER_VALUE_LENGTH: - msg = ( - f"region {region_code!r} filter value exceeds " - f"{MAX_REGION_FILTER_VALUE_LENGTH} characters" - ) - raise CatalogExtractionError(msg) - return serialized + return value def _type_name(value: object) -> str | None: @@ -236,17 +228,6 @@ def _optional_aware_datetime(value: object, *, field_name: str) -> datetime | No return _aware_datetime(value, field_name=field_name) -def _dataset_name_from_path(path: str) -> str: - without_revision = path.split("@", maxsplit=1)[0] - parsed_path = urlsplit(without_revision).path or without_revision - filename = parsed_path.rstrip("/").rsplit("/", maxsplit=1)[-1] - for suffix in (".hdf5", ".parquet", ".csv", ".h5"): - if filename.endswith(suffix): - filename = filename[: -len(suffix)] - break - return _required_text(filename, field_name="dataset name", maximum=255) - - def _dataset_year(name: str) -> int: matches = DATASET_YEAR_PATTERN.findall(name) if not matches: @@ -334,6 +315,12 @@ def _normalize_variable( else: normalized_possible_values = None + enum_values = _normalize_enum_values( + name, + getattr(source, "enum_values", None), + possible_values=normalized_possible_values, + ) + def optional_names(attribute: str) -> list[str] | None: values = getattr(source, attribute, None) if values is None: @@ -373,9 +360,87 @@ def optional_names(attribute: str) -> list[str] | None: default_value=normalize_json_value(getattr(source, "default_value", None)), adds=optional_names("adds"), subtracts=optional_names("subtracts"), + is_input_variable=_optional_bool( + getattr(source, "is_input_variable", None), + field_name=f"variable {name} is_input_variable", + ), + hidden_input=_optional_bool( + getattr(source, "hidden_input", None), + field_name=f"variable {name} hidden_input", + ), + module_name=_optional_text( + getattr(source, "module_name", None), + field_name=f"variable {name} module_name", + maximum=VARIABLE_TEXT_LIMITS["module_name"], + ), + adds_parameter_path=_optional_text( + getattr(source, "adds_parameter_path", None), + field_name=f"variable {name} adds_parameter_path", + maximum=VARIABLE_TEXT_LIMITS["adds_parameter_path"], + ), + subtracts_parameter_path=_optional_text( + getattr(source, "subtracts_parameter_path", None), + field_name=f"variable {name} subtracts_parameter_path", + maximum=VARIABLE_TEXT_LIMITS["subtracts_parameter_path"], + ), + enum_values=enum_values, + unit=_optional_text( + getattr(source, "unit", None), + field_name=f"variable {name} unit", + maximum=VARIABLE_TEXT_LIMITS["unit"], + ), + definition_period=_optional_text( + getattr(source, "definition_period", None), + field_name=f"variable {name} definition_period", + maximum=VARIABLE_TEXT_LIMITS["definition_period"], + ), ) +def _normalize_enum_values( + name: str, + source_values: object, + *, + possible_values: list[str] | None, +) -> tuple[EnumValueRecord, ...] | None: + """Normalize an enum variable's value labels, which must match its values. + + Each label is kept as declared when it is a JSON scalar. + """ + if source_values is None and possible_values is None: + return None + if ( + source_values is None + or not isinstance(source_values, Sequence) + or isinstance(source_values, (str, bytes)) + ): + msg = f"variable {name} enum values do not match its possible values" + raise CatalogExtractionError(msg) + records: list[EnumValueRecord] = [] + for source_value in source_values: + value = _required_text( + getattr(source_value, "value", None), + field_name=f"variable {name} enum value", + maximum=255, + ) + try: + label = normalize_json_value(getattr(source_value, "label", None)) + except CatalogExtractionError as error: + msg = ( + f"variable {name} enum label for {value} is not JSON-compatible: " + f"{error}" + ) + raise CatalogExtractionError(msg) from error + if isinstance(label, (dict, list)): + msg = f"variable {name} enum label for {value} must be a JSON scalar" + raise CatalogExtractionError(msg) + records.append(EnumValueRecord(value=value, label=label)) + if [record.value for record in records] != possible_values: + msg = f"variable {name} enum values do not match its possible values" + raise CatalogExtractionError(msg) + return tuple(records) + + def _normalize_parameter( source: object, *, @@ -466,6 +531,14 @@ def _normalize_parameter( maximum=128, ), values=tuple(normalized_values), + economy=_required_bool( + getattr(source, "economy", None), + field_name=f"parameter {name} economy", + ), + household=_required_bool( + getattr(source, "household", None), + field_name=f"parameter {name} household", + ), ) @@ -492,18 +565,6 @@ def _public_named_records( return tuple(normalized) -class _RegionDatasetSelection(NamedTuple): - """The logical input datasets a country's regions select. - - Names every selected dataset, each region code's dataset, and how many - regions of each type fall back to the reviewed default dataset. - """ - - dataset_names: set[str] - region_dataset_names: dict[str, str] - fallback_counts: Counter[str] - - def _verified_model_name(country_id: str, source_model: object) -> str: """Return the public model's name once it matches the country's model.""" expected_model_name = f"policyengine-{country_id}" @@ -595,184 +656,232 @@ def _normalize_parameter_nodes( return tuple(normalized_nodes) -def _verified_default_dataset( - country_id: str, - release_manifest: object, -) -> tuple[str, object]: - """Return the reviewed default dataset and the manifest's URI for it. - - The manifest's default dataset must be the reviewed one. - """ - reviewed_default = REVIEWED_DEFAULT_DATASETS[country_id] - manifest_default = getattr(release_manifest, "default_dataset", None) - if manifest_default != reviewed_default: - msg = f"{country_id} default dataset does not match the reviewed selection" - raise CatalogExtractionError(msg) - return reviewed_default, getattr(release_manifest, "default_dataset_uri", None) +# The public fields catalog format 2 reads, by the record that carries them. +FORMAT_2_FIELDS: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ("model version", "", ("entities", "basic_inputs")), + ( + "variable", + "variables_by_name", + ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + ), + ), + ("parameter", "parameters_by_name", ("economy", "household")), +) -def _verified_source_regions(country_id: str, source: object) -> tuple[Any, ...]: - """Return the country's public regions once their codes are unique and complete. +def _require_format_2_fields(country_id: str, source: object) -> None: + """Fail first, naming each field, when PolicyEngine.py does not publish them. - The registry must belong to the country, and its codes must be unique and - include the national region. + Older releases, such as 6.7.5 and the locked 6.10.0, publish only the + variables' unit and definition period of catalog format 2's fields. """ - registry = getattr(source, "region_registry", None) - if registry is None or getattr(registry, "country_id", None) != country_id: - msg = f"{country_id} public region registry is absent" + missing: list[str] = [] + for record_name, mapping_attribute, field_names in FORMAT_2_FIELDS: + if mapping_attribute: + records = getattr(source, mapping_attribute, None) + holders = list(records.values()) if isinstance(records, Mapping) else [] + else: + holders = [source] + absent = [ + field_name + for field_name in field_names + if any(not hasattr(holder, field_name) for holder in holders) + ] + if absent: + missing.append(f"{record_name} {', '.join(absent)}") + if missing: + msg = ( + "the installed PolicyEngine.py does not publish catalog format 2's " + f"fields for {country_id}: {'; '.join(missing)}" + ) raise CatalogExtractionError(msg) - source_regions = tuple(registry) - region_codes = [getattr(region, "code", None) for region in source_regions] - duplicate_region_codes = sorted( - code for code, count in Counter(region_codes).items() if count > 1 - ) - if duplicate_region_codes: - msg = f"duplicate region natural keys: {duplicate_region_codes[:3]}" + +def _source_sequence(country_id: str, source: object, attribute: str) -> Sequence: + values = getattr(source, attribute, None) + if not isinstance(values, Sequence) or isinstance(values, (str, bytes)): + msg = f"{country_id} public model {attribute} must be a sequence" raise CatalogExtractionError(msg) - if country_id not in region_codes: - msg = f"{country_id} national region is absent" + return values + + +def _normalize_entities(country_id: str, source: object) -> tuple[EntityRecord, ...]: + """Normalize the public model's entities, in its order, with one person entity.""" + source_entities = _source_sequence(country_id, source, "entities") + entities: list[EntityRecord] = [] + for source_entity in source_entities: + key = _required_text( + getattr(source_entity, "key", None), + field_name=f"{country_id} entity key", + maximum=128, + ) + if any(entity.key == key for entity in entities): + msg = f"{country_id} entity key {key!r} is repeated" + raise CatalogExtractionError(msg) + entities.append( + EntityRecord( + key=key, + plural=_required_text( + getattr(source_entity, "plural", None), + field_name=f"{country_id} entity {key} plural", + maximum=128, + ), + is_person=_required_bool( + getattr(source_entity, "is_person", None), + field_name=f"{country_id} entity {key} is_person", + ), + ) + ) + if sum(entity.is_person for entity in entities) != 1: + msg = f"{country_id} public model must have exactly one person entity" raise CatalogExtractionError(msg) - return source_regions + return tuple(entities) -def _select_region_datasets( +def _normalize_basic_inputs( country_id: str, - source_regions: tuple[Any, ...], + source: object, *, - reviewed_default: str, - default_dataset_uri: object, -) -> _RegionDatasetSelection: - """Choose each region's logical input dataset. - - A subnational region with a dataset path uses that dataset: US states and - districts, and the UK constituencies and local authorities that .py routes - to its local-area release. Every other region, such as a UK country that - filters the national file, uses the reviewed default, and the national - region may only name it. A US subnational region without a path is - counted as a fallback. + variable_names: frozenset[str], +) -> tuple[str, ...]: + """Normalize the country's default household fields; each is a variable. + + An empty list, which PolicyEngine.py gives a package that declares none, is + stored as empty. """ - selected_dataset_names: set[str] = {reviewed_default} - region_dataset_names: dict[str, str] = {} - fallback_counts: Counter[str] = Counter() - for region in source_regions: - code = _required_text( - getattr(region, "code", None), - field_name="region code", - maximum=255, - ) - region_type = _required_text( - getattr(region, "region_type", None), - field_name=f"region {code} type", - maximum=64, - ) - if region_type not in SUPPORTED_REGION_TYPES: - msg = f"region {code!r} has unsupported type {region_type!r}" - raise CatalogExtractionError(msg) - dataset_path = getattr(region, "dataset_path", None) - if region_type != "national" and dataset_path: - dataset_name = _dataset_name_from_path(dataset_path) - selected_dataset_names.add(dataset_name) - else: - dataset_name = reviewed_default - if country_id == "us" and region_type != "national" and not dataset_path: - fallback_counts[region_type] += 1 - if region_type == "national" and dataset_path not in { - None, - default_dataset_uri, - }: - msg = f"{country_id} national region does not use the reviewed dataset" - raise CatalogExtractionError(msg) - region_dataset_names[code] = dataset_name - return _RegionDatasetSelection( - dataset_names=selected_dataset_names, - region_dataset_names=region_dataset_names, - fallback_counts=fallback_counts, + source_inputs = _source_sequence(country_id, source, "basic_inputs") + basic_inputs = tuple( + _required_text(value, field_name=f"{country_id} basic input", maximum=512) + for value in source_inputs ) + for name in basic_inputs: + if name not in variable_names: + msg = f"{country_id} basic input {name!r} is not a published variable" + raise CatalogExtractionError(msg) + return basic_inputs -def _region_filter(strategy: object, code: str) -> tuple[str, str, str]: - """Return a filtered region's strategy type, filter field and filter value.""" - strategy_type = _required_text( - getattr(strategy, "strategy_type", None), - field_name=f"region {code} filter strategy", - maximum=64, +def _verify_parameter_paths( + variables: tuple[VariableRecord, ...], + *, + parameter_names: frozenset[str], +) -> None: + """Require each parameter named by ``adds`` or ``subtracts`` to be published.""" + for variable in variables: + for field_name, path in ( + ("adds_parameter_path", variable.adds_parameter_path), + ("subtracts_parameter_path", variable.subtracts_parameter_path), + ): + if path is not None and path not in parameter_names: + msg = ( + f"variable {variable.name} {field_name} {path!r} is not a " + "published parameter" + ) + raise CatalogExtractionError(msg) + + +@dataclass(frozen=True, slots=True) +class ModelMetadata: + """The model metadata of one country's catalog, without regions or datasets.""" + + entities: tuple[EntityRecord, ...] + basic_inputs: tuple[str, ...] + variables: tuple[VariableRecord, ...] + parameter_nodes: tuple[ParameterNodeRecord, ...] + parameters: tuple[ParameterRecord, ...] + + +def normalize_model_metadata( + country_id: str, + source: object, + *, + model_version_id: UUID, +) -> ModelMetadata: + """Normalize a public model's entities, variables and parameter tree. + + Everything comes from PolicyEngine.py's public records. Separate from the + certified data release, so the metadata can be checked on its own. + """ + _require_format_2_fields(country_id, source) + variables = tuple( + _normalize_variable(variable, model_version_id=model_version_id) + for variable in _public_named_records( + source, + attribute="variables_by_name", + entity_name="variable", + ) ) - filter_field = _required_text( - getattr(strategy, "variable_name", None), - field_name=f"region {code} filter field", - maximum=128, + parameters = tuple( + _normalize_parameter(parameter, model_version_id=model_version_id) + for parameter in _public_named_records( + source, + attribute="parameters_by_name", + entity_name="parameter", + ) ) - filter_value = _serialize_region_filter_value( - getattr(strategy, "variable_value", None), region_code=code + _verify_parameter_paths( + variables, + parameter_names=frozenset(parameter.name for parameter in parameters), + ) + return ModelMetadata( + entities=_normalize_entities(country_id, source), + basic_inputs=_normalize_basic_inputs( + country_id, + source, + variable_names=frozenset(variable.name for variable in variables), + ), + variables=variables, + parameter_nodes=_normalize_parameter_nodes( + source, model_version_id=model_version_id + ), + parameters=parameters, ) - additional_filters = getattr(strategy, "additional_filters", {}) - if additional_filters: - msg = f"region {code!r} has unsupported additional filters" - raise CatalogExtractionError(msg) - return strategy_type, filter_field, filter_value -class _SourceRegion(Protocol): - """The attributes of a PolicyEngine.py region that _normalize_region reads directly. +def _release_default_dataset(country_id: str, release_manifest: object) -> str: + """Return the default dataset the installed PolicyEngine.py release names. - It reads every other region attribute through getattr, with a default. + The Public API sets no default of its own; region scoping then requires the + release to certify it. """ - - @property - def code(self) -> object: ... - - @property - def region_type(self) -> object: ... + default_dataset = getattr(release_manifest, "default_dataset", None) + if not isinstance(default_dataset, str) or not default_dataset: + msg = f"{country_id} release manifest names no default dataset" + raise CatalogExtractionError(msg) + return default_dataset def _normalize_region( - source_region: _SourceRegion, + scoped_region: ScopedRegion, *, model_version_id: UUID, - region_dataset_ids: Mapping[str, UUID], + dataset_ids: Mapping[str, UUID], ) -> RegionRecord: - """Normalize one public region, with its filter and default dataset.""" - code = str(source_region.code) - strategy = getattr(source_region, "scoping_strategy", None) - requires_filter = bool(getattr(source_region, "requires_filter", False)) - if requires_filter: - strategy_type, filter_field, filter_value = _region_filter(strategy, code) - else: - strategy_type = None - filter_field = None - filter_value = None - + """Build one region's record from its reference, dataset and filter.""" + reference = scoped_region.reference + requires_filter = scoped_region.filter_strategy is not None return RegionRecord( - id=_identifier("region", model_version_id, code), + id=_identifier("region", model_version_id, reference.code), model_version_id=model_version_id, - default_dataset_id=region_dataset_ids[code], - code=code, - label=_required_text( - getattr(source_region, "label", None), - field_name=f"region {code} label", - maximum=255, - ), - region_type=str(source_region.region_type), + default_dataset_id=dataset_ids[scoped_region.dataset_name], + code=reference.code, + label=reference.label, + region_type=reference.region_type, requires_filter=requires_filter, - filter_field=filter_field, - filter_value=filter_value, - filter_strategy=strategy_type, - parent_code=_optional_text( - getattr(source_region, "parent_code", None), - field_name=f"region {code} parent_code", - maximum=255, - ), - state_code=_optional_text( - getattr(source_region, "state_code", None), - field_name=f"region {code} state_code", - maximum=16, - ), - state_name=_optional_text( - getattr(source_region, "state_name", None), - field_name=f"region {code} state_name", - maximum=128, - ), + filter_field=scoped_region.filter_field, + filter_value=reference.filter_value if requires_filter else None, + filter_strategy=scoped_region.filter_strategy, + parent_code=reference.parent_code, + state_code=reference.state_code, + state_name=reference.state_name, ) @@ -782,6 +891,7 @@ def _normalize_country( source: object, policyengine_version: str, expected_country_package_version: str, + region_references: tuple[RegionReference, ...], ) -> CountryCatalog: source_model = getattr(source, "model", None) model_name = _verified_model_name(country_id, source_model) @@ -794,35 +904,15 @@ def _normalize_country( expected_country_package_version=expected_country_package_version, ) - variables = tuple( - _normalize_variable(variable, model_version_id=model_version_id) - for variable in _public_named_records( - source, - attribute="variables_by_name", - entity_name="variable", - ) - ) - parameter_nodes = _normalize_parameter_nodes( - source, model_version_id=model_version_id - ) - parameters = tuple( - _normalize_parameter(parameter, model_version_id=model_version_id) - for parameter in _public_named_records( - source, - attribute="parameters_by_name", - entity_name="parameter", - ) + metadata = normalize_model_metadata( + country_id, source, model_version_id=model_version_id ) - reviewed_default, default_dataset_uri = _verified_default_dataset( - country_id, release_manifest - ) - source_regions = _verified_source_regions(country_id, source) - dataset_selection = _select_region_datasets( + scoping = scope_regions( country_id, - source_regions, - reviewed_default=reviewed_default, - default_dataset_uri=default_dataset_uri, + region_references, + release_manifest, + default_dataset=_release_default_dataset(country_id, release_manifest), ) datasets = tuple( @@ -833,20 +923,16 @@ def _normalize_country( description=f"PolicyEngine.py logical input dataset {name}", year=_dataset_year(name), ) - for name in sorted(dataset_selection.dataset_names) + for name in sorted(scoping.dataset_names) ) dataset_ids = {dataset.name: dataset.id for dataset in datasets} - region_dataset_ids = { - code: dataset_ids[name] - for code, name in dataset_selection.region_dataset_names.items() - } regions = [ _normalize_region( - source_region, + scoped_region, model_version_id=model_version_id, - region_dataset_ids=region_dataset_ids, + dataset_ids=dataset_ids, ) - for source_region in source_regions + for scoped_region in scoping.regions ] return CountryCatalog( @@ -870,16 +956,24 @@ def _normalize_country( ), current_law_id=CURRENT_LAW_IDS[country_id], metadata_time_periods=METADATA_TIME_PERIODS[country_id], + catalog_format=CATALOG_FORMAT, + entities=metadata.entities, + basic_inputs=metadata.basic_inputs, ), - variables=variables, - parameter_nodes=parameter_nodes, - parameters=parameters, + variables=metadata.variables, + parameter_nodes=metadata.parameter_nodes, + parameters=metadata.parameters, datasets=datasets, regions=tuple(sorted(regions, key=lambda record: record.code)), fallback_summaries=tuple( FallbackSummary(region_type=region_type, count=count) - for region_type, count in sorted(dataset_selection.fallback_counts.items()) + for region_type, count in scoping.fallback_counts.items() + ), + omitted_region_summaries=tuple( + OmittedRegionSummary(region_type=region_type, count=count) + for region_type, count in scoping.omitted_counts.items() ), + certified_artifacts=filtered_artifacts(scoping), ) @@ -888,9 +982,14 @@ def extract_catalog( bundle: Mapping[str, Any], policyengine_version: str, models: Mapping[str, object], + region_references: Mapping[str, Sequence[RegionReference]], installed_version: Callable[[str], str] = importlib_metadata.version, ) -> NormalizedCatalog: - """Validate and normalize supplied PolicyEngine.py public objects.""" + """Validate and normalize supplied PolicyEngine.py public objects. + + Regions come from the supplied reference files, scoped by each country's + certified data release. + """ dependencies = _verify_bundle( bundle=bundle, policyengine_version=policyengine_version, @@ -900,6 +999,12 @@ def extract_catalog( if missing: msg = f"PolicyEngine.py public models are absent for: {', '.join(missing)}" raise CatalogExtractionError(msg) + missing = [ + country for country in SUPPORTED_COUNTRIES if country not in region_references + ] + if missing: + msg = f"region references are absent for: {', '.join(missing)}" + raise CatalogExtractionError(msg) package_map = bundle["packages"] countries = tuple( @@ -910,6 +1015,7 @@ def extract_catalog( expected_country_package_version=package_map[f"policyengine-{country_id}"][ "version" ], + region_references=tuple(region_references[country_id]), ) for country_id in SUPPORTED_COUNTRIES ) @@ -921,7 +1027,10 @@ def extract_catalog( def extract_installed_catalog() -> NormalizedCatalog: - """Load and normalize the installed PolicyEngine.py certified catalog.""" + """Load and normalize the installed PolicyEngine.py certified catalog. + + Regions come from the Public API's checked-in reference files. + """ import policyengine from policyengine.bundle import ( get_current_bundle, @@ -932,7 +1041,11 @@ def extract_installed_catalog() -> NormalizedCatalog: bundle=get_current_bundle(), policyengine_version=policyengine_version, models={ - "us": policyengine.us.model, - "uk": policyengine.uk.model, + country_id: getattr(policyengine, country_id).model + for country_id in SUPPORTED_COUNTRIES + }, + region_references={ + country_id: load_region_references(country_id) + for country_id in SUPPORTED_COUNTRIES }, ) diff --git a/policyengine_api/data/v2/catalog/initialization.py b/policyengine_api/data/v2/catalog/initialization.py index 10efdda6f..6c974f121 100644 --- a/policyengine_api/data/v2/catalog/initialization.py +++ b/policyengine_api/data/v2/catalog/initialization.py @@ -3,12 +3,17 @@ from __future__ import annotations import json +import os import sys +from dataclasses import dataclass from typing import TYPE_CHECKING from sqlalchemy import Engine, create_engine from sqlalchemy.pool import NullPool +from policyengine_api.data.v2.catalog.certified_data_inspection import ( + HuggingFaceDataInspector, +) from policyengine_api.data.v2.catalog.extraction import ( CatalogExtractionError, extract_installed_catalog, @@ -17,6 +22,15 @@ CatalogPublicationError, PublicationEvidence, publish_catalog, + unpublished_countries, +) +from policyengine_api.data.v2.catalog.region_data_verification import ( + RegionDataVerificationError, + verify_region_data, +) +from policyengine_api.data.v2.catalog.spm_availability import ( + record_published_spm_availability, + simulation_entrypoint_url, ) from policyengine_api.data.v2.settings import ( V2ConfigurationError, @@ -25,31 +39,111 @@ ) if TYPE_CHECKING: - from collections.abc import Callable, Mapping + from collections.abc import Callable, Mapping, Sequence from policyengine_api.data.v2.catalog.records import NormalizedCatalog +# The environment variable holding the Hugging Face read token; PolicyEngine +# Core's data downloads read the same variable. +HUGGING_FACE_CREDENTIAL_VARIABLE = "HUGGING_FACE_TOKEN" +# Countries whose certified data repository needs the read token. +PRIVATE_DATA_COUNTRIES = frozenset({"uk"}) + + def build_data_write_engine(settings: V2DatabaseSettings) -> Engine: """Build an isolated, unpooled engine for one initialization attempt.""" return create_engine(settings.connection.url, poolclass=NullPool) +def verify_installed_region_data( + catalog: NormalizedCatalog, + countries: Sequence[str], + environ: Mapping[str, str] | None = None, +) -> None: + """Check the named countries' regions against their certified artifacts. + + The artifacts are read from Hugging Face with ``HUGGING_FACE_TOKEN``. + Countries whose certified data repository is private need it, so their + check fails with a configuration error before any read when it is unset. + """ + token = (os.environ if environ is None else environ).get( + HUGGING_FACE_CREDENTIAL_VARIABLE + ) + needing_token = [ + country for country in countries if country in PRIVATE_DATA_COUNTRIES + ] + if needing_token and not token: + msg = ( + f"{HUGGING_FACE_CREDENTIAL_VARIABLE} must be set to read the certified " + f"{', '.join(needing_token)} data, whose repository is private" + ) + raise V2ConfigurationError(msg) + verify_region_data( + catalog, + HuggingFaceDataInspector(token=token or None), + countries=countries, + ) + + +def verify_unpublished_region_data( + engine: Engine, + catalog: NormalizedCatalog, + environ: Mapping[str, str] | None = None, +) -> None: + """Check the regions of countries whose catalog version is not stored yet. + + Publication writes nothing for the other countries and only compares them + with what is stored, so their artifacts are not read. + """ + verify_installed_region_data( + catalog, unpublished_countries(engine, catalog), environ + ) + + +@dataclass(frozen=True) +class PublicationSteps: + """What initialization does with the database, in order, after extraction. + + ``verify_region_data`` checks the regions' certified data before anything + is written, ``publish`` writes the catalog, and ``record_spm_availability`` + then records each version's SPM availability from the simulation service + (``spm_availability``). + """ + + verify_region_data: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = verify_unpublished_region_data + publish: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( + publish_catalog + ) + record_spm_availability: Callable[ + [Engine, NormalizedCatalog, Mapping[str, str] | None], None + ] = record_published_spm_availability + + def initialize_catalog( environ: Mapping[str, str] | None = None, *, extractor: Callable[[], NormalizedCatalog] = extract_installed_catalog, engine_builder: Callable[[V2DatabaseSettings], Engine] = build_data_write_engine, - publisher: Callable[[Engine, NormalizedCatalog], PublicationEvidence] = ( - publish_catalog - ), + steps: PublicationSteps | None = None, ) -> PublicationEvidence: - """Load only row-write settings, extract fully, then publish explicitly.""" + """Load only row-write settings, extract fully, verify, then publish explicitly. + + After publishing, record each version's SPM availability. The simulation + service's URL, which that needs, is checked before anything else. + """ + selected_steps = steps or PublicationSteps() settings = load_v2_data_write_database_settings(environ) + simulation_entrypoint_url(os.environ if environ is None else environ) catalog = extractor() engine = engine_builder(settings) try: - return publisher(engine, catalog) + selected_steps.verify_region_data(engine, catalog, environ) + evidence = selected_steps.publish(engine, catalog) + selected_steps.record_spm_availability(engine, catalog, environ) + return evidence finally: engine.dispose() @@ -58,6 +152,7 @@ def _error_payload(error: Exception) -> dict[str, object]: safe_errors = ( V2ConfigurationError, CatalogExtractionError, + RegionDataVerificationError, CatalogPublicationError, ) message = ( diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index 5e629fb05..a8951d6b5 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -12,6 +12,8 @@ from policyengine_api.data.v2.catalog.publication_reconciliation import ( assert_canonical_value_uniqueness, assert_country_matches, + count_region_differences, + count_stored_regions, publish_new_country, version_exists, ) @@ -34,7 +36,10 @@ if TYPE_CHECKING: from collections.abc import Callable - from policyengine_api.data.v2.catalog.records import NormalizedCatalog + from policyengine_api.data.v2.catalog.records import ( + CountryCatalog, + NormalizedCatalog, + ) # Stable application-defined PostgreSQL lock ID shared by all v2 catalog publishers. PUBLICATION_ADVISORY_LOCK_KEY = 8_629_020_026_090_001 @@ -56,6 +61,7 @@ "CatalogPublicationError", "PublicationEvidence", "publish_catalog", + "unpublished_countries", ] @@ -87,6 +93,25 @@ def _protected_row_counts(connection: Connection) -> tuple[int, int, int, int]: ) +def unpublished_countries( + engine: Engine, catalog: NormalizedCatalog +) -> tuple[str, ...]: + """Return the countries whose catalog version is not stored yet, in catalog order. + + It first requires the expected v2 revision, as publication does, so a + missing or older schema is reported as such. A read outside publication's + transaction: a version another publisher adds meanwhile is still compared + with the staged one when this one publishes. + """ + with engine.connect() as connection: + _verify_expected_revision(connection) + return tuple( + country.country_id + for country in catalog.countries + if not version_exists(connection, country) + ) + + def publish_catalog( engine: Engine, catalog: NormalizedCatalog, @@ -104,11 +129,7 @@ def publish_catalog( if checkpoint is not None: checkpoint("after_copy", connection) - existing: set[str] = set() - for country in catalog.countries: - if version_exists(connection, country): - assert_country_matches(connection, country) - existing.add(country.country_id) + existing, kept_region_summaries = _check_stored_versions(connection, catalog) for country in catalog.countries: if country.country_id not in existing: @@ -117,7 +138,11 @@ def publish_catalog( checkpoint("after_reconciliation", connection) for country in catalog.countries: - assert_country_matches(connection, country) + assert_country_matches( + connection, + country, + include_regions=country.country_id not in existing, + ) assert_canonical_value_uniqueness(connection) if _protected_row_counts(connection) != before: msg = "publication changed simulation, report, or dataset-version rows" @@ -125,18 +150,108 @@ def publish_catalog( if checkpoint is not None: checkpoint("after_validation", connection) + return _evidence(catalog, kept_region_summaries, started=started) + + +def _check_stored_versions( + connection: Connection, + catalog: NormalizedCatalog, +) -> tuple[set[str], tuple[tuple[str, str, int, int], ...]]: + """Compare each stored catalog version and return its countries and kept regions. + + A stored catalog version never changes. Its regions stay as they were + published even when the reference files differ now; every other table + must still match exactly. + """ + existing: set[str] = set() + kept_region_summaries: list[tuple[str, str, int, int]] = [] + for country in catalog.countries: + if version_exists(connection, country): + assert_country_matches(connection, country, include_regions=False) + kept = _kept_region_summary(connection, country) + if kept is not None: + kept_region_summaries.append(kept) + existing.add(country.country_id) + return existing, tuple(kept_region_summaries) + + +def _evidence( + catalog: NormalizedCatalog, + kept_region_summaries: tuple[tuple[str, str, int, int], ...], + *, + started: float, +) -> PublicationEvidence: + """Describe what is stored after publication. + + Every table of a stored version matches the staged catalog except regions + that were kept: for those countries the evidence counts the stored region + rows and reports none of the staged fallbacks or omissions, which describe + rows that were not published. + """ + kept_countries = {country_id for country_id, *_counts in kept_region_summaries} + entity_counts = catalog.entity_counts() + entity_counts["regions"] += sum( + stored - staged for _country, _version, stored, staged in kept_region_summaries + ) fallback_summaries = tuple( (country.country_id, summary.region_type, summary.count) for country in catalog.countries + if country.country_id not in kept_countries for summary in country.fallback_summaries ) _log_fallback_warning(fallback_summaries) return PublicationEvidence( policyengine_version=catalog.policyengine_version, dependency_versions=catalog.dependency_versions, - entity_counts=catalog.entity_counts(), + entity_counts=entity_counts, fallback_summaries=fallback_summaries, elapsed_seconds=time.monotonic() - started, + omitted_region_summaries=tuple( + (country.country_id, summary.region_type, summary.count) + for country in catalog.countries + if country.country_id not in kept_countries + for summary in country.omitted_region_summaries + ), + kept_region_summaries=kept_region_summaries, + ) + + +def _kept_region_summary( + connection: Connection, + country: CountryCatalog, +) -> tuple[str, str, int, int] | None: + """Return and log how a stored version's kept regions differ from the staged. + + The summary is the country, the version and the stored and staged region + counts; there is none when the regions are the same. The log names counts + only. + """ + staged_not_stored, stored_not_staged = count_region_differences(connection, country) + if not staged_not_stored and not stored_not_staged: + return None + # The seed command configures no logging handler, so the details are in the + # message itself; the attributes serve structured handlers. + LOGGER.warning( + "kept the published regions of the stored %s catalog version %s: %d staged " + "region rows are not stored and %d stored region rows are not staged", + country.country_id, + country.model_version.version, + staged_not_stored, + stored_not_staged, + extra={ + "policyengine_attributes": { + "country_id": country.country_id, + "policyengine_version": country.model_version.version, + "staged_region_rows_not_stored": staged_not_stored, + "stored_region_rows_not_staged": stored_not_staged, + } + }, + ) + return ( + country.country_id, + country.model_version.version, + count_stored_regions(connection, country), + len(country.regions), ) diff --git a/policyengine_api/data/v2/catalog/publication_reconciliation.py b/policyengine_api/data/v2/catalog/publication_reconciliation.py index 0977676c6..14390b6ee 100644 --- a/policyengine_api/data/v2/catalog/publication_reconciliation.py +++ b/policyengine_api/data/v2/catalog/publication_reconciliation.py @@ -76,9 +76,45 @@ def _versioned_table_from(table: sa.Table) -> sa.Join: ) +def stored_catalog_format( + connection: Connection, country: CountryCatalog +) -> int | None: + """Return the catalog format this country's stored version was published with. + + None means it was published before catalog format 2, so its format 2 + fields are empty. + """ + statement = ( + sa.select(MODEL_VERSIONS.c.catalog_format) + .select_from( + MODEL_VERSIONS.join( + MODELS, + MODELS.c.id == MODEL_VERSIONS.c.model_id, + ) + ) + .where( + MODELS.c.name == country.model.name, + MODEL_VERSIONS.c.version == country.model_version.version, + ) + ) + return connection.execute(statement).scalar_one_or_none() + + def _comparison_pairs( country: CountryCatalog, -) -> tuple[tuple[Select, Select], ...]: + *, + with_format_2: bool, +) -> tuple[tuple[str, Select, Select], ...]: + """Return each compared table's name with its staged and stored rows. + + Without ``with_format_2``, catalog format 2's fields are left out: a + version published before format 2 keeps them empty, and publication never + rewrites a stored version. + """ + + def format_2(*columns: sa.ColumnElement) -> tuple[sa.ColumnElement, ...]: + return columns if with_format_2 else () + staged_country = STAGE_CATALOG_MODELS.c.country_id == country.country_id selected_version = sa.and_( MODELS.c.name == country.model.name, @@ -89,12 +125,22 @@ def _comparison_pairs( STAGE_CATALOG_MODELS.c.version_description, STAGE_CATALOG_MODELS.c.current_law_id, STAGE_CATALOG_MODELS.c.metadata_time_periods, + *format_2( + STAGE_CATALOG_MODELS.c.catalog_format, + STAGE_CATALOG_MODELS.c.entities, + STAGE_CATALOG_MODELS.c.basic_inputs, + ), ).where(staged_country) actual_model_version = ( sa.select( MODEL_VERSIONS.c.description, MODEL_VERSIONS.c.current_law_id, sa.cast(MODEL_VERSIONS.c.metadata_time_periods, JSONB), + *format_2( + MODEL_VERSIONS.c.catalog_format, + sa.cast(MODEL_VERSIONS.c.entities, JSONB), + sa.cast(MODEL_VERSIONS.c.basic_inputs, JSONB), + ), ) .select_from( MODEL_VERSIONS.join( @@ -115,6 +161,16 @@ def _comparison_pairs( STAGE_CATALOG_VARIABLES.c.default_value, STAGE_CATALOG_VARIABLES.c.adds, STAGE_CATALOG_VARIABLES.c.subtracts, + *format_2( + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.adds_parameter_path, + STAGE_CATALOG_VARIABLES.c.subtracts_parameter_path, + STAGE_CATALOG_VARIABLES.c.enum_values, + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.definition_period, + ), ).where(STAGE_CATALOG_VARIABLES.c.country_id == country.country_id) actual_variables = ( sa.select( @@ -127,6 +183,16 @@ def _comparison_pairs( sa.cast(VARIABLES.c.default_value, JSONB), sa.cast(VARIABLES.c.adds, JSONB), sa.cast(VARIABLES.c.subtracts, JSONB), + *format_2( + VARIABLES.c.is_input_variable, + VARIABLES.c.hidden_input, + VARIABLES.c.module_name, + VARIABLES.c.adds_parameter_path, + VARIABLES.c.subtracts_parameter_path, + sa.cast(VARIABLES.c.enum_values, JSONB), + VARIABLES.c.unit, + VARIABLES.c.definition_period, + ), ) .select_from(_versioned_table_from(VARIABLES)) .where(selected_version) @@ -153,6 +219,10 @@ def _comparison_pairs( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + *format_2( + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, + ), ).where(STAGE_CATALOG_PARAMETERS.c.country_id == country.country_id) actual_parameters = ( sa.select( @@ -161,6 +231,7 @@ def _comparison_pairs( PARAMETERS.c.description, PARAMETERS.c.data_type, PARAMETERS.c.unit, + *format_2(PARAMETERS.c.economy, PARAMETERS.c.household), ) .select_from(_versioned_table_from(PARAMETERS)) .where(selected_version) @@ -220,6 +291,22 @@ def _comparison_pairs( ) ) + return ( + ("model version", staged_model_version, actual_model_version), + ("variables", staged_variables, actual_variables), + ("parameter nodes", staged_parameter_nodes, actual_parameter_nodes), + ("parameters", staged_parameters, actual_parameters), + ("parameter values", staged_parameter_values, actual_parameter_values), + ("datasets", staged_datasets, actual_datasets), + ) + + +def _region_comparison_pair(country: CountryCatalog) -> tuple[Select, Select]: + """Return the staged and stored region rows of one country's catalog version.""" + selected_version = sa.and_( + MODELS.c.name == country.model.name, + MODEL_VERSIONS.c.version == country.model_version.version, + ) staged_regions = sa.select( STAGE_CATALOG_REGIONS.c.code, STAGE_CATALOG_REGIONS.c.label, @@ -256,15 +343,7 @@ def _comparison_pairs( .where(selected_version) ) - return ( - (staged_model_version, actual_model_version), - (staged_variables, actual_variables), - (staged_parameter_nodes, actual_parameter_nodes), - (staged_parameters, actual_parameters), - (staged_parameter_values, actual_parameter_values), - (staged_datasets, actual_datasets), - (staged_regions, actual_regions), - ) + return staged_regions, actual_regions def _sets_differ( @@ -284,18 +363,58 @@ def _sets_differ( def assert_country_matches( connection: Connection, country: CountryCatalog, + *, + include_regions: bool = True, ) -> None: """Raise CatalogPublicationError if the stored version differs from the staged one. Rows are compared as sets in both directions, so a missing row and an extra - row both fail. + row both fail; the error names the first table that differs, never its + rows. Without ``include_regions``, region rows are not compared: they come + from the Public API's reference files, which may change while the + PolicyEngine.py version stays the same. A version published before catalog + format 2 is compared without format 2's fields. """ - for staged, actual in _comparison_pairs(country): + stored_format = stored_catalog_format(connection, country) + pairs = _comparison_pairs(country, with_format_2=stored_format is not None) + if include_regions: + pairs = (*pairs, ("regions", *_region_comparison_pair(country))) + for table_name, staged, actual in pairs: if _sets_differ(connection, staged, actual): - msg = f"persisted {country.country_id} catalog differs from PolicyEngine.py" + msg = ( + f"the stored {country.country_id} catalog version " + f"{country.model_version.version} differs from the staged catalog " + f"in its {table_name}" + ) raise CatalogPublicationError(msg) +def count_stored_regions(connection: Connection, country: CountryCatalog) -> int: + """Count the region rows stored for one country's catalog version.""" + _staged, actual = _region_comparison_pair(country) + return connection.execute( + sa.select(sa.func.count()).select_from(actual.subquery()) + ).scalar_one() + + +def count_region_differences( + connection: Connection, + country: CountryCatalog, +) -> tuple[int, int]: + """Count staged region rows the stored version lacks, and stored rows not staged.""" + staged, actual = _region_comparison_pair(country) + staged_not_stored = sa.except_(staged, actual).subquery() + stored_not_staged = sa.except_(actual, staged).subquery() + return ( + connection.execute( + sa.select(sa.func.count()).select_from(staged_not_stored) + ).scalar_one(), + connection.execute( + sa.select(sa.func.count()).select_from(stored_not_staged) + ).scalar_one(), + ) + + INSERT_MODEL = ( postgresql_insert(MODELS) .from_select( @@ -320,6 +439,9 @@ def assert_country_matches( "description", "current_law_id", "metadata_time_periods", + "catalog_format", + "entities", + "basic_inputs", ], sa.select( STAGE_CATALOG_MODELS.c.version_id, @@ -332,6 +454,12 @@ def assert_country_matches( STAGE_CATALOG_MODELS.c.metadata_time_periods, MODEL_VERSIONS.c.metadata_time_periods.type, ), + STAGE_CATALOG_MODELS.c.catalog_format, + sa.cast(STAGE_CATALOG_MODELS.c.entities, MODEL_VERSIONS.c.entities.type), + sa.cast( + STAGE_CATALOG_MODELS.c.basic_inputs, + MODEL_VERSIONS.c.basic_inputs.type, + ), ) .select_from( STAGE_CATALOG_MODELS.join( @@ -395,6 +523,14 @@ def assert_country_matches( "default_value", "adds", "subtracts", + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", "tax_benefit_model_version_id", ], sa.select( @@ -418,6 +554,17 @@ def assert_country_matches( STAGE_CATALOG_VARIABLES.c.subtracts, VARIABLES.c.subtracts.type, ), + STAGE_CATALOG_VARIABLES.c.is_input_variable, + STAGE_CATALOG_VARIABLES.c.hidden_input, + STAGE_CATALOG_VARIABLES.c.module_name, + STAGE_CATALOG_VARIABLES.c.adds_parameter_path, + STAGE_CATALOG_VARIABLES.c.subtracts_parameter_path, + sa.cast( + STAGE_CATALOG_VARIABLES.c.enum_values, + VARIABLES.c.enum_values.type, + ), + STAGE_CATALOG_VARIABLES.c.unit, + STAGE_CATALOG_VARIABLES.c.definition_period, MODEL_VERSIONS.c.id, ) .select_from( @@ -481,6 +628,8 @@ def assert_country_matches( "description", "data_type", "unit", + "economy", + "household", "tax_benefit_model_version_id", ], sa.select( @@ -491,6 +640,8 @@ def assert_country_matches( STAGE_CATALOG_PARAMETERS.c.description, STAGE_CATALOG_PARAMETERS.c.data_type, STAGE_CATALOG_PARAMETERS.c.unit, + STAGE_CATALOG_PARAMETERS.c.economy, + STAGE_CATALOG_PARAMETERS.c.household, MODEL_VERSIONS.c.id, ) .select_from( diff --git a/policyengine_api/data/v2/catalog/publication_staging.py b/policyengine_api/data/v2/catalog/publication_staging.py index 76820b27b..f4243fa64 100644 --- a/policyengine_api/data/v2/catalog/publication_staging.py +++ b/policyengine_api/data/v2/catalog/publication_staging.py @@ -16,8 +16,10 @@ from policyengine_api.data.v2.catalog.records import ( CountryCatalog, NormalizedCatalog, + VariableRecord, iter_batches, ) +from policyengine_api.data.v2.models import CatalogEntity, CatalogEnumValue if TYPE_CHECKING: from collections.abc import Callable, Iterable, Iterator, Sequence @@ -39,6 +41,9 @@ sa.Column("version_description", sa.Text), sa.Column("current_law_id", sa.Integer, nullable=False), sa.Column("metadata_time_periods", JSONB, nullable=False), + sa.Column("catalog_format", sa.Integer, nullable=False), + sa.Column("entities", JSONB, nullable=False), + sa.Column("basic_inputs", JSONB, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -57,6 +62,14 @@ sa.Column("default_value", JSONB, nullable=False), sa.Column("adds", JSONB), sa.Column("subtracts", JSONB), + sa.Column("is_input_variable", sa.Boolean), + sa.Column("hidden_input", sa.Boolean), + sa.Column("module_name", sa.Text), + sa.Column("adds_parameter_path", sa.Text), + sa.Column("subtracts_parameter_path", sa.Text), + sa.Column("enum_values", JSONB), + sa.Column("unit", sa.Text), + sa.Column("definition_period", sa.Text), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -83,6 +96,8 @@ sa.Column("description", sa.Text), sa.Column("data_type", sa.Text), sa.Column("unit", sa.Text), + sa.Column("economy", sa.Boolean, nullable=False), + sa.Column("household", sa.Boolean, nullable=False), prefixes=["TEMPORARY"], postgresql_on_commit="DROP", ) @@ -163,6 +178,30 @@ def _model_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: country.model_version.description, country.model_version.current_law_id, Jsonb(country.model_version.metadata_time_periods), + country.model_version.catalog_format, + Jsonb( + [ + CatalogEntity( + key=entity.key, + plural=entity.plural, + is_person=entity.is_person, + ) + for entity in country.model_version.entities + ] + ), + Jsonb(list(country.model_version.basic_inputs)), + ) + + +def _enum_values_json(record: VariableRecord) -> Jsonb | None: + """Return an enum variable's values as ordered value and label objects.""" + if record.enum_values is None: + return None + return Jsonb( + [ + CatalogEnumValue(value=item.value, label=item.label) + for item in record.enum_values + ] ) @@ -182,6 +221,14 @@ def _variable_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: Jsonb(record.default_value), _optional_json(record.adds), _optional_json(record.subtracts), + record.is_input_variable, + record.hidden_input, + record.module_name, + record.adds_parameter_path, + record.subtracts_parameter_path, + _enum_values_json(record), + record.unit, + record.definition_period, ) @@ -213,6 +260,8 @@ def _parameter_rows(country: CountryCatalog) -> Iterator[tuple[object, ...]]: record.description, record.data_type, record.unit, + record.economy, + record.household, ) diff --git a/policyengine_api/data/v2/catalog/publication_types.py b/policyengine_api/data/v2/catalog/publication_types.py index 884bcef07..515232560 100644 --- a/policyengine_api/data/v2/catalog/publication_types.py +++ b/policyengine_api/data/v2/catalog/publication_types.py @@ -18,6 +18,10 @@ class PublicationEvidence: entity_counts: dict[str, int] fallback_summaries: tuple[tuple[str, str, int], ...] elapsed_seconds: float + omitted_region_summaries: tuple[tuple[str, str, int], ...] = () + # Stored catalog versions whose region rows were kept although the staged + # regions differ: country, version, stored and staged region counts. + kept_region_summaries: tuple[tuple[str, str, int, int], ...] = () def as_dict(self) -> dict[str, object]: """Return the evidence as the JSON object printed after a publication.""" @@ -34,5 +38,22 @@ def as_dict(self) -> dict[str, object]: } for country_id, region_type, count in self.fallback_summaries ], + "omitted_region_summaries": [ + { + "country_id": country_id, + "region_type": region_type, + "count": count, + } + for country_id, region_type, count in self.omitted_region_summaries + ], + "kept_regions": [ + { + "country_id": country_id, + "policyengine_version": version, + "stored_regions": stored, + "staged_regions": staged, + } + for country_id, version, stored, staged in self.kept_region_summaries + ], "elapsed_seconds": round(self.elapsed_seconds, 3), } diff --git a/policyengine_api/data/v2/catalog/records.py b/policyengine_api/data/v2/catalog/records.py index afccc4add..622a35cb6 100644 --- a/policyengine_api/data/v2/catalog/records.py +++ b/policyengine_api/data/v2/catalog/records.py @@ -11,6 +11,37 @@ from uuid import UUID +CATALOG_FORMAT = 2 +"""The catalog content format this extractor publishes. + +Format 2 adds the metadata the app's v3 workspace reads, from PolicyEngine.py's +public records: entities and basic inputs; each variable's input and +hidden-input flags, module, the parameter its ``adds`` or ``subtracts`` names, +and enum value labels; each parameter's economy and household flags. Versions +published before it store null for these fields. +""" + + +@dataclass(frozen=True, slots=True) +class EntityRecord: + """A model entity: its key, plural name, and whether it is the person entity.""" + + key: str + plural: str + is_person: bool + + +@dataclass(frozen=True, slots=True) +class EnumValueRecord: + """One value of an enum variable: its stored name and its declared label. + + The label is kept as the country package declares it, usually a string. + """ + + value: str + label: str | int | float | bool | None + + @dataclass(frozen=True, slots=True) class ModelRecord: """One country's tax-benefit model as named in the catalog.""" @@ -35,11 +66,19 @@ class ModelVersionRecord: description: str | None current_law_id: int metadata_time_periods: tuple[int, ...] + catalog_format: int + entities: tuple[EntityRecord, ...] + basic_inputs: tuple[str, ...] @dataclass(frozen=True, slots=True) class VariableRecord: - """One model variable, with its entity, type and the variables it sums.""" + """One model variable, with its entity, type and the variables it sums. + + ``adds_parameter_path`` and ``subtracts_parameter_path`` name the parameter + whose dated value lists the variables summed; ``adds`` and ``subtracts`` + keep the list PolicyEngine.py resolved from it. + """ id: UUID model_version_id: UUID @@ -52,11 +91,22 @@ class VariableRecord: default_value: Any adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: tuple[EnumValueRecord, ...] | None + unit: str | None + definition_period: str | None @dataclass(frozen=True, slots=True) class ParameterNodeRecord: - """A non-leaf node of the parameter tree; it groups parameters and has no values.""" + """A non-leaf node of the parameter tree; it groups parameters and has no values. + + Bracket schedules are nodes too, with their stored label or none. + """ id: UUID model_version_id: UUID @@ -88,6 +138,8 @@ class ParameterRecord: data_type: str | None unit: str | None values: tuple[ParameterValueRecord, ...] + economy: bool + household: bool @dataclass(frozen=True, slots=True) @@ -126,11 +178,37 @@ class RegionRecord: state_name: str | None +@dataclass(frozen=True, slots=True) +class CertifiedArtifact: + """One artifact of a country's certified data release, pinned to a revision. + + ``repo_type`` is the Hugging Face repository type, ``model`` or ``dataset``. + """ + + name: str + repo_id: str + repo_type: str + path: str + revision: str + sha256: str | None + + @dataclass(frozen=True, slots=True) class FallbackSummary: - """How many US sub-national regions of one type fell back to the default dataset. + """How many sub-national regions of one type fell back to the default dataset. + + The certified data release names no dataset for their type. + """ + + region_type: str + count: int + + +@dataclass(frozen=True, slots=True) +class OmittedRegionSummary: + """How many reference regions of one type publication left out. - These regions had no dataset path of their own. + No scoping rule can select their households yet, such as US places. """ region_type: str @@ -166,6 +244,9 @@ class CountryCatalog: datasets: tuple[DatasetRecord, ...] regions: tuple[RegionRecord, ...] fallback_summaries: tuple[FallbackSummary, ...] + omitted_region_summaries: tuple[OmittedRegionSummary, ...] = () + # The release datasets the regions' row filters read. + certified_artifacts: tuple[CertifiedArtifact, ...] = () def parameter_value_batches( self, diff --git a/policyengine_api/data/v2/catalog/region_data_verification.py b/policyengine_api/data/v2/catalog/region_data_verification.py new file mode 100644 index 000000000..e7eda396a --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_data_verification.py @@ -0,0 +1,126 @@ +"""Refuse to publish regions whose households the certified data cannot select. + +A row-filtered region needs its filter column among the household columns of +its dataset. +""" + +from __future__ import annotations + +from collections import defaultdict +from functools import partial +from typing import TYPE_CHECKING, Protocol, TypeVar + +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError +from policyengine_api.data.v2.catalog.region_scoping import ROW_FILTER + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from policyengine_api.data.v2.catalog.records import ( + CertifiedArtifact, + CountryCatalog, + NormalizedCatalog, + ) + +# Region codes named in an error message, at most. +MAX_LISTED_REGIONS = 3 + +_Result = TypeVar("_Result") + + +class RegionDataVerificationError(RuntimeError): + """Raised when certified data cannot select a region's households. + + Also raised when an artifact cannot be read for that check. Verification + runs after the stored catalog versions are read and before anything is + published; its messages name only artifacts, columns, codes and counts. + """ + + +class CertifiedDataInspector(Protocol): + """Reads what verification needs from one certified artifact.""" + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + """Return the household column names of a dataset artifact.""" + ... + + +def _listed(codes: Sequence[str]) -> str: + shown = ", ".join(codes[:MAX_LISTED_REGIONS]) + more = len(codes) - MAX_LISTED_REGIONS + return f"{shown} and {more} more" if more > 0 else shown + + +def _noun(count: int) -> str: + return "region" if count == 1 else "regions" + + +def _read(artifact: CertifiedArtifact, read: Callable[[], _Result]) -> _Result: + """Run one read of an artifact, reporting a failure safely. + + A structure error keeps its own message; any other failure is reduced to + the artifact's name and the error type, since its text may hold a location. + """ + try: + return read() + except RegionDataVerificationError: + raise + except CertifiedDataReadError as error: + # Its message names only HDF5 entries and counts. + msg = f"the certified artifact {artifact.name} cannot be checked: {error}" + raise RegionDataVerificationError(msg) from error + except Exception as error: + # Only the artifact's name and the error type: the error text may hold + # its location, and command output names no dataset release. + msg = ( + f"could not read the certified artifact {artifact.name}: " + f"{type(error).__name__}" + ) + raise RegionDataVerificationError(msg) from error + + +def _verify_row_filters( + country: CountryCatalog, + artifacts: Mapping[str, CertifiedArtifact], + inspector: CertifiedDataInspector, +) -> None: + dataset_names = {dataset.id: dataset.name for dataset in country.datasets} + regions_by_column: dict[tuple[str, str], list[str]] = defaultdict(list) + for region in country.regions: + if region.filter_strategy == ROW_FILTER and region.filter_field: + dataset_name = dataset_names[region.default_dataset_id] + regions_by_column[dataset_name, region.filter_field].append(region.code) + columns_by_dataset: dict[str, frozenset[str]] = {} + for (dataset_name, column), codes in sorted(regions_by_column.items()): + if dataset_name not in columns_by_dataset: + artifact = artifacts[dataset_name] + columns_by_dataset[dataset_name] = _read( + artifact, + partial(inspector.household_columns, artifact), + ) + if column not in columns_by_dataset[dataset_name]: + msg = ( + f"the certified dataset {dataset_name} has no household column " + f"{column}, which {len(codes)} {country.country_id} " + f"{_noun(len(codes))} {'filters' if len(codes) == 1 else 'filter'} " + f"on ({_listed(codes)})" + ) + raise RegionDataVerificationError(msg) + + +def verify_region_data( + catalog: NormalizedCatalog, + inspector: CertifiedDataInspector, + *, + countries: Sequence[str], +) -> None: + """Check the regions of the named countries against their certified artifacts. + + Each artifact is read at most once. + """ + for country_id in countries: + country = catalog.country(country_id) + artifacts = { + artifact.name: artifact for artifact in country.certified_artifacts + } + _verify_row_filters(country, artifacts, inspector) diff --git a/policyengine_api/data/v2/catalog/region_reference_files/uk.csv b/policyengine_api/data/v2/catalog/region_reference_files/uk.csv new file mode 100644 index 000000000..e20f450de --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_reference_files/uk.csv @@ -0,0 +1,1017 @@ +code,label,region_type,parent_code,state_code,state_name,filter_value,v1_code +uk,United Kingdom,national,,,,,uk +country/england,England,country,uk,,,"[""NORTH_EAST"",""NORTH_WEST"",""YORKSHIRE"",""EAST_MIDLANDS"",""WEST_MIDLANDS"",""EAST_OF_ENGLAND"",""LONDON"",""SOUTH_EAST"",""SOUTH_WEST""]",country/england +country/scotland,Scotland,country,uk,,,SCOTLAND,country/scotland +country/wales,Wales,country,uk,,,WALES,country/wales +country/northern_ireland,Northern Ireland,country,uk,,,NORTHERN_IRELAND,country/ni +constituency/E14001063,Aldershot,constituency,country/england,,,E14001063,constituency/Aldershot +constituency/E14001064,Aldridge-Brownhills,constituency,country/england,,,E14001064,constituency/Aldridge-Brownhills +constituency/E14001065,Altrincham and Sale West,constituency,country/england,,,E14001065,constituency/Altrincham and Sale West +constituency/E14001066,Amber Valley,constituency,country/england,,,E14001066,constituency/Amber Valley +constituency/E14001067,Arundel and South Downs,constituency,country/england,,,E14001067,constituency/Arundel and South Downs +constituency/E14001068,Ashfield,constituency,country/england,,,E14001068,constituency/Ashfield +constituency/E14001069,Ashford,constituency,country/england,,,E14001069,constituency/Ashford +constituency/E14001070,Ashton-under-Lyne,constituency,country/england,,,E14001070,constituency/Ashton-under-Lyne +constituency/E14001071,Aylesbury,constituency,country/england,,,E14001071,constituency/Aylesbury +constituency/E14001072,Banbury,constituency,country/england,,,E14001072,constituency/Banbury +constituency/E14001073,Barking,constituency,country/england,,,E14001073,constituency/Barking +constituency/E14001074,Barnsley North,constituency,country/england,,,E14001074,constituency/Barnsley North +constituency/E14001075,Barnsley South,constituency,country/england,,,E14001075,constituency/Barnsley South +constituency/E14001076,Barrow and Furness,constituency,country/england,,,E14001076,constituency/Barrow and Furness +constituency/E14001077,Basildon and Billericay,constituency,country/england,,,E14001077,constituency/Basildon and Billericay +constituency/E14001078,Basingstoke,constituency,country/england,,,E14001078,constituency/Basingstoke +constituency/E14001079,Bassetlaw,constituency,country/england,,,E14001079,constituency/Bassetlaw +constituency/E14001080,Bath,constituency,country/england,,,E14001080,constituency/Bath +constituency/E14001081,Battersea,constituency,country/england,,,E14001081,constituency/Battersea +constituency/E14001082,Beaconsfield,constituency,country/england,,,E14001082,constituency/Beaconsfield +constituency/E14001083,Beckenham and Penge,constituency,country/england,,,E14001083,constituency/Beckenham and Penge +constituency/E14001084,Bedford,constituency,country/england,,,E14001084,constituency/Bedford +constituency/E14001085,Bermondsey and Old Southwark,constituency,country/england,,,E14001085,constituency/Bermondsey and Old Southwark +constituency/E14001086,Bethnal Green and Stepney,constituency,country/england,,,E14001086,constituency/Bethnal Green and Stepney +constituency/E14001087,Beverley and Holderness,constituency,country/england,,,E14001087,constituency/Beverley and Holderness +constituency/E14001088,Bexhill and Battle,constituency,country/england,,,E14001088,constituency/Bexhill and Battle +constituency/E14001089,Bexleyheath and Crayford,constituency,country/england,,,E14001089,constituency/Bexleyheath and Crayford +constituency/E14001090,Bicester and Woodstock,constituency,country/england,,,E14001090,constituency/Bicester and Woodstock +constituency/E14001091,Birkenhead,constituency,country/england,,,E14001091,constituency/Birkenhead +constituency/E14001092,Birmingham Edgbaston,constituency,country/england,,,E14001092,constituency/Birmingham Edgbaston +constituency/E14001093,Birmingham Erdington,constituency,country/england,,,E14001093,constituency/Birmingham Erdington +constituency/E14001094,Birmingham Hall Green and Moseley,constituency,country/england,,,E14001094,constituency/Birmingham Hall Green and Moseley +constituency/E14001095,Birmingham Hodge Hill and Solihull North,constituency,country/england,,,E14001095,constituency/Birmingham Hodge Hill and Solihull North +constituency/E14001096,Birmingham Ladywood,constituency,country/england,,,E14001096,constituency/Birmingham Ladywood +constituency/E14001097,Birmingham Northfield,constituency,country/england,,,E14001097,constituency/Birmingham Northfield +constituency/E14001098,Birmingham Perry Barr,constituency,country/england,,,E14001098,constituency/Birmingham Perry Barr +constituency/E14001099,Birmingham Selly Oak,constituency,country/england,,,E14001099,constituency/Birmingham Selly Oak +constituency/E14001100,Birmingham Yardley,constituency,country/england,,,E14001100,constituency/Birmingham Yardley +constituency/E14001101,Bishop Auckland,constituency,country/england,,,E14001101,constituency/Bishop Auckland +constituency/E14001102,Blackburn,constituency,country/england,,,E14001102,constituency/Blackburn +constituency/E14001103,Blackley and Middleton South,constituency,country/england,,,E14001103,constituency/Blackley and Middleton South +constituency/E14001104,Blackpool North and Fleetwood,constituency,country/england,,,E14001104,constituency/Blackpool North and Fleetwood +constituency/E14001105,Blackpool South,constituency,country/england,,,E14001105,constituency/Blackpool South +constituency/E14001106,Blaydon and Consett,constituency,country/england,,,E14001106,constituency/Blaydon and Consett +constituency/E14001107,Blyth and Ashington,constituency,country/england,,,E14001107,constituency/Blyth and Ashington +constituency/E14001108,Bognor Regis and Littlehampton,constituency,country/england,,,E14001108,constituency/Bognor Regis and Littlehampton +constituency/E14001109,Bolsover,constituency,country/england,,,E14001109,constituency/Bolsover +constituency/E14001110,Bolton North East,constituency,country/england,,,E14001110,constituency/Bolton North East +constituency/E14001111,Bolton South and Walkden,constituency,country/england,,,E14001111,constituency/Bolton South and Walkden +constituency/E14001112,Bolton West,constituency,country/england,,,E14001112,constituency/Bolton West +constituency/E14001113,Bootle,constituency,country/england,,,E14001113,constituency/Bootle +constituency/E14001114,Boston and Skegness,constituency,country/england,,,E14001114,constituency/Boston and Skegness +constituency/E14001115,Bournemouth East,constituency,country/england,,,E14001115,constituency/Bournemouth East +constituency/E14001116,Bournemouth West,constituency,country/england,,,E14001116,constituency/Bournemouth West +constituency/E14001117,Bracknell,constituency,country/england,,,E14001117,constituency/Bracknell +constituency/E14001118,Bradford East,constituency,country/england,,,E14001118,constituency/Bradford East +constituency/E14001119,Bradford South,constituency,country/england,,,E14001119,constituency/Bradford South +constituency/E14001120,Bradford West,constituency,country/england,,,E14001120,constituency/Bradford West +constituency/E14001121,Braintree,constituency,country/england,,,E14001121,constituency/Braintree +constituency/E14001122,Brent East,constituency,country/england,,,E14001122,constituency/Brent East +constituency/E14001123,Brent West,constituency,country/england,,,E14001123,constituency/Brent West +constituency/E14001124,Brentford and Isleworth,constituency,country/england,,,E14001124,constituency/Brentford and Isleworth +constituency/E14001125,Brentwood and Ongar,constituency,country/england,,,E14001125,constituency/Brentwood and Ongar +constituency/E14001126,Bridgwater,constituency,country/england,,,E14001126,constituency/Bridgwater +constituency/E14001127,Bridlington and The Wolds,constituency,country/england,,,E14001127,constituency/Bridlington and The Wolds +constituency/E14001128,Brigg and Immingham,constituency,country/england,,,E14001128,constituency/Brigg and Immingham +constituency/E14001129,Brighton Kemptown and Peacehaven,constituency,country/england,,,E14001129,constituency/Brighton Kemptown and Peacehaven +constituency/E14001130,Brighton Pavilion,constituency,country/england,,,E14001130,constituency/Brighton Pavilion +constituency/E14001131,Bristol Central,constituency,country/england,,,E14001131,constituency/Bristol Central +constituency/E14001132,Bristol East,constituency,country/england,,,E14001132,constituency/Bristol East +constituency/E14001133,Bristol North East,constituency,country/england,,,E14001133,constituency/Bristol North East +constituency/E14001134,Bristol North West,constituency,country/england,,,E14001134,constituency/Bristol North West +constituency/E14001135,Bristol South,constituency,country/england,,,E14001135,constituency/Bristol South +constituency/E14001136,Broadland and Fakenham,constituency,country/england,,,E14001136,constituency/Broadland and Fakenham +constituency/E14001137,Bromley and Biggin Hill,constituency,country/england,,,E14001137,constituency/Bromley and Biggin Hill +constituency/E14001138,Bromsgrove,constituency,country/england,,,E14001138,constituency/Bromsgrove +constituency/E14001139,Broxbourne,constituency,country/england,,,E14001139,constituency/Broxbourne +constituency/E14001140,Broxtowe,constituency,country/england,,,E14001140,constituency/Broxtowe +constituency/E14001141,Buckingham and Bletchley,constituency,country/england,,,E14001141,constituency/Buckingham and Bletchley +constituency/E14001142,Burnley,constituency,country/england,,,E14001142,constituency/Burnley +constituency/E14001143,Burton and Uttoxeter,constituency,country/england,,,E14001143,constituency/Burton and Uttoxeter +constituency/E14001144,Bury North,constituency,country/england,,,E14001144,constituency/Bury North +constituency/E14001145,Bury South,constituency,country/england,,,E14001145,constituency/Bury South +constituency/E14001146,Bury St Edmunds and Stowmarket,constituency,country/england,,,E14001146,constituency/Bury St Edmunds and Stowmarket +constituency/E14001147,Calder Valley,constituency,country/england,,,E14001147,constituency/Calder Valley +constituency/E14001148,Camborne and Redruth,constituency,country/england,,,E14001148,constituency/Camborne and Redruth +constituency/E14001149,Cambridge,constituency,country/england,,,E14001149,constituency/Cambridge +constituency/E14001150,Cannock Chase,constituency,country/england,,,E14001150,constituency/Cannock Chase +constituency/E14001151,Canterbury,constituency,country/england,,,E14001151,constituency/Canterbury +constituency/E14001152,Carlisle,constituency,country/england,,,E14001152,constituency/Carlisle +constituency/E14001153,Carshalton and Wallington,constituency,country/england,,,E14001153,constituency/Carshalton and Wallington +constituency/E14001154,Castle Point,constituency,country/england,,,E14001154,constituency/Castle Point +constituency/E14001155,Central Devon,constituency,country/england,,,E14001155,constituency/Central Devon +constituency/E14001156,Central Suffolk and North Ipswich,constituency,country/england,,,E14001156,constituency/Central Suffolk and North Ipswich +constituency/E14001157,Chatham and Aylesford,constituency,country/england,,,E14001157,constituency/Chatham and Aylesford +constituency/E14001158,Cheadle,constituency,country/england,,,E14001158,constituency/Cheadle +constituency/E14001159,Chelmsford,constituency,country/england,,,E14001159,constituency/Chelmsford +constituency/E14001160,Chelsea and Fulham,constituency,country/england,,,E14001160,constituency/Chelsea and Fulham +constituency/E14001161,Cheltenham,constituency,country/england,,,E14001161,constituency/Cheltenham +constituency/E14001162,Chesham and Amersham,constituency,country/england,,,E14001162,constituency/Chesham and Amersham +constituency/E14001163,Chester North and Neston,constituency,country/england,,,E14001163,constituency/Chester North and Neston +constituency/E14001164,Chester South and Eddisbury,constituency,country/england,,,E14001164,constituency/Chester South and Eddisbury +constituency/E14001165,Chesterfield,constituency,country/england,,,E14001165,constituency/Chesterfield +constituency/E14001166,Chichester,constituency,country/england,,,E14001166,constituency/Chichester +constituency/E14001167,Chingford and Woodford Green,constituency,country/england,,,E14001167,constituency/Chingford and Woodford Green +constituency/E14001168,Chippenham,constituency,country/england,,,E14001168,constituency/Chippenham +constituency/E14001169,Chipping Barnet,constituency,country/england,,,E14001169,constituency/Chipping Barnet +constituency/E14001170,Chorley,constituency,country/england,,,E14001170,constituency/Chorley +constituency/E14001171,Christchurch,constituency,country/england,,,E14001171,constituency/Christchurch +constituency/E14001172,Cities of London and Westminster,constituency,country/england,,,E14001172,constituency/Cities of London and Westminster +constituency/E14001173,City of Durham,constituency,country/england,,,E14001173,constituency/City of Durham +constituency/E14001174,Clacton,constituency,country/england,,,E14001174,constituency/Clacton +constituency/E14001175,Clapham and Brixton Hill,constituency,country/england,,,E14001175,constituency/Clapham and Brixton Hill +constituency/E14001176,Colchester,constituency,country/england,,,E14001176,constituency/Colchester +constituency/E14001177,Colne Valley,constituency,country/england,,,E14001177,constituency/Colne Valley +constituency/E14001178,Congleton,constituency,country/england,,,E14001178,constituency/Congleton +constituency/E14001179,Corby and East Northamptonshire,constituency,country/england,,,E14001179,constituency/Corby and East Northamptonshire +constituency/E14001180,Coventry East,constituency,country/england,,,E14001180,constituency/Coventry East +constituency/E14001181,Coventry North West,constituency,country/england,,,E14001181,constituency/Coventry North West +constituency/E14001182,Coventry South,constituency,country/england,,,E14001182,constituency/Coventry South +constituency/E14001183,Cramlington and Killingworth,constituency,country/england,,,E14001183,constituency/Cramlington and Killingworth +constituency/E14001184,Crawley,constituency,country/england,,,E14001184,constituency/Crawley +constituency/E14001185,Crewe and Nantwich,constituency,country/england,,,E14001185,constituency/Crewe and Nantwich +constituency/E14001186,Croydon East,constituency,country/england,,,E14001186,constituency/Croydon East +constituency/E14001187,Croydon South,constituency,country/england,,,E14001187,constituency/Croydon South +constituency/E14001188,Croydon West,constituency,country/england,,,E14001188,constituency/Croydon West +constituency/E14001189,Dagenham and Rainham,constituency,country/england,,,E14001189,constituency/Dagenham and Rainham +constituency/E14001190,Darlington,constituency,country/england,,,E14001190,constituency/Darlington +constituency/E14001191,Dartford,constituency,country/england,,,E14001191,constituency/Dartford +constituency/E14001192,Daventry,constituency,country/england,,,E14001192,constituency/Daventry +constituency/E14001193,Derby North,constituency,country/england,,,E14001193,constituency/Derby North +constituency/E14001194,Derby South,constituency,country/england,,,E14001194,constituency/Derby South +constituency/E14001195,Derbyshire Dales,constituency,country/england,,,E14001195,constituency/Derbyshire Dales +constituency/E14001196,Dewsbury and Batley,constituency,country/england,,,E14001196,constituency/Dewsbury and Batley +constituency/E14001197,Didcot and Wantage,constituency,country/england,,,E14001197,constituency/Didcot and Wantage +constituency/E14001198,Doncaster Central,constituency,country/england,,,E14001198,constituency/Doncaster Central +constituency/E14001199,Doncaster East and the Isle of Axholme,constituency,country/england,,,E14001199,constituency/Doncaster East and the Isle of Axholme +constituency/E14001200,Doncaster North,constituency,country/england,,,E14001200,constituency/Doncaster North +constituency/E14001201,Dorking and Horley,constituency,country/england,,,E14001201,constituency/Dorking and Horley +constituency/E14001202,Dover and Deal,constituency,country/england,,,E14001202,constituency/Dover and Deal +constituency/E14001203,Droitwich and Evesham,constituency,country/england,,,E14001203,constituency/Droitwich and Evesham +constituency/E14001204,Dudley,constituency,country/england,,,E14001204,constituency/Dudley +constituency/E14001205,Dulwich and West Norwood,constituency,country/england,,,E14001205,constituency/Dulwich and West Norwood +constituency/E14001206,Dunstable and Leighton Buzzard,constituency,country/england,,,E14001206,constituency/Dunstable and Leighton Buzzard +constituency/E14001207,Ealing Central and Acton,constituency,country/england,,,E14001207,constituency/Ealing Central and Acton +constituency/E14001208,Ealing North,constituency,country/england,,,E14001208,constituency/Ealing North +constituency/E14001209,Ealing Southall,constituency,country/england,,,E14001209,constituency/Ealing Southall +constituency/E14001210,Earley and Woodley,constituency,country/england,,,E14001210,constituency/Earley and Woodley +constituency/E14001211,Easington,constituency,country/england,,,E14001211,constituency/Easington +constituency/E14001212,East Grinstead and Uckfield,constituency,country/england,,,E14001212,constituency/East Grinstead and Uckfield +constituency/E14001213,East Ham,constituency,country/england,,,E14001213,constituency/East Ham +constituency/E14001214,East Hampshire,constituency,country/england,,,E14001214,constituency/East Hampshire +constituency/E14001215,East Surrey,constituency,country/england,,,E14001215,constituency/East Surrey +constituency/E14001216,East Thanet,constituency,country/england,,,E14001216,constituency/East Thanet +constituency/E14001217,East Wiltshire,constituency,country/england,,,E14001217,constituency/East Wiltshire +constituency/E14001218,East Worthing and Shoreham,constituency,country/england,,,E14001218,constituency/East Worthing and Shoreham +constituency/E14001219,Eastbourne,constituency,country/england,,,E14001219,constituency/Eastbourne +constituency/E14001220,Eastleigh,constituency,country/england,,,E14001220,constituency/Eastleigh +constituency/E14001221,Edmonton and Winchmore Hill,constituency,country/england,,,E14001221,constituency/Edmonton and Winchmore Hill +constituency/E14001222,Ellesmere Port and Bromborough,constituency,country/england,,,E14001222,constituency/Ellesmere Port and Bromborough +constituency/E14001223,Eltham and Chislehurst,constituency,country/england,,,E14001223,constituency/Eltham and Chislehurst +constituency/E14001224,Ely and East Cambridgeshire,constituency,country/england,,,E14001224,constituency/Ely and East Cambridgeshire +constituency/E14001225,Enfield North,constituency,country/england,,,E14001225,constituency/Enfield North +constituency/E14001226,Epping Forest,constituency,country/england,,,E14001226,constituency/Epping Forest +constituency/E14001227,Epsom and Ewell,constituency,country/england,,,E14001227,constituency/Epsom and Ewell +constituency/E14001228,Erewash,constituency,country/england,,,E14001228,constituency/Erewash +constituency/E14001229,Erith and Thamesmead,constituency,country/england,,,E14001229,constituency/Erith and Thamesmead +constituency/E14001230,Esher and Walton,constituency,country/england,,,E14001230,constituency/Esher and Walton +constituency/E14001231,Exeter,constituency,country/england,,,E14001231,constituency/Exeter +constituency/E14001232,Exmouth and Exeter East,constituency,country/england,,,E14001232,constituency/Exmouth and Exeter East +constituency/E14001233,Fareham and Waterlooville,constituency,country/england,,,E14001233,constituency/Fareham and Waterlooville +constituency/E14001234,Farnham and Bordon,constituency,country/england,,,E14001234,constituency/Farnham and Bordon +constituency/E14001235,Faversham and Mid Kent,constituency,country/england,,,E14001235,constituency/Faversham and Mid Kent +constituency/E14001236,Feltham and Heston,constituency,country/england,,,E14001236,constituency/Feltham and Heston +constituency/E14001237,Filton and Bradley Stoke,constituency,country/england,,,E14001237,constituency/Filton and Bradley Stoke +constituency/E14001238,Finchley and Golders Green,constituency,country/england,,,E14001238,constituency/Finchley and Golders Green +constituency/E14001239,Folkestone and Hythe,constituency,country/england,,,E14001239,constituency/Folkestone and Hythe +constituency/E14001240,Forest of Dean,constituency,country/england,,,E14001240,constituency/Forest of Dean +constituency/E14001241,Frome and East Somerset,constituency,country/england,,,E14001241,constituency/Frome and East Somerset +constituency/E14001242,Fylde,constituency,country/england,,,E14001242,constituency/Fylde +constituency/E14001243,Gainsborough,constituency,country/england,,,E14001243,constituency/Gainsborough +constituency/E14001244,Gateshead Central and Whickham,constituency,country/england,,,E14001244,constituency/Gateshead Central and Whickham +constituency/E14001245,Gedling,constituency,country/england,,,E14001245,constituency/Gedling +constituency/E14001246,Gillingham and Rainham,constituency,country/england,,,E14001246,constituency/Gillingham and Rainham +constituency/E14001247,Glastonbury and Somerton,constituency,country/england,,,E14001247,constituency/Glastonbury and Somerton +constituency/E14001248,Gloucester,constituency,country/england,,,E14001248,constituency/Gloucester +constituency/E14001249,Godalming and Ash,constituency,country/england,,,E14001249,constituency/Godalming and Ash +constituency/E14001250,Goole and Pocklington,constituency,country/england,,,E14001250,constituency/Goole and Pocklington +constituency/E14001251,Gorton and Denton,constituency,country/england,,,E14001251,constituency/Gorton and Denton +constituency/E14001252,Gosport,constituency,country/england,,,E14001252,constituency/Gosport +constituency/E14001253,Grantham and Bourne,constituency,country/england,,,E14001253,constituency/Grantham and Bourne +constituency/E14001254,Gravesham,constituency,country/england,,,E14001254,constituency/Gravesham +constituency/E14001255,Great Grimsby and Cleethorpes,constituency,country/england,,,E14001255,constituency/Great Grimsby and Cleethorpes +constituency/E14001256,Great Yarmouth,constituency,country/england,,,E14001256,constituency/Great Yarmouth +constituency/E14001257,Greenwich and Woolwich,constituency,country/england,,,E14001257,constituency/Greenwich and Woolwich +constituency/E14001258,Guildford,constituency,country/england,,,E14001258,constituency/Guildford +constituency/E14001259,Hackney North and Stoke Newington,constituency,country/england,,,E14001259,constituency/Hackney North and Stoke Newington +constituency/E14001260,Hackney South and Shoreditch,constituency,country/england,,,E14001260,constituency/Hackney South and Shoreditch +constituency/E14001261,Halesowen,constituency,country/england,,,E14001261,constituency/Halesowen +constituency/E14001262,Halifax,constituency,country/england,,,E14001262,constituency/Halifax +constituency/E14001263,Hamble Valley,constituency,country/england,,,E14001263,constituency/Hamble Valley +constituency/E14001264,Hammersmith and Chiswick,constituency,country/england,,,E14001264,constituency/Hammersmith and Chiswick +constituency/E14001265,Hampstead and Highgate,constituency,country/england,,,E14001265,constituency/Hampstead and Highgate +constituency/E14001266,"Harborough, Oadby and Wigston",constituency,country/england,,,E14001266,"constituency/Harborough, Oadby and Wigston" +constituency/E14001267,Harlow,constituency,country/england,,,E14001267,constituency/Harlow +constituency/E14001268,Harpenden and Berkhamsted,constituency,country/england,,,E14001268,constituency/Harpenden and Berkhamsted +constituency/E14001269,Harrogate and Knaresborough,constituency,country/england,,,E14001269,constituency/Harrogate and Knaresborough +constituency/E14001270,Harrow East,constituency,country/england,,,E14001270,constituency/Harrow East +constituency/E14001271,Harrow West,constituency,country/england,,,E14001271,constituency/Harrow West +constituency/E14001272,Hartlepool,constituency,country/england,,,E14001272,constituency/Hartlepool +constituency/E14001273,Harwich and North Essex,constituency,country/england,,,E14001273,constituency/Harwich and North Essex +constituency/E14001274,Hastings and Rye,constituency,country/england,,,E14001274,constituency/Hastings and Rye +constituency/E14001275,Havant,constituency,country/england,,,E14001275,constituency/Havant +constituency/E14001276,Hayes and Harlington,constituency,country/england,,,E14001276,constituency/Hayes and Harlington +constituency/E14001277,Hazel Grove,constituency,country/england,,,E14001277,constituency/Hazel Grove +constituency/E14001278,Hemel Hempstead,constituency,country/england,,,E14001278,constituency/Hemel Hempstead +constituency/E14001279,Hendon,constituency,country/england,,,E14001279,constituency/Hendon +constituency/E14001280,Henley and Thame,constituency,country/england,,,E14001280,constituency/Henley and Thame +constituency/E14001281,Hereford and South Herefordshire,constituency,country/england,,,E14001281,constituency/Hereford and South Herefordshire +constituency/E14001282,Herne Bay and Sandwich,constituency,country/england,,,E14001282,constituency/Herne Bay and Sandwich +constituency/E14001283,Hertford and Stortford,constituency,country/england,,,E14001283,constituency/Hertford and Stortford +constituency/E14001284,Hertsmere,constituency,country/england,,,E14001284,constituency/Hertsmere +constituency/E14001285,Hexham,constituency,country/england,,,E14001285,constituency/Hexham +constituency/E14001286,Heywood and Middleton North,constituency,country/england,,,E14001286,constituency/Heywood and Middleton North +constituency/E14001287,High Peak,constituency,country/england,,,E14001287,constituency/High Peak +constituency/E14001288,Hinckley and Bosworth,constituency,country/england,,,E14001288,constituency/Hinckley and Bosworth +constituency/E14001289,Hitchin,constituency,country/england,,,E14001289,constituency/Hitchin +constituency/E14001290,Holborn and St Pancras,constituency,country/england,,,E14001290,constituency/Holborn and St Pancras +constituency/E14001291,Honiton and Sidmouth,constituency,country/england,,,E14001291,constituency/Honiton and Sidmouth +constituency/E14001292,Hornchurch and Upminster,constituency,country/england,,,E14001292,constituency/Hornchurch and Upminster +constituency/E14001293,Hornsey and Friern Barnet,constituency,country/england,,,E14001293,constituency/Hornsey and Friern Barnet +constituency/E14001294,Horsham,constituency,country/england,,,E14001294,constituency/Horsham +constituency/E14001295,Houghton and Sunderland South,constituency,country/england,,,E14001295,constituency/Houghton and Sunderland South +constituency/E14001296,Hove and Portslade,constituency,country/england,,,E14001296,constituency/Hove and Portslade +constituency/E14001297,Huddersfield,constituency,country/england,,,E14001297,constituency/Huddersfield +constituency/E14001298,Huntingdon,constituency,country/england,,,E14001298,constituency/Huntingdon +constituency/E14001299,Hyndburn,constituency,country/england,,,E14001299,constituency/Hyndburn +constituency/E14001300,Ilford North,constituency,country/england,,,E14001300,constituency/Ilford North +constituency/E14001301,Ilford South,constituency,country/england,,,E14001301,constituency/Ilford South +constituency/E14001302,Ipswich,constituency,country/england,,,E14001302,constituency/Ipswich +constituency/E14001303,Isle of Wight East,constituency,country/england,,,E14001303,constituency/Isle of Wight East +constituency/E14001304,Isle of Wight West,constituency,country/england,,,E14001304,constituency/Isle of Wight West +constituency/E14001305,Islington North,constituency,country/england,,,E14001305,constituency/Islington North +constituency/E14001306,Islington South and Finsbury,constituency,country/england,,,E14001306,constituency/Islington South and Finsbury +constituency/E14001307,Jarrow and Gateshead East,constituency,country/england,,,E14001307,constituency/Jarrow and Gateshead East +constituency/E14001308,Keighley and Ilkley,constituency,country/england,,,E14001308,constituency/Keighley and Ilkley +constituency/E14001309,Kenilworth and Southam,constituency,country/england,,,E14001309,constituency/Kenilworth and Southam +constituency/E14001310,Kensington and Bayswater,constituency,country/england,,,E14001310,constituency/Kensington and Bayswater +constituency/E14001311,Kettering,constituency,country/england,,,E14001311,constituency/Kettering +constituency/E14001312,Kingston and Surbiton,constituency,country/england,,,E14001312,constituency/Kingston and Surbiton +constituency/E14001313,Kingston upon Hull East,constituency,country/england,,,E14001313,constituency/Kingston upon Hull East +constituency/E14001314,Kingston upon Hull North and Cottingham,constituency,country/england,,,E14001314,constituency/Kingston upon Hull North and Cottingham +constituency/E14001315,Kingston upon Hull West and Haltemprice,constituency,country/england,,,E14001315,constituency/Kingston upon Hull West and Haltemprice +constituency/E14001316,Kingswinford and South Staffordshire,constituency,country/england,,,E14001316,constituency/Kingswinford and South Staffordshire +constituency/E14001317,Knowsley,constituency,country/england,,,E14001317,constituency/Knowsley +constituency/E14001318,Lancaster and Wyre,constituency,country/england,,,E14001318,constituency/Lancaster and Wyre +constituency/E14001319,Leeds Central and Headingley,constituency,country/england,,,E14001319,constituency/Leeds Central and Headingley +constituency/E14001320,Leeds East,constituency,country/england,,,E14001320,constituency/Leeds East +constituency/E14001321,Leeds North East,constituency,country/england,,,E14001321,constituency/Leeds North East +constituency/E14001322,Leeds North West,constituency,country/england,,,E14001322,constituency/Leeds North West +constituency/E14001323,Leeds South,constituency,country/england,,,E14001323,constituency/Leeds South +constituency/E14001324,Leeds South West and Morley,constituency,country/england,,,E14001324,constituency/Leeds South West and Morley +constituency/E14001325,Leeds West and Pudsey,constituency,country/england,,,E14001325,constituency/Leeds West and Pudsey +constituency/E14001326,Leicester East,constituency,country/england,,,E14001326,constituency/Leicester East +constituency/E14001327,Leicester South,constituency,country/england,,,E14001327,constituency/Leicester South +constituency/E14001328,Leicester West,constituency,country/england,,,E14001328,constituency/Leicester West +constituency/E14001329,Leigh and Atherton,constituency,country/england,,,E14001329,constituency/Leigh and Atherton +constituency/E14001330,Lewes,constituency,country/england,,,E14001330,constituency/Lewes +constituency/E14001331,Lewisham East,constituency,country/england,,,E14001331,constituency/Lewisham East +constituency/E14001332,Lewisham North,constituency,country/england,,,E14001332,constituency/Lewisham North +constituency/E14001333,Lewisham West and East Dulwich,constituency,country/england,,,E14001333,constituency/Lewisham West and East Dulwich +constituency/E14001334,Leyton and Wanstead,constituency,country/england,,,E14001334,constituency/Leyton and Wanstead +constituency/E14001335,Lichfield,constituency,country/england,,,E14001335,constituency/Lichfield +constituency/E14001336,Lincoln,constituency,country/england,,,E14001336,constituency/Lincoln +constituency/E14001337,Liverpool Garston,constituency,country/england,,,E14001337,constituency/Liverpool Garston +constituency/E14001338,Liverpool Riverside,constituency,country/england,,,E14001338,constituency/Liverpool Riverside +constituency/E14001339,Liverpool Walton,constituency,country/england,,,E14001339,constituency/Liverpool Walton +constituency/E14001340,Liverpool Wavertree,constituency,country/england,,,E14001340,constituency/Liverpool Wavertree +constituency/E14001341,Liverpool West Derby,constituency,country/england,,,E14001341,constituency/Liverpool West Derby +constituency/E14001342,Loughborough,constituency,country/england,,,E14001342,constituency/Loughborough +constituency/E14001343,Louth and Horncastle,constituency,country/england,,,E14001343,constituency/Louth and Horncastle +constituency/E14001344,Lowestoft,constituency,country/england,,,E14001344,constituency/Lowestoft +constituency/E14001345,Luton North,constituency,country/england,,,E14001345,constituency/Luton North +constituency/E14001346,Luton South and South Bedfordshire,constituency,country/england,,,E14001346,constituency/Luton South and South Bedfordshire +constituency/E14001347,Macclesfield,constituency,country/england,,,E14001347,constituency/Macclesfield +constituency/E14001348,Maidenhead,constituency,country/england,,,E14001348,constituency/Maidenhead +constituency/E14001349,Maidstone and Malling,constituency,country/england,,,E14001349,constituency/Maidstone and Malling +constituency/E14001350,Makerfield,constituency,country/england,,,E14001350,constituency/Makerfield +constituency/E14001351,Maldon,constituency,country/england,,,E14001351,constituency/Maldon +constituency/E14001352,Manchester Central,constituency,country/england,,,E14001352,constituency/Manchester Central +constituency/E14001353,Manchester Rusholme,constituency,country/england,,,E14001353,constituency/Manchester Rusholme +constituency/E14001354,Manchester Withington,constituency,country/england,,,E14001354,constituency/Manchester Withington +constituency/E14001355,Mansfield,constituency,country/england,,,E14001355,constituency/Mansfield +constituency/E14001356,Melksham and Devizes,constituency,country/england,,,E14001356,constituency/Melksham and Devizes +constituency/E14001357,Melton and Syston,constituency,country/england,,,E14001357,constituency/Melton and Syston +constituency/E14001358,Meriden and Solihull East,constituency,country/england,,,E14001358,constituency/Meriden and Solihull East +constituency/E14001359,Mid Bedfordshire,constituency,country/england,,,E14001359,constituency/Mid Bedfordshire +constituency/E14001360,Mid Buckinghamshire,constituency,country/england,,,E14001360,constituency/Mid Buckinghamshire +constituency/E14001361,Mid Cheshire,constituency,country/england,,,E14001361,constituency/Mid Cheshire +constituency/E14001362,Mid Derbyshire,constituency,country/england,,,E14001362,constituency/Mid Derbyshire +constituency/E14001363,Mid Dorset and North Poole,constituency,country/england,,,E14001363,constituency/Mid Dorset and North Poole +constituency/E14001364,Mid Leicestershire,constituency,country/england,,,E14001364,constituency/Mid Leicestershire +constituency/E14001365,Mid Norfolk,constituency,country/england,,,E14001365,constituency/Mid Norfolk +constituency/E14001366,Mid Sussex,constituency,country/england,,,E14001366,constituency/Mid Sussex +constituency/E14001367,Middlesbrough and Thornaby East,constituency,country/england,,,E14001367,constituency/Middlesbrough and Thornaby East +constituency/E14001368,Middlesbrough South and East Cleveland,constituency,country/england,,,E14001368,constituency/Middlesbrough South and East Cleveland +constituency/E14001369,Milton Keynes Central,constituency,country/england,,,E14001369,constituency/Milton Keynes Central +constituency/E14001370,Milton Keynes North,constituency,country/england,,,E14001370,constituency/Milton Keynes North +constituency/E14001371,Mitcham and Morden,constituency,country/england,,,E14001371,constituency/Mitcham and Morden +constituency/E14001372,Morecambe and Lunesdale,constituency,country/england,,,E14001372,constituency/Morecambe and Lunesdale +constituency/E14001373,New Forest East,constituency,country/england,,,E14001373,constituency/New Forest East +constituency/E14001374,New Forest West,constituency,country/england,,,E14001374,constituency/New Forest West +constituency/E14001375,Newark,constituency,country/england,,,E14001375,constituency/Newark +constituency/E14001376,Newbury,constituency,country/england,,,E14001376,constituency/Newbury +constituency/E14001377,Newcastle upon Tyne Central and West,constituency,country/england,,,E14001377,constituency/Newcastle upon Tyne Central and West +constituency/E14001378,Newcastle upon Tyne East and Wallsend,constituency,country/england,,,E14001378,constituency/Newcastle upon Tyne East and Wallsend +constituency/E14001379,Newcastle upon Tyne North,constituency,country/england,,,E14001379,constituency/Newcastle upon Tyne North +constituency/E14001380,Newcastle-under-Lyme,constituency,country/england,,,E14001380,constituency/Newcastle-under-Lyme +constituency/E14001381,Newton Abbot,constituency,country/england,,,E14001381,constituency/Newton Abbot +constituency/E14001382,Newton Aycliffe and Spennymoor,constituency,country/england,,,E14001382,constituency/Newton Aycliffe and Spennymoor +constituency/E14001383,Normanton and Hemsworth,constituency,country/england,,,E14001383,constituency/Normanton and Hemsworth +constituency/E14001384,North Bedfordshire,constituency,country/england,,,E14001384,constituency/North Bedfordshire +constituency/E14001385,North Cornwall,constituency,country/england,,,E14001385,constituency/North Cornwall +constituency/E14001386,North Cotswolds,constituency,country/england,,,E14001386,constituency/North Cotswolds +constituency/E14001387,North Devon,constituency,country/england,,,E14001387,constituency/North Devon +constituency/E14001388,North Dorset,constituency,country/england,,,E14001388,constituency/North Dorset +constituency/E14001389,North Durham,constituency,country/england,,,E14001389,constituency/North Durham +constituency/E14001390,North East Cambridgeshire,constituency,country/england,,,E14001390,constituency/North East Cambridgeshire +constituency/E14001391,North East Derbyshire,constituency,country/england,,,E14001391,constituency/North East Derbyshire +constituency/E14001392,North East Hampshire,constituency,country/england,,,E14001392,constituency/North East Hampshire +constituency/E14001393,North East Hertfordshire,constituency,country/england,,,E14001393,constituency/North East Hertfordshire +constituency/E14001394,North East Somerset and Hanham,constituency,country/england,,,E14001394,constituency/North East Somerset and Hanham +constituency/E14001395,North Herefordshire,constituency,country/england,,,E14001395,constituency/North Herefordshire +constituency/E14001396,North Norfolk,constituency,country/england,,,E14001396,constituency/North Norfolk +constituency/E14001397,North Northumberland,constituency,country/england,,,E14001397,constituency/North Northumberland +constituency/E14001398,North Shropshire,constituency,country/england,,,E14001398,constituency/North Shropshire +constituency/E14001399,North Somerset,constituency,country/england,,,E14001399,constituency/North Somerset +constituency/E14001400,North Warwickshire and Bedworth,constituency,country/england,,,E14001400,constituency/North Warwickshire and Bedworth +constituency/E14001401,North West Cambridgeshire,constituency,country/england,,,E14001401,constituency/North West Cambridgeshire +constituency/E14001402,North West Essex,constituency,country/england,,,E14001402,constituency/North West Essex +constituency/E14001403,North West Hampshire,constituency,country/england,,,E14001403,constituency/North West Hampshire +constituency/E14001404,North West Leicestershire,constituency,country/england,,,E14001404,constituency/North West Leicestershire +constituency/E14001405,North West Norfolk,constituency,country/england,,,E14001405,constituency/North West Norfolk +constituency/E14001406,Northampton North,constituency,country/england,,,E14001406,constituency/Northampton North +constituency/E14001407,Northampton South,constituency,country/england,,,E14001407,constituency/Northampton South +constituency/E14001408,Norwich North,constituency,country/england,,,E14001408,constituency/Norwich North +constituency/E14001409,Norwich South,constituency,country/england,,,E14001409,constituency/Norwich South +constituency/E14001410,Nottingham East,constituency,country/england,,,E14001410,constituency/Nottingham East +constituency/E14001411,Nottingham North and Kimberley,constituency,country/england,,,E14001411,constituency/Nottingham North and Kimberley +constituency/E14001412,Nottingham South,constituency,country/england,,,E14001412,constituency/Nottingham South +constituency/E14001413,Nuneaton,constituency,country/england,,,E14001413,constituency/Nuneaton +constituency/E14001414,Old Bexley and Sidcup,constituency,country/england,,,E14001414,constituency/Old Bexley and Sidcup +constituency/E14001415,Oldham East and Saddleworth,constituency,country/england,,,E14001415,constituency/Oldham East and Saddleworth +constituency/E14001416,"Oldham West, Chadderton and Royton",constituency,country/england,,,E14001416,"constituency/Oldham West, Chadderton and Royton" +constituency/E14001417,Orpington,constituency,country/england,,,E14001417,constituency/Orpington +constituency/E14001418,Ossett and Denby Dale,constituency,country/england,,,E14001418,constituency/Ossett and Denby Dale +constituency/E14001419,Oxford East,constituency,country/england,,,E14001419,constituency/Oxford East +constituency/E14001420,Oxford West and Abingdon,constituency,country/england,,,E14001420,constituency/Oxford West and Abingdon +constituency/E14001421,Peckham,constituency,country/england,,,E14001421,constituency/Peckham +constituency/E14001422,Pendle and Clitheroe,constituency,country/england,,,E14001422,constituency/Pendle and Clitheroe +constituency/E14001423,Penistone and Stocksbridge,constituency,country/england,,,E14001423,constituency/Penistone and Stocksbridge +constituency/E14001424,Penrith and Solway,constituency,country/england,,,E14001424,constituency/Penrith and Solway +constituency/E14001425,Peterborough,constituency,country/england,,,E14001425,constituency/Peterborough +constituency/E14001426,Plymouth Moor View,constituency,country/england,,,E14001426,constituency/Plymouth Moor View +constituency/E14001427,Plymouth Sutton and Devonport,constituency,country/england,,,E14001427,constituency/Plymouth Sutton and Devonport +constituency/E14001428,"Pontefract, Castleford and Knottingley",constituency,country/england,,,E14001428,"constituency/Pontefract, Castleford and Knottingley" +constituency/E14001429,Poole,constituency,country/england,,,E14001429,constituency/Poole +constituency/E14001430,Poplar and Limehouse,constituency,country/england,,,E14001430,constituency/Poplar and Limehouse +constituency/E14001431,Portsmouth North,constituency,country/england,,,E14001431,constituency/Portsmouth North +constituency/E14001432,Portsmouth South,constituency,country/england,,,E14001432,constituency/Portsmouth South +constituency/E14001433,Preston,constituency,country/england,,,E14001433,constituency/Preston +constituency/E14001434,Putney,constituency,country/england,,,E14001434,constituency/Putney +constituency/E14001435,Queen's Park and Maida Vale,constituency,country/england,,,E14001435,constituency/Queen's Park and Maida Vale +constituency/E14001436,Rawmarsh and Conisbrough,constituency,country/england,,,E14001436,constituency/Rawmarsh and Conisbrough +constituency/E14001437,Rayleigh and Wickford,constituency,country/england,,,E14001437,constituency/Rayleigh and Wickford +constituency/E14001438,Reading Central,constituency,country/england,,,E14001438,constituency/Reading Central +constituency/E14001439,Reading West and Mid Berkshire,constituency,country/england,,,E14001439,constituency/Reading West and Mid Berkshire +constituency/E14001440,Redcar,constituency,country/england,,,E14001440,constituency/Redcar +constituency/E14001441,Redditch,constituency,country/england,,,E14001441,constituency/Redditch +constituency/E14001442,Reigate,constituency,country/england,,,E14001442,constituency/Reigate +constituency/E14001443,Ribble Valley,constituency,country/england,,,E14001443,constituency/Ribble Valley +constituency/E14001444,Richmond and Northallerton,constituency,country/england,,,E14001444,constituency/Richmond and Northallerton +constituency/E14001445,Richmond Park,constituency,country/england,,,E14001445,constituency/Richmond Park +constituency/E14001446,Rochdale,constituency,country/england,,,E14001446,constituency/Rochdale +constituency/E14001447,Rochester and Strood,constituency,country/england,,,E14001447,constituency/Rochester and Strood +constituency/E14001448,Romford,constituency,country/england,,,E14001448,constituency/Romford +constituency/E14001449,Romsey and Southampton North,constituency,country/england,,,E14001449,constituency/Romsey and Southampton North +constituency/E14001450,Rossendale and Darwen,constituency,country/england,,,E14001450,constituency/Rossendale and Darwen +constituency/E14001451,Rother Valley,constituency,country/england,,,E14001451,constituency/Rother Valley +constituency/E14001452,Rotherham,constituency,country/england,,,E14001452,constituency/Rotherham +constituency/E14001453,Rugby,constituency,country/england,,,E14001453,constituency/Rugby +constituency/E14001454,"Ruislip, Northwood and Pinner",constituency,country/england,,,E14001454,"constituency/Ruislip, Northwood and Pinner" +constituency/E14001455,Runcorn and Helsby,constituency,country/england,,,E14001455,constituency/Runcorn and Helsby +constituency/E14001456,Runnymede and Weybridge,constituency,country/england,,,E14001456,constituency/Runnymede and Weybridge +constituency/E14001457,Rushcliffe,constituency,country/england,,,E14001457,constituency/Rushcliffe +constituency/E14001458,Rutland and Stamford,constituency,country/england,,,E14001458,constituency/Rutland and Stamford +constituency/E14001459,Salford,constituency,country/england,,,E14001459,constituency/Salford +constituency/E14001460,Salisbury,constituency,country/england,,,E14001460,constituency/Salisbury +constituency/E14001461,Scarborough and Whitby,constituency,country/england,,,E14001461,constituency/Scarborough and Whitby +constituency/E14001462,Scunthorpe,constituency,country/england,,,E14001462,constituency/Scunthorpe +constituency/E14001463,Sefton Central,constituency,country/england,,,E14001463,constituency/Sefton Central +constituency/E14001464,Selby,constituency,country/england,,,E14001464,constituency/Selby +constituency/E14001465,Sevenoaks,constituency,country/england,,,E14001465,constituency/Sevenoaks +constituency/E14001466,Sheffield Brightside and Hillsborough,constituency,country/england,,,E14001466,constituency/Sheffield Brightside and Hillsborough +constituency/E14001467,Sheffield Central,constituency,country/england,,,E14001467,constituency/Sheffield Central +constituency/E14001468,Sheffield Hallam,constituency,country/england,,,E14001468,constituency/Sheffield Hallam +constituency/E14001469,Sheffield Heeley,constituency,country/england,,,E14001469,constituency/Sheffield Heeley +constituency/E14001470,Sheffield South East,constituency,country/england,,,E14001470,constituency/Sheffield South East +constituency/E14001471,Sherwood Forest,constituency,country/england,,,E14001471,constituency/Sherwood Forest +constituency/E14001472,Shipley,constituency,country/england,,,E14001472,constituency/Shipley +constituency/E14001473,Shrewsbury,constituency,country/england,,,E14001473,constituency/Shrewsbury +constituency/E14001474,Sittingbourne and Sheppey,constituency,country/england,,,E14001474,constituency/Sittingbourne and Sheppey +constituency/E14001475,Skipton and Ripon,constituency,country/england,,,E14001475,constituency/Skipton and Ripon +constituency/E14001476,Sleaford and North Hykeham,constituency,country/england,,,E14001476,constituency/Sleaford and North Hykeham +constituency/E14001477,Slough,constituency,country/england,,,E14001477,constituency/Slough +constituency/E14001478,Smethwick,constituency,country/england,,,E14001478,constituency/Smethwick +constituency/E14001479,Solihull West and Shirley,constituency,country/england,,,E14001479,constituency/Solihull West and Shirley +constituency/E14001480,South Basildon and East Thurrock,constituency,country/england,,,E14001480,constituency/South Basildon and East Thurrock +constituency/E14001481,South Cambridgeshire,constituency,country/england,,,E14001481,constituency/South Cambridgeshire +constituency/E14001482,South Cotswolds,constituency,country/england,,,E14001482,constituency/South Cotswolds +constituency/E14001483,South Derbyshire,constituency,country/england,,,E14001483,constituency/South Derbyshire +constituency/E14001484,South Devon,constituency,country/england,,,E14001484,constituency/South Devon +constituency/E14001485,South Dorset,constituency,country/england,,,E14001485,constituency/South Dorset +constituency/E14001486,South East Cornwall,constituency,country/england,,,E14001486,constituency/South East Cornwall +constituency/E14001487,South Holland and The Deepings,constituency,country/england,,,E14001487,constituency/South Holland and The Deepings +constituency/E14001488,South Leicestershire,constituency,country/england,,,E14001488,constituency/South Leicestershire +constituency/E14001489,South Norfolk,constituency,country/england,,,E14001489,constituency/South Norfolk +constituency/E14001490,South Northamptonshire,constituency,country/england,,,E14001490,constituency/South Northamptonshire +constituency/E14001491,South Ribble,constituency,country/england,,,E14001491,constituency/South Ribble +constituency/E14001492,South Shields,constituency,country/england,,,E14001492,constituency/South Shields +constituency/E14001493,South Shropshire,constituency,country/england,,,E14001493,constituency/South Shropshire +constituency/E14001494,South Suffolk,constituency,country/england,,,E14001494,constituency/South Suffolk +constituency/E14001495,South West Devon,constituency,country/england,,,E14001495,constituency/South West Devon +constituency/E14001496,South West Hertfordshire,constituency,country/england,,,E14001496,constituency/South West Hertfordshire +constituency/E14001497,South West Norfolk,constituency,country/england,,,E14001497,constituency/South West Norfolk +constituency/E14001498,South West Wiltshire,constituency,country/england,,,E14001498,constituency/South West Wiltshire +constituency/E14001499,Southampton Itchen,constituency,country/england,,,E14001499,constituency/Southampton Itchen +constituency/E14001500,Southampton Test,constituency,country/england,,,E14001500,constituency/Southampton Test +constituency/E14001501,Southend East and Rochford,constituency,country/england,,,E14001501,constituency/Southend East and Rochford +constituency/E14001502,Southend West and Leigh,constituency,country/england,,,E14001502,constituency/Southend West and Leigh +constituency/E14001503,Southgate and Wood Green,constituency,country/england,,,E14001503,constituency/Southgate and Wood Green +constituency/E14001504,Southport,constituency,country/england,,,E14001504,constituency/Southport +constituency/E14001505,Spelthorne,constituency,country/england,,,E14001505,constituency/Spelthorne +constituency/E14001506,Spen Valley,constituency,country/england,,,E14001506,constituency/Spen Valley +constituency/E14001507,St Albans,constituency,country/england,,,E14001507,constituency/St Albans +constituency/E14001508,St Austell and Newquay,constituency,country/england,,,E14001508,constituency/St Austell and Newquay +constituency/E14001509,St Helens North,constituency,country/england,,,E14001509,constituency/St Helens North +constituency/E14001510,St Helens South and Whiston,constituency,country/england,,,E14001510,constituency/St Helens South and Whiston +constituency/E14001511,St Ives,constituency,country/england,,,E14001511,constituency/St Ives +constituency/E14001512,St Neots and Mid Cambridgeshire,constituency,country/england,,,E14001512,constituency/St Neots and Mid Cambridgeshire +constituency/E14001513,Stafford,constituency,country/england,,,E14001513,constituency/Stafford +constituency/E14001514,Staffordshire Moorlands,constituency,country/england,,,E14001514,constituency/Staffordshire Moorlands +constituency/E14001515,Stalybridge and Hyde,constituency,country/england,,,E14001515,constituency/Stalybridge and Hyde +constituency/E14001516,Stevenage,constituency,country/england,,,E14001516,constituency/Stevenage +constituency/E14001517,Stockport,constituency,country/england,,,E14001517,constituency/Stockport +constituency/E14001518,Stockton North,constituency,country/england,,,E14001518,constituency/Stockton North +constituency/E14001519,Stockton West,constituency,country/england,,,E14001519,constituency/Stockton West +constituency/E14001520,Stoke-on-Trent Central,constituency,country/england,,,E14001520,constituency/Stoke-on-Trent Central +constituency/E14001521,Stoke-on-Trent North,constituency,country/england,,,E14001521,constituency/Stoke-on-Trent North +constituency/E14001522,Stoke-on-Trent South,constituency,country/england,,,E14001522,constituency/Stoke-on-Trent South +constituency/E14001523,"Stone, Great Wyrley and Penkridge",constituency,country/england,,,E14001523,"constituency/Stone, Great Wyrley and Penkridge" +constituency/E14001524,Stourbridge,constituency,country/england,,,E14001524,constituency/Stourbridge +constituency/E14001525,Stratford and Bow,constituency,country/england,,,E14001525,constituency/Stratford and Bow +constituency/E14001526,Stratford-on-Avon,constituency,country/england,,,E14001526,constituency/Stratford-on-Avon +constituency/E14001527,Streatham and Croydon North,constituency,country/england,,,E14001527,constituency/Streatham and Croydon North +constituency/E14001528,Stretford and Urmston,constituency,country/england,,,E14001528,constituency/Stretford and Urmston +constituency/E14001529,Stroud,constituency,country/england,,,E14001529,constituency/Stroud +constituency/E14001530,Suffolk Coastal,constituency,country/england,,,E14001530,constituency/Suffolk Coastal +constituency/E14001531,Sunderland Central,constituency,country/england,,,E14001531,constituency/Sunderland Central +constituency/E14001532,Surrey Heath,constituency,country/england,,,E14001532,constituency/Surrey Heath +constituency/E14001533,Sussex Weald,constituency,country/england,,,E14001533,constituency/Sussex Weald +constituency/E14001534,Sutton and Cheam,constituency,country/england,,,E14001534,constituency/Sutton and Cheam +constituency/E14001535,Sutton Coldfield,constituency,country/england,,,E14001535,constituency/Sutton Coldfield +constituency/E14001536,Swindon North,constituency,country/england,,,E14001536,constituency/Swindon North +constituency/E14001537,Swindon South,constituency,country/england,,,E14001537,constituency/Swindon South +constituency/E14001538,Tamworth,constituency,country/england,,,E14001538,constituency/Tamworth +constituency/E14001539,Tatton,constituency,country/england,,,E14001539,constituency/Tatton +constituency/E14001540,Taunton and Wellington,constituency,country/england,,,E14001540,constituency/Taunton and Wellington +constituency/E14001541,Telford,constituency,country/england,,,E14001541,constituency/Telford +constituency/E14001542,Tewkesbury,constituency,country/england,,,E14001542,constituency/Tewkesbury +constituency/E14001543,The Wrekin,constituency,country/england,,,E14001543,constituency/The Wrekin +constituency/E14001544,Thirsk and Malton,constituency,country/england,,,E14001544,constituency/Thirsk and Malton +constituency/E14001545,Thornbury and Yate,constituency,country/england,,,E14001545,constituency/Thornbury and Yate +constituency/E14001546,Thurrock,constituency,country/england,,,E14001546,constituency/Thurrock +constituency/E14001547,Tipton and Wednesbury,constituency,country/england,,,E14001547,constituency/Tipton and Wednesbury +constituency/E14001548,Tiverton and Minehead,constituency,country/england,,,E14001548,constituency/Tiverton and Minehead +constituency/E14001549,Tonbridge,constituency,country/england,,,E14001549,constituency/Tonbridge +constituency/E14001550,Tooting,constituency,country/england,,,E14001550,constituency/Tooting +constituency/E14001551,Torbay,constituency,country/england,,,E14001551,constituency/Torbay +constituency/E14001552,Torridge and Tavistock,constituency,country/england,,,E14001552,constituency/Torridge and Tavistock +constituency/E14001553,Tottenham,constituency,country/england,,,E14001553,constituency/Tottenham +constituency/E14001554,Truro and Falmouth,constituency,country/england,,,E14001554,constituency/Truro and Falmouth +constituency/E14001555,Tunbridge Wells,constituency,country/england,,,E14001555,constituency/Tunbridge Wells +constituency/E14001556,Twickenham,constituency,country/england,,,E14001556,constituency/Twickenham +constituency/E14001557,Tynemouth,constituency,country/england,,,E14001557,constituency/Tynemouth +constituency/E14001558,Uxbridge and South Ruislip,constituency,country/england,,,E14001558,constituency/Uxbridge and South Ruislip +constituency/E14001559,Vauxhall and Camberwell Green,constituency,country/england,,,E14001559,constituency/Vauxhall and Camberwell Green +constituency/E14001560,Wakefield and Rothwell,constituency,country/england,,,E14001560,constituency/Wakefield and Rothwell +constituency/E14001561,Wallasey,constituency,country/england,,,E14001561,constituency/Wallasey +constituency/E14001562,Walsall and Bloxwich,constituency,country/england,,,E14001562,constituency/Walsall and Bloxwich +constituency/E14001563,Walthamstow,constituency,country/england,,,E14001563,constituency/Walthamstow +constituency/E14001564,Warrington North,constituency,country/england,,,E14001564,constituency/Warrington North +constituency/E14001565,Warrington South,constituency,country/england,,,E14001565,constituency/Warrington South +constituency/E14001566,Warwick and Leamington,constituency,country/england,,,E14001566,constituency/Warwick and Leamington +constituency/E14001567,Washington and Gateshead South,constituency,country/england,,,E14001567,constituency/Washington and Gateshead South +constituency/E14001568,Watford,constituency,country/england,,,E14001568,constituency/Watford +constituency/E14001569,Waveney Valley,constituency,country/england,,,E14001569,constituency/Waveney Valley +constituency/E14001570,Weald of Kent,constituency,country/england,,,E14001570,constituency/Weald of Kent +constituency/E14001571,Wellingborough and Rushden,constituency,country/england,,,E14001571,constituency/Wellingborough and Rushden +constituency/E14001572,Wells and Mendip Hills,constituency,country/england,,,E14001572,constituency/Wells and Mendip Hills +constituency/E14001573,Welwyn Hatfield,constituency,country/england,,,E14001573,constituency/Welwyn Hatfield +constituency/E14001574,West Bromwich,constituency,country/england,,,E14001574,constituency/West Bromwich +constituency/E14001575,West Dorset,constituency,country/england,,,E14001575,constituency/West Dorset +constituency/E14001576,West Ham and Beckton,constituency,country/england,,,E14001576,constituency/West Ham and Beckton +constituency/E14001577,West Lancashire,constituency,country/england,,,E14001577,constituency/West Lancashire +constituency/E14001578,West Suffolk,constituency,country/england,,,E14001578,constituency/West Suffolk +constituency/E14001579,West Worcestershire,constituency,country/england,,,E14001579,constituency/West Worcestershire +constituency/E14001580,Westmorland and Lonsdale,constituency,country/england,,,E14001580,constituency/Westmorland and Lonsdale +constituency/E14001581,Weston-super-Mare,constituency,country/england,,,E14001581,constituency/Weston-super-Mare +constituency/E14001582,Wetherby and Easingwold,constituency,country/england,,,E14001582,constituency/Wetherby and Easingwold +constituency/E14001583,Whitehaven and Workington,constituency,country/england,,,E14001583,constituency/Whitehaven and Workington +constituency/E14001584,Widnes and Halewood,constituency,country/england,,,E14001584,constituency/Widnes and Halewood +constituency/E14001585,Wigan,constituency,country/england,,,E14001585,constituency/Wigan +constituency/E14001586,Wimbledon,constituency,country/england,,,E14001586,constituency/Wimbledon +constituency/E14001587,Winchester,constituency,country/england,,,E14001587,constituency/Winchester +constituency/E14001588,Windsor,constituency,country/england,,,E14001588,constituency/Windsor +constituency/E14001589,Wirral West,constituency,country/england,,,E14001589,constituency/Wirral West +constituency/E14001590,Witham,constituency,country/england,,,E14001590,constituency/Witham +constituency/E14001591,Witney,constituency,country/england,,,E14001591,constituency/Witney +constituency/E14001592,Woking,constituency,country/england,,,E14001592,constituency/Woking +constituency/E14001593,Wokingham,constituency,country/england,,,E14001593,constituency/Wokingham +constituency/E14001594,Wolverhampton North East,constituency,country/england,,,E14001594,constituency/Wolverhampton North East +constituency/E14001595,Wolverhampton South East,constituency,country/england,,,E14001595,constituency/Wolverhampton South East +constituency/E14001596,Wolverhampton West,constituency,country/england,,,E14001596,constituency/Wolverhampton West +constituency/E14001597,Worcester,constituency,country/england,,,E14001597,constituency/Worcester +constituency/E14001598,Worsley and Eccles,constituency,country/england,,,E14001598,constituency/Worsley and Eccles +constituency/E14001599,Worthing West,constituency,country/england,,,E14001599,constituency/Worthing West +constituency/E14001600,Wycombe,constituency,country/england,,,E14001600,constituency/Wycombe +constituency/E14001601,Wyre Forest,constituency,country/england,,,E14001601,constituency/Wyre Forest +constituency/E14001602,Wythenshawe and Sale East,constituency,country/england,,,E14001602,constituency/Wythenshawe and Sale East +constituency/E14001603,Yeovil,constituency,country/england,,,E14001603,constituency/Yeovil +constituency/E14001604,York Central,constituency,country/england,,,E14001604,constituency/York Central +constituency/E14001605,York Outer,constituency,country/england,,,E14001605,constituency/York Outer +constituency/N05000001,Belfast East,constituency,country/northern_ireland,,,N05000001,constituency/Belfast East +constituency/N05000002,Belfast North,constituency,country/northern_ireland,,,N05000002,constituency/Belfast North +constituency/N05000003,Belfast South and Mid Down,constituency,country/northern_ireland,,,N05000003,constituency/Belfast South and Mid Down +constituency/N05000004,Belfast West,constituency,country/northern_ireland,,,N05000004,constituency/Belfast West +constituency/N05000005,East Antrim,constituency,country/northern_ireland,,,N05000005,constituency/East Antrim +constituency/N05000006,East Londonderry,constituency,country/northern_ireland,,,N05000006,constituency/East Londonderry +constituency/N05000007,Fermanagh and South Tyrone,constituency,country/northern_ireland,,,N05000007,constituency/Fermanagh and South Tyrone +constituency/N05000008,Foyle,constituency,country/northern_ireland,,,N05000008,constituency/Foyle +constituency/N05000009,Lagan Valley,constituency,country/northern_ireland,,,N05000009,constituency/Lagan Valley +constituency/N05000010,Mid Ulster,constituency,country/northern_ireland,,,N05000010,constituency/Mid Ulster +constituency/N05000011,Newry and Armagh,constituency,country/northern_ireland,,,N05000011,constituency/Newry and Armagh +constituency/N05000012,North Antrim,constituency,country/northern_ireland,,,N05000012,constituency/North Antrim +constituency/N05000013,North Down,constituency,country/northern_ireland,,,N05000013,constituency/North Down +constituency/N05000014,South Antrim,constituency,country/northern_ireland,,,N05000014,constituency/South Antrim +constituency/N05000015,South Down,constituency,country/northern_ireland,,,N05000015,constituency/South Down +constituency/N05000016,Strangford,constituency,country/northern_ireland,,,N05000016,constituency/Strangford +constituency/N05000017,Upper Bann,constituency,country/northern_ireland,,,N05000017,constituency/Upper Bann +constituency/N05000018,West Tyrone,constituency,country/northern_ireland,,,N05000018,constituency/West Tyrone +constituency/S14000021,East Renfrewshire,constituency,country/scotland,,,S14000021,constituency/East Renfrewshire +constituency/S14000027,Na h-Eileanan an Iar,constituency,country/scotland,,,S14000027,constituency/Na h-Eileanan an Iar +constituency/S14000045,Midlothian,constituency,country/scotland,,,S14000045,constituency/Midlothian +constituency/S14000048,North Ayrshire and Arran,constituency,country/scotland,,,S14000048,constituency/North Ayrshire and Arran +constituency/S14000051,Orkney and Shetland,constituency,country/scotland,,,S14000051,constituency/Orkney and Shetland +constituency/S14000060,Aberdeen North,constituency,country/scotland,,,S14000060,constituency/Aberdeen North +constituency/S14000061,Aberdeen South,constituency,country/scotland,,,S14000061,constituency/Aberdeen South +constituency/S14000062,Aberdeenshire North and Moray East,constituency,country/scotland,,,S14000062,constituency/Aberdeenshire North and Moray East +constituency/S14000063,Airdrie and Shotts,constituency,country/scotland,,,S14000063,constituency/Airdrie and Shotts +constituency/S14000064,Alloa and Grangemouth,constituency,country/scotland,,,S14000064,constituency/Alloa and Grangemouth +constituency/S14000065,Angus and Perthshire Glens,constituency,country/scotland,,,S14000065,constituency/Angus and Perthshire Glens +constituency/S14000066,Arbroath and Broughty Ferry,constituency,country/scotland,,,S14000066,constituency/Arbroath and Broughty Ferry +constituency/S14000067,"Argyll, Bute and South Lochaber",constituency,country/scotland,,,S14000067,"constituency/Argyll, Bute and South Lochaber" +constituency/S14000068,Bathgate and Linlithgow,constituency,country/scotland,,,S14000068,constituency/Bathgate and Linlithgow +constituency/S14000069,"Caithness, Sutherland and Easter Ross",constituency,country/scotland,,,S14000069,"constituency/Caithness, Sutherland and Easter Ross" +constituency/S14000070,Coatbridge and Bellshill,constituency,country/scotland,,,S14000070,constituency/Coatbridge and Bellshill +constituency/S14000071,Cowdenbeath and Kirkcaldy,constituency,country/scotland,,,S14000071,constituency/Cowdenbeath and Kirkcaldy +constituency/S14000072,Cumbernauld and Kirkintilloch,constituency,country/scotland,,,S14000072,constituency/Cumbernauld and Kirkintilloch +constituency/S14000073,Dumfries and Galloway,constituency,country/scotland,,,S14000073,constituency/Dumfries and Galloway +constituency/S14000074,"Dumfriesshire, Clydesdale and Tweeddale",constituency,country/scotland,,,S14000074,"constituency/Dumfriesshire, Clydesdale and Tweeddale" +constituency/S14000075,Dundee Central,constituency,country/scotland,,,S14000075,constituency/Dundee Central +constituency/S14000076,Dunfermline and Dollar,constituency,country/scotland,,,S14000076,constituency/Dunfermline and Dollar +constituency/S14000077,East Kilbride and Strathaven,constituency,country/scotland,,,S14000077,constituency/East Kilbride and Strathaven +constituency/S14000078,Edinburgh East and Musselburgh,constituency,country/scotland,,,S14000078,constituency/Edinburgh East and Musselburgh +constituency/S14000079,Edinburgh North and Leith,constituency,country/scotland,,,S14000079,constituency/Edinburgh North and Leith +constituency/S14000080,Edinburgh South,constituency,country/scotland,,,S14000080,constituency/Edinburgh South +constituency/S14000081,Edinburgh South West,constituency,country/scotland,,,S14000081,constituency/Edinburgh South West +constituency/S14000082,Edinburgh West,constituency,country/scotland,,,S14000082,constituency/Edinburgh West +constituency/S14000083,Falkirk,constituency,country/scotland,,,S14000083,constituency/Falkirk +constituency/S14000084,Glasgow East,constituency,country/scotland,,,S14000084,constituency/Glasgow East +constituency/S14000085,Glasgow North,constituency,country/scotland,,,S14000085,constituency/Glasgow North +constituency/S14000086,Glasgow North East,constituency,country/scotland,,,S14000086,constituency/Glasgow North East +constituency/S14000087,Glasgow South,constituency,country/scotland,,,S14000087,constituency/Glasgow South +constituency/S14000088,Glasgow South West,constituency,country/scotland,,,S14000088,constituency/Glasgow South West +constituency/S14000089,Glasgow West,constituency,country/scotland,,,S14000089,constituency/Glasgow West +constituency/S14000090,Glenrothes and Mid Fife,constituency,country/scotland,,,S14000090,constituency/Glenrothes and Mid Fife +constituency/S14000091,Gordon and Buchan,constituency,country/scotland,,,S14000091,constituency/Gordon and Buchan +constituency/S14000092,Hamilton and Clyde Valley,constituency,country/scotland,,,S14000092,constituency/Hamilton and Clyde Valley +constituency/S14000093,Inverclyde and Renfrewshire West,constituency,country/scotland,,,S14000093,constituency/Inverclyde and Renfrewshire West +constituency/S14000094,"Inverness, Skye and West Ross-shire",constituency,country/scotland,,,S14000094,"constituency/Inverness, Skye and West Ross-shire" +constituency/S14000095,Livingston,constituency,country/scotland,,,S14000095,constituency/Livingston +constituency/S14000096,Lothian East,constituency,country/scotland,,,S14000096,constituency/Lothian East +constituency/S14000097,Mid Dunbartonshire,constituency,country/scotland,,,S14000097,constituency/Mid Dunbartonshire +constituency/S14000098,"Moray West, Nairn and Strathspey",constituency,country/scotland,,,S14000098,"constituency/Moray West, Nairn and Strathspey" +constituency/S14000099,"Motherwell, Wishaw and Carluke",constituency,country/scotland,,,S14000099,"constituency/Motherwell, Wishaw and Carluke" +constituency/S14000100,North East Fife,constituency,country/scotland,,,S14000100,constituency/North East Fife +constituency/S14000101,Paisley and Renfrewshire North,constituency,country/scotland,,,S14000101,constituency/Paisley and Renfrewshire North +constituency/S14000102,Paisley and Renfrewshire South,constituency,country/scotland,,,S14000102,constituency/Paisley and Renfrewshire South +constituency/S14000103,Perth and Kinross-shire,constituency,country/scotland,,,S14000103,constituency/Perth and Kinross-shire +constituency/S14000104,Rutherglen,constituency,country/scotland,,,S14000104,constituency/Rutherglen +constituency/S14000105,Stirling and Strathallan,constituency,country/scotland,,,S14000105,constituency/Stirling and Strathallan +constituency/S14000106,West Dunbartonshire,constituency,country/scotland,,,S14000106,constituency/West Dunbartonshire +constituency/S14000107,"Ayr, Carrick and Cumnock",constituency,country/scotland,,,S14000107,"constituency/Ayr, Carrick and Cumnock" +constituency/S14000108,"Berwickshire, Roxburgh and Selkirk",constituency,country/scotland,,,S14000108,"constituency/Berwickshire, Roxburgh and Selkirk" +constituency/S14000109,Central Ayrshire,constituency,country/scotland,,,S14000109,constituency/Central Ayrshire +constituency/S14000110,Kilmarnock and Loudoun,constituency,country/scotland,,,S14000110,constituency/Kilmarnock and Loudoun +constituency/S14000111,West Aberdeenshire and Kincardine,constituency,country/scotland,,,S14000111,constituency/West Aberdeenshire and Kincardine +constituency/W07000081,Aberafan Maesteg,constituency,country/wales,,,W07000081,constituency/Aberafan Maesteg +constituency/W07000082,Alyn and Deeside,constituency,country/wales,,,W07000082,constituency/Alyn and Deeside +constituency/W07000083,Bangor Aberconwy,constituency,country/wales,,,W07000083,constituency/Bangor Aberconwy +constituency/W07000084,Blaenau Gwent and Rhymney,constituency,country/wales,,,W07000084,constituency/Blaenau Gwent and Rhymney +constituency/W07000085,"Brecon, Radnor and Cwm Tawe",constituency,country/wales,,,W07000085,"constituency/Brecon, Radnor and Cwm Tawe" +constituency/W07000086,Bridgend,constituency,country/wales,,,W07000086,constituency/Bridgend +constituency/W07000087,Caerfyrddin,constituency,country/wales,,,W07000087,constituency/Caerfyrddin +constituency/W07000088,Caerphilly,constituency,country/wales,,,W07000088,constituency/Caerphilly +constituency/W07000089,Cardiff East,constituency,country/wales,,,W07000089,constituency/Cardiff East +constituency/W07000090,Cardiff North,constituency,country/wales,,,W07000090,constituency/Cardiff North +constituency/W07000091,Cardiff South and Penarth,constituency,country/wales,,,W07000091,constituency/Cardiff South and Penarth +constituency/W07000092,Cardiff West,constituency,country/wales,,,W07000092,constituency/Cardiff West +constituency/W07000093,Ceredigion Preseli,constituency,country/wales,,,W07000093,constituency/Ceredigion Preseli +constituency/W07000094,Clwyd East,constituency,country/wales,,,W07000094,constituency/Clwyd East +constituency/W07000095,Clwyd North,constituency,country/wales,,,W07000095,constituency/Clwyd North +constituency/W07000096,Dwyfor Meirionnydd,constituency,country/wales,,,W07000096,constituency/Dwyfor Meirionnydd +constituency/W07000097,Gower,constituency,country/wales,,,W07000097,constituency/Gower +constituency/W07000098,Llanelli,constituency,country/wales,,,W07000098,constituency/Llanelli +constituency/W07000099,Merthyr Tydfil and Aberdare,constituency,country/wales,,,W07000099,constituency/Merthyr Tydfil and Aberdare +constituency/W07000100,Mid and South Pembrokeshire,constituency,country/wales,,,W07000100,constituency/Mid and South Pembrokeshire +constituency/W07000101,Monmouthshire,constituency,country/wales,,,W07000101,constituency/Monmouthshire +constituency/W07000102,Montgomeryshire and Glyndwr,constituency,country/wales,,,W07000102,constituency/Montgomeryshire and Glyndwr +constituency/W07000103,Neath and Swansea East,constituency,country/wales,,,W07000103,constituency/Neath and Swansea East +constituency/W07000104,Newport East,constituency,country/wales,,,W07000104,constituency/Newport East +constituency/W07000105,Newport West and Islwyn,constituency,country/wales,,,W07000105,constituency/Newport West and Islwyn +constituency/W07000106,Pontypridd,constituency,country/wales,,,W07000106,constituency/Pontypridd +constituency/W07000107,Rhondda and Ogmore,constituency,country/wales,,,W07000107,constituency/Rhondda and Ogmore +constituency/W07000108,Swansea West,constituency,country/wales,,,W07000108,constituency/Swansea West +constituency/W07000109,Torfaen,constituency,country/wales,,,W07000109,constituency/Torfaen +constituency/W07000110,Vale of Glamorgan,constituency,country/wales,,,W07000110,constituency/Vale of Glamorgan +constituency/W07000111,Wrexham,constituency,country/wales,,,W07000111,constituency/Wrexham +constituency/W07000112,Ynys Môn,constituency,country/wales,,,W07000112,constituency/Ynys Môn +local_authority/E06000001,Hartlepool,local_authority,country/england,,,E06000001,local_authority/Hartlepool +local_authority/E06000002,Middlesbrough,local_authority,country/england,,,E06000002,local_authority/Middlesbrough +local_authority/E06000003,Redcar and Cleveland,local_authority,country/england,,,E06000003,local_authority/Redcar and Cleveland +local_authority/E06000004,Stockton-on-Tees,local_authority,country/england,,,E06000004,local_authority/Stockton-on-Tees +local_authority/E06000005,Darlington,local_authority,country/england,,,E06000005,local_authority/Darlington +local_authority/E06000006,Halton,local_authority,country/england,,,E06000006,local_authority/Halton +local_authority/E06000007,Warrington,local_authority,country/england,,,E06000007,local_authority/Warrington +local_authority/E06000008,Blackburn with Darwen,local_authority,country/england,,,E06000008,local_authority/Blackburn with Darwen +local_authority/E06000009,Blackpool,local_authority,country/england,,,E06000009,local_authority/Blackpool +local_authority/E06000010,"Kingston upon Hull, City of",local_authority,country/england,,,E06000010,"local_authority/Kingston upon Hull, City of" +local_authority/E06000011,East Riding of Yorkshire,local_authority,country/england,,,E06000011,local_authority/East Riding of Yorkshire +local_authority/E06000012,North East Lincolnshire,local_authority,country/england,,,E06000012,local_authority/North East Lincolnshire +local_authority/E06000013,North Lincolnshire,local_authority,country/england,,,E06000013,local_authority/North Lincolnshire +local_authority/E06000014,York,local_authority,country/england,,,E06000014,local_authority/York +local_authority/E06000015,Derby,local_authority,country/england,,,E06000015,local_authority/Derby +local_authority/E06000016,Leicester,local_authority,country/england,,,E06000016,local_authority/Leicester +local_authority/E06000017,Rutland,local_authority,country/england,,,E06000017,local_authority/Rutland +local_authority/E06000018,Nottingham,local_authority,country/england,,,E06000018,local_authority/Nottingham +local_authority/E06000019,"Herefordshire, County of",local_authority,country/england,,,E06000019,"local_authority/Herefordshire, County of" +local_authority/E06000020,Telford and Wrekin,local_authority,country/england,,,E06000020,local_authority/Telford and Wrekin +local_authority/E06000021,Stoke-on-Trent,local_authority,country/england,,,E06000021,local_authority/Stoke-on-Trent +local_authority/E06000022,Bath and North East Somerset,local_authority,country/england,,,E06000022,local_authority/Bath and North East Somerset +local_authority/E06000023,"Bristol, City of",local_authority,country/england,,,E06000023,"local_authority/Bristol, City of" +local_authority/E06000024,North Somerset,local_authority,country/england,,,E06000024,local_authority/North Somerset +local_authority/E06000025,South Gloucestershire,local_authority,country/england,,,E06000025,local_authority/South Gloucestershire +local_authority/E06000026,Plymouth,local_authority,country/england,,,E06000026,local_authority/Plymouth +local_authority/E06000027,Torbay,local_authority,country/england,,,E06000027,local_authority/Torbay +local_authority/E06000030,Swindon,local_authority,country/england,,,E06000030,local_authority/Swindon +local_authority/E06000031,Peterborough,local_authority,country/england,,,E06000031,local_authority/Peterborough +local_authority/E06000032,Luton,local_authority,country/england,,,E06000032,local_authority/Luton +local_authority/E06000033,Southend-on-Sea,local_authority,country/england,,,E06000033,local_authority/Southend-on-Sea +local_authority/E06000034,Thurrock,local_authority,country/england,,,E06000034,local_authority/Thurrock +local_authority/E06000035,Medway,local_authority,country/england,,,E06000035,local_authority/Medway +local_authority/E06000036,Bracknell Forest,local_authority,country/england,,,E06000036,local_authority/Bracknell Forest +local_authority/E06000037,West Berkshire,local_authority,country/england,,,E06000037,local_authority/West Berkshire +local_authority/E06000038,Reading,local_authority,country/england,,,E06000038,local_authority/Reading +local_authority/E06000039,Slough,local_authority,country/england,,,E06000039,local_authority/Slough +local_authority/E06000040,Windsor and Maidenhead,local_authority,country/england,,,E06000040,local_authority/Windsor and Maidenhead +local_authority/E06000041,Wokingham,local_authority,country/england,,,E06000041,local_authority/Wokingham +local_authority/E06000042,Milton Keynes,local_authority,country/england,,,E06000042,local_authority/Milton Keynes +local_authority/E06000043,Brighton and Hove,local_authority,country/england,,,E06000043,local_authority/Brighton and Hove +local_authority/E06000044,Portsmouth,local_authority,country/england,,,E06000044,local_authority/Portsmouth +local_authority/E06000045,Southampton,local_authority,country/england,,,E06000045,local_authority/Southampton +local_authority/E06000046,Isle of Wight,local_authority,country/england,,,E06000046,local_authority/Isle of Wight +local_authority/E06000047,County Durham,local_authority,country/england,,,E06000047,local_authority/County Durham +local_authority/E06000049,Cheshire East,local_authority,country/england,,,E06000049,local_authority/Cheshire East +local_authority/E06000050,Cheshire West and Chester,local_authority,country/england,,,E06000050,local_authority/Cheshire West and Chester +local_authority/E06000051,Shropshire,local_authority,country/england,,,E06000051,local_authority/Shropshire +local_authority/E06000052,Cornwall,local_authority,country/england,,,E06000052,local_authority/Cornwall +local_authority/E06000053,Isles of Scilly,local_authority,country/england,,,E06000053,local_authority/Isles of Scilly +local_authority/E06000054,Wiltshire,local_authority,country/england,,,E06000054,local_authority/Wiltshire +local_authority/E06000055,Bedford,local_authority,country/england,,,E06000055,local_authority/Bedford +local_authority/E06000056,Central Bedfordshire,local_authority,country/england,,,E06000056,local_authority/Central Bedfordshire +local_authority/E06000057,Northumberland,local_authority,country/england,,,E06000057,local_authority/Northumberland +local_authority/E06000058,"Bournemouth, Christchurch and Poole",local_authority,country/england,,,E06000058,"local_authority/Bournemouth, Christchurch and Poole" +local_authority/E06000059,Dorset,local_authority,country/england,,,E06000059,local_authority/Dorset +local_authority/E06000060,Buckinghamshire,local_authority,country/england,,,E06000060,local_authority/Buckinghamshire +local_authority/E06000061,North Northamptonshire,local_authority,country/england,,,E06000061,local_authority/North Northamptonshire +local_authority/E06000062,West Northamptonshire,local_authority,country/england,,,E06000062,local_authority/West Northamptonshire +local_authority/E06000063,Cumberland,local_authority,country/england,,,E06000063,local_authority/Cumberland +local_authority/E06000064,Westmorland and Furness,local_authority,country/england,,,E06000064,local_authority/Westmorland and Furness +local_authority/E06000065,North Yorkshire,local_authority,country/england,,,E06000065,local_authority/North Yorkshire +local_authority/E06000066,Somerset,local_authority,country/england,,,E06000066,local_authority/Somerset +local_authority/E07000008,Cambridge,local_authority,country/england,,,E07000008,local_authority/Cambridge +local_authority/E07000009,East Cambridgeshire,local_authority,country/england,,,E07000009,local_authority/East Cambridgeshire +local_authority/E07000010,Fenland,local_authority,country/england,,,E07000010,local_authority/Fenland +local_authority/E07000011,Huntingdonshire,local_authority,country/england,,,E07000011,local_authority/Huntingdonshire +local_authority/E07000012,South Cambridgeshire,local_authority,country/england,,,E07000012,local_authority/South Cambridgeshire +local_authority/E07000032,Amber Valley,local_authority,country/england,,,E07000032,local_authority/Amber Valley +local_authority/E07000033,Bolsover,local_authority,country/england,,,E07000033,local_authority/Bolsover +local_authority/E07000034,Chesterfield,local_authority,country/england,,,E07000034,local_authority/Chesterfield +local_authority/E07000035,Derbyshire Dales,local_authority,country/england,,,E07000035,local_authority/Derbyshire Dales +local_authority/E07000036,Erewash,local_authority,country/england,,,E07000036,local_authority/Erewash +local_authority/E07000037,High Peak,local_authority,country/england,,,E07000037,local_authority/High Peak +local_authority/E07000038,North East Derbyshire,local_authority,country/england,,,E07000038,local_authority/North East Derbyshire +local_authority/E07000039,South Derbyshire,local_authority,country/england,,,E07000039,local_authority/South Derbyshire +local_authority/E07000040,East Devon,local_authority,country/england,,,E07000040,local_authority/East Devon +local_authority/E07000041,Exeter,local_authority,country/england,,,E07000041,local_authority/Exeter +local_authority/E07000042,Mid Devon,local_authority,country/england,,,E07000042,local_authority/Mid Devon +local_authority/E07000043,North Devon,local_authority,country/england,,,E07000043,local_authority/North Devon +local_authority/E07000044,South Hams,local_authority,country/england,,,E07000044,local_authority/South Hams +local_authority/E07000045,Teignbridge,local_authority,country/england,,,E07000045,local_authority/Teignbridge +local_authority/E07000046,Torridge,local_authority,country/england,,,E07000046,local_authority/Torridge +local_authority/E07000047,West Devon,local_authority,country/england,,,E07000047,local_authority/West Devon +local_authority/E07000061,Eastbourne,local_authority,country/england,,,E07000061,local_authority/Eastbourne +local_authority/E07000062,Hastings,local_authority,country/england,,,E07000062,local_authority/Hastings +local_authority/E07000063,Lewes,local_authority,country/england,,,E07000063,local_authority/Lewes +local_authority/E07000064,Rother,local_authority,country/england,,,E07000064,local_authority/Rother +local_authority/E07000065,Wealden,local_authority,country/england,,,E07000065,local_authority/Wealden +local_authority/E07000066,Basildon,local_authority,country/england,,,E07000066,local_authority/Basildon +local_authority/E07000067,Braintree,local_authority,country/england,,,E07000067,local_authority/Braintree +local_authority/E07000068,Brentwood,local_authority,country/england,,,E07000068,local_authority/Brentwood +local_authority/E07000069,Castle Point,local_authority,country/england,,,E07000069,local_authority/Castle Point +local_authority/E07000070,Chelmsford,local_authority,country/england,,,E07000070,local_authority/Chelmsford +local_authority/E07000071,Colchester,local_authority,country/england,,,E07000071,local_authority/Colchester +local_authority/E07000072,Epping Forest,local_authority,country/england,,,E07000072,local_authority/Epping Forest +local_authority/E07000073,Harlow,local_authority,country/england,,,E07000073,local_authority/Harlow +local_authority/E07000074,Maldon,local_authority,country/england,,,E07000074,local_authority/Maldon +local_authority/E07000075,Rochford,local_authority,country/england,,,E07000075,local_authority/Rochford +local_authority/E07000076,Tendring,local_authority,country/england,,,E07000076,local_authority/Tendring +local_authority/E07000077,Uttlesford,local_authority,country/england,,,E07000077,local_authority/Uttlesford +local_authority/E07000078,Cheltenham,local_authority,country/england,,,E07000078,local_authority/Cheltenham +local_authority/E07000079,Cotswold,local_authority,country/england,,,E07000079,local_authority/Cotswold +local_authority/E07000080,Forest of Dean,local_authority,country/england,,,E07000080,local_authority/Forest of Dean +local_authority/E07000081,Gloucester,local_authority,country/england,,,E07000081,local_authority/Gloucester +local_authority/E07000082,Stroud,local_authority,country/england,,,E07000082,local_authority/Stroud +local_authority/E07000083,Tewkesbury,local_authority,country/england,,,E07000083,local_authority/Tewkesbury +local_authority/E07000084,Basingstoke and Deane,local_authority,country/england,,,E07000084,local_authority/Basingstoke and Deane +local_authority/E07000085,East Hampshire,local_authority,country/england,,,E07000085,local_authority/East Hampshire +local_authority/E07000086,Eastleigh,local_authority,country/england,,,E07000086,local_authority/Eastleigh +local_authority/E07000087,Fareham,local_authority,country/england,,,E07000087,local_authority/Fareham +local_authority/E07000088,Gosport,local_authority,country/england,,,E07000088,local_authority/Gosport +local_authority/E07000089,Hart,local_authority,country/england,,,E07000089,local_authority/Hart +local_authority/E07000090,Havant,local_authority,country/england,,,E07000090,local_authority/Havant +local_authority/E07000091,New Forest,local_authority,country/england,,,E07000091,local_authority/New Forest +local_authority/E07000092,Rushmoor,local_authority,country/england,,,E07000092,local_authority/Rushmoor +local_authority/E07000093,Test Valley,local_authority,country/england,,,E07000093,local_authority/Test Valley +local_authority/E07000094,Winchester,local_authority,country/england,,,E07000094,local_authority/Winchester +local_authority/E07000095,Broxbourne,local_authority,country/england,,,E07000095,local_authority/Broxbourne +local_authority/E07000096,Dacorum,local_authority,country/england,,,E07000096,local_authority/Dacorum +local_authority/E07000098,Hertsmere,local_authority,country/england,,,E07000098,local_authority/Hertsmere +local_authority/E07000099,North Hertfordshire,local_authority,country/england,,,E07000099,local_authority/North Hertfordshire +local_authority/E07000102,Three Rivers,local_authority,country/england,,,E07000102,local_authority/Three Rivers +local_authority/E07000103,Watford,local_authority,country/england,,,E07000103,local_authority/Watford +local_authority/E07000105,Ashford,local_authority,country/england,,,E07000105,local_authority/Ashford +local_authority/E07000106,Canterbury,local_authority,country/england,,,E07000106,local_authority/Canterbury +local_authority/E07000107,Dartford,local_authority,country/england,,,E07000107,local_authority/Dartford +local_authority/E07000108,Dover,local_authority,country/england,,,E07000108,local_authority/Dover +local_authority/E07000109,Gravesham,local_authority,country/england,,,E07000109,local_authority/Gravesham +local_authority/E07000110,Maidstone,local_authority,country/england,,,E07000110,local_authority/Maidstone +local_authority/E07000111,Sevenoaks,local_authority,country/england,,,E07000111,local_authority/Sevenoaks +local_authority/E07000112,Folkestone and Hythe,local_authority,country/england,,,E07000112,local_authority/Folkestone and Hythe +local_authority/E07000113,Swale,local_authority,country/england,,,E07000113,local_authority/Swale +local_authority/E07000114,Thanet,local_authority,country/england,,,E07000114,local_authority/Thanet +local_authority/E07000115,Tonbridge and Malling,local_authority,country/england,,,E07000115,local_authority/Tonbridge and Malling +local_authority/E07000116,Tunbridge Wells,local_authority,country/england,,,E07000116,local_authority/Tunbridge Wells +local_authority/E07000117,Burnley,local_authority,country/england,,,E07000117,local_authority/Burnley +local_authority/E07000118,Chorley,local_authority,country/england,,,E07000118,local_authority/Chorley +local_authority/E07000119,Fylde,local_authority,country/england,,,E07000119,local_authority/Fylde +local_authority/E07000120,Hyndburn,local_authority,country/england,,,E07000120,local_authority/Hyndburn +local_authority/E07000121,Lancaster,local_authority,country/england,,,E07000121,local_authority/Lancaster +local_authority/E07000122,Pendle,local_authority,country/england,,,E07000122,local_authority/Pendle +local_authority/E07000123,Preston,local_authority,country/england,,,E07000123,local_authority/Preston +local_authority/E07000124,Ribble Valley,local_authority,country/england,,,E07000124,local_authority/Ribble Valley +local_authority/E07000125,Rossendale,local_authority,country/england,,,E07000125,local_authority/Rossendale +local_authority/E07000126,South Ribble,local_authority,country/england,,,E07000126,local_authority/South Ribble +local_authority/E07000127,West Lancashire,local_authority,country/england,,,E07000127,local_authority/West Lancashire +local_authority/E07000128,Wyre,local_authority,country/england,,,E07000128,local_authority/Wyre +local_authority/E07000129,Blaby,local_authority,country/england,,,E07000129,local_authority/Blaby +local_authority/E07000130,Charnwood,local_authority,country/england,,,E07000130,local_authority/Charnwood +local_authority/E07000131,Harborough,local_authority,country/england,,,E07000131,local_authority/Harborough +local_authority/E07000132,Hinckley and Bosworth,local_authority,country/england,,,E07000132,local_authority/Hinckley and Bosworth +local_authority/E07000133,Melton,local_authority,country/england,,,E07000133,local_authority/Melton +local_authority/E07000134,North West Leicestershire,local_authority,country/england,,,E07000134,local_authority/North West Leicestershire +local_authority/E07000135,Oadby and Wigston,local_authority,country/england,,,E07000135,local_authority/Oadby and Wigston +local_authority/E07000136,Boston,local_authority,country/england,,,E07000136,local_authority/Boston +local_authority/E07000137,East Lindsey,local_authority,country/england,,,E07000137,local_authority/East Lindsey +local_authority/E07000138,Lincoln,local_authority,country/england,,,E07000138,local_authority/Lincoln +local_authority/E07000139,North Kesteven,local_authority,country/england,,,E07000139,local_authority/North Kesteven +local_authority/E07000140,South Holland,local_authority,country/england,,,E07000140,local_authority/South Holland +local_authority/E07000141,South Kesteven,local_authority,country/england,,,E07000141,local_authority/South Kesteven +local_authority/E07000142,West Lindsey,local_authority,country/england,,,E07000142,local_authority/West Lindsey +local_authority/E07000143,Breckland,local_authority,country/england,,,E07000143,local_authority/Breckland +local_authority/E07000144,Broadland,local_authority,country/england,,,E07000144,local_authority/Broadland +local_authority/E07000145,Great Yarmouth,local_authority,country/england,,,E07000145,local_authority/Great Yarmouth +local_authority/E07000146,King's Lynn and West Norfolk,local_authority,country/england,,,E07000146,local_authority/King's Lynn and West Norfolk +local_authority/E07000147,North Norfolk,local_authority,country/england,,,E07000147,local_authority/North Norfolk +local_authority/E07000148,Norwich,local_authority,country/england,,,E07000148,local_authority/Norwich +local_authority/E07000149,South Norfolk,local_authority,country/england,,,E07000149,local_authority/South Norfolk +local_authority/E07000170,Ashfield,local_authority,country/england,,,E07000170,local_authority/Ashfield +local_authority/E07000171,Bassetlaw,local_authority,country/england,,,E07000171,local_authority/Bassetlaw +local_authority/E07000172,Broxtowe,local_authority,country/england,,,E07000172,local_authority/Broxtowe +local_authority/E07000173,Gedling,local_authority,country/england,,,E07000173,local_authority/Gedling +local_authority/E07000174,Mansfield,local_authority,country/england,,,E07000174,local_authority/Mansfield +local_authority/E07000175,Newark and Sherwood,local_authority,country/england,,,E07000175,local_authority/Newark and Sherwood +local_authority/E07000176,Rushcliffe,local_authority,country/england,,,E07000176,local_authority/Rushcliffe +local_authority/E07000177,Cherwell,local_authority,country/england,,,E07000177,local_authority/Cherwell +local_authority/E07000178,Oxford,local_authority,country/england,,,E07000178,local_authority/Oxford +local_authority/E07000179,South Oxfordshire,local_authority,country/england,,,E07000179,local_authority/South Oxfordshire +local_authority/E07000180,Vale of White Horse,local_authority,country/england,,,E07000180,local_authority/Vale of White Horse +local_authority/E07000181,West Oxfordshire,local_authority,country/england,,,E07000181,local_authority/West Oxfordshire +local_authority/E07000192,Cannock Chase,local_authority,country/england,,,E07000192,local_authority/Cannock Chase +local_authority/E07000193,East Staffordshire,local_authority,country/england,,,E07000193,local_authority/East Staffordshire +local_authority/E07000194,Lichfield,local_authority,country/england,,,E07000194,local_authority/Lichfield +local_authority/E07000195,Newcastle-under-Lyme,local_authority,country/england,,,E07000195,local_authority/Newcastle-under-Lyme +local_authority/E07000196,South Staffordshire,local_authority,country/england,,,E07000196,local_authority/South Staffordshire +local_authority/E07000197,Stafford,local_authority,country/england,,,E07000197,local_authority/Stafford +local_authority/E07000198,Staffordshire Moorlands,local_authority,country/england,,,E07000198,local_authority/Staffordshire Moorlands +local_authority/E07000199,Tamworth,local_authority,country/england,,,E07000199,local_authority/Tamworth +local_authority/E07000200,Babergh,local_authority,country/england,,,E07000200,local_authority/Babergh +local_authority/E07000202,Ipswich,local_authority,country/england,,,E07000202,local_authority/Ipswich +local_authority/E07000203,Mid Suffolk,local_authority,country/england,,,E07000203,local_authority/Mid Suffolk +local_authority/E07000207,Elmbridge,local_authority,country/england,,,E07000207,local_authority/Elmbridge +local_authority/E07000208,Epsom and Ewell,local_authority,country/england,,,E07000208,local_authority/Epsom and Ewell +local_authority/E07000209,Guildford,local_authority,country/england,,,E07000209,local_authority/Guildford +local_authority/E07000210,Mole Valley,local_authority,country/england,,,E07000210,local_authority/Mole Valley +local_authority/E07000211,Reigate and Banstead,local_authority,country/england,,,E07000211,local_authority/Reigate and Banstead +local_authority/E07000212,Runnymede,local_authority,country/england,,,E07000212,local_authority/Runnymede +local_authority/E07000213,Spelthorne,local_authority,country/england,,,E07000213,local_authority/Spelthorne +local_authority/E07000214,Surrey Heath,local_authority,country/england,,,E07000214,local_authority/Surrey Heath +local_authority/E07000215,Tandridge,local_authority,country/england,,,E07000215,local_authority/Tandridge +local_authority/E07000216,Waverley,local_authority,country/england,,,E07000216,local_authority/Waverley +local_authority/E07000217,Woking,local_authority,country/england,,,E07000217,local_authority/Woking +local_authority/E07000218,North Warwickshire,local_authority,country/england,,,E07000218,local_authority/North Warwickshire +local_authority/E07000219,Nuneaton and Bedworth,local_authority,country/england,,,E07000219,local_authority/Nuneaton and Bedworth +local_authority/E07000220,Rugby,local_authority,country/england,,,E07000220,local_authority/Rugby +local_authority/E07000221,Stratford-on-Avon,local_authority,country/england,,,E07000221,local_authority/Stratford-on-Avon +local_authority/E07000222,Warwick,local_authority,country/england,,,E07000222,local_authority/Warwick +local_authority/E07000223,Adur,local_authority,country/england,,,E07000223,local_authority/Adur +local_authority/E07000224,Arun,local_authority,country/england,,,E07000224,local_authority/Arun +local_authority/E07000225,Chichester,local_authority,country/england,,,E07000225,local_authority/Chichester +local_authority/E07000226,Crawley,local_authority,country/england,,,E07000226,local_authority/Crawley +local_authority/E07000227,Horsham,local_authority,country/england,,,E07000227,local_authority/Horsham +local_authority/E07000228,Mid Sussex,local_authority,country/england,,,E07000228,local_authority/Mid Sussex +local_authority/E07000229,Worthing,local_authority,country/england,,,E07000229,local_authority/Worthing +local_authority/E07000234,Bromsgrove,local_authority,country/england,,,E07000234,local_authority/Bromsgrove +local_authority/E07000235,Malvern Hills,local_authority,country/england,,,E07000235,local_authority/Malvern Hills +local_authority/E07000236,Redditch,local_authority,country/england,,,E07000236,local_authority/Redditch +local_authority/E07000237,Worcester,local_authority,country/england,,,E07000237,local_authority/Worcester +local_authority/E07000238,Wychavon,local_authority,country/england,,,E07000238,local_authority/Wychavon +local_authority/E07000239,Wyre Forest,local_authority,country/england,,,E07000239,local_authority/Wyre Forest +local_authority/E07000240,St Albans,local_authority,country/england,,,E07000240,local_authority/St Albans +local_authority/E07000241,Welwyn Hatfield,local_authority,country/england,,,E07000241,local_authority/Welwyn Hatfield +local_authority/E07000242,East Hertfordshire,local_authority,country/england,,,E07000242,local_authority/East Hertfordshire +local_authority/E07000243,Stevenage,local_authority,country/england,,,E07000243,local_authority/Stevenage +local_authority/E07000244,East Suffolk,local_authority,country/england,,,E07000244,local_authority/East Suffolk +local_authority/E07000245,West Suffolk,local_authority,country/england,,,E07000245,local_authority/West Suffolk +local_authority/E08000001,Bolton,local_authority,country/england,,,E08000001,local_authority/Bolton +local_authority/E08000002,Bury,local_authority,country/england,,,E08000002,local_authority/Bury +local_authority/E08000003,Manchester,local_authority,country/england,,,E08000003,local_authority/Manchester +local_authority/E08000004,Oldham,local_authority,country/england,,,E08000004,local_authority/Oldham +local_authority/E08000005,Rochdale,local_authority,country/england,,,E08000005,local_authority/Rochdale +local_authority/E08000006,Salford,local_authority,country/england,,,E08000006,local_authority/Salford +local_authority/E08000007,Stockport,local_authority,country/england,,,E08000007,local_authority/Stockport +local_authority/E08000008,Tameside,local_authority,country/england,,,E08000008,local_authority/Tameside +local_authority/E08000009,Trafford,local_authority,country/england,,,E08000009,local_authority/Trafford +local_authority/E08000010,Wigan,local_authority,country/england,,,E08000010,local_authority/Wigan +local_authority/E08000011,Knowsley,local_authority,country/england,,,E08000011,local_authority/Knowsley +local_authority/E08000012,Liverpool,local_authority,country/england,,,E08000012,local_authority/Liverpool +local_authority/E08000013,St. Helens,local_authority,country/england,,,E08000013,local_authority/St. Helens +local_authority/E08000014,Sefton,local_authority,country/england,,,E08000014,local_authority/Sefton +local_authority/E08000015,Wirral,local_authority,country/england,,,E08000015,local_authority/Wirral +local_authority/E08000016,Barnsley,local_authority,country/england,,,E08000016,local_authority/Barnsley +local_authority/E08000017,Doncaster,local_authority,country/england,,,E08000017,local_authority/Doncaster +local_authority/E08000018,Rotherham,local_authority,country/england,,,E08000018,local_authority/Rotherham +local_authority/E08000019,Sheffield,local_authority,country/england,,,E08000019,local_authority/Sheffield +local_authority/E08000021,Newcastle upon Tyne,local_authority,country/england,,,E08000021,local_authority/Newcastle upon Tyne +local_authority/E08000022,North Tyneside,local_authority,country/england,,,E08000022,local_authority/North Tyneside +local_authority/E08000023,South Tyneside,local_authority,country/england,,,E08000023,local_authority/South Tyneside +local_authority/E08000024,Sunderland,local_authority,country/england,,,E08000024,local_authority/Sunderland +local_authority/E08000025,Birmingham,local_authority,country/england,,,E08000025,local_authority/Birmingham +local_authority/E08000026,Coventry,local_authority,country/england,,,E08000026,local_authority/Coventry +local_authority/E08000027,Dudley,local_authority,country/england,,,E08000027,local_authority/Dudley +local_authority/E08000028,Sandwell,local_authority,country/england,,,E08000028,local_authority/Sandwell +local_authority/E08000029,Solihull,local_authority,country/england,,,E08000029,local_authority/Solihull +local_authority/E08000030,Walsall,local_authority,country/england,,,E08000030,local_authority/Walsall +local_authority/E08000031,Wolverhampton,local_authority,country/england,,,E08000031,local_authority/Wolverhampton +local_authority/E08000032,Bradford,local_authority,country/england,,,E08000032,local_authority/Bradford +local_authority/E08000033,Calderdale,local_authority,country/england,,,E08000033,local_authority/Calderdale +local_authority/E08000034,Kirklees,local_authority,country/england,,,E08000034,local_authority/Kirklees +local_authority/E08000035,Leeds,local_authority,country/england,,,E08000035,local_authority/Leeds +local_authority/E08000036,Wakefield,local_authority,country/england,,,E08000036,local_authority/Wakefield +local_authority/E08000037,Gateshead,local_authority,country/england,,,E08000037,local_authority/Gateshead +local_authority/E09000001,City of London,local_authority,country/england,,,E09000001,local_authority/City of London +local_authority/E09000002,Barking and Dagenham,local_authority,country/england,,,E09000002,local_authority/Barking and Dagenham +local_authority/E09000003,Barnet,local_authority,country/england,,,E09000003,local_authority/Barnet +local_authority/E09000004,Bexley,local_authority,country/england,,,E09000004,local_authority/Bexley +local_authority/E09000005,Brent,local_authority,country/england,,,E09000005,local_authority/Brent +local_authority/E09000006,Bromley,local_authority,country/england,,,E09000006,local_authority/Bromley +local_authority/E09000007,Camden,local_authority,country/england,,,E09000007,local_authority/Camden +local_authority/E09000008,Croydon,local_authority,country/england,,,E09000008,local_authority/Croydon +local_authority/E09000009,Ealing,local_authority,country/england,,,E09000009,local_authority/Ealing +local_authority/E09000010,Enfield,local_authority,country/england,,,E09000010,local_authority/Enfield +local_authority/E09000011,Greenwich,local_authority,country/england,,,E09000011,local_authority/Greenwich +local_authority/E09000012,Hackney,local_authority,country/england,,,E09000012,local_authority/Hackney +local_authority/E09000013,Hammersmith and Fulham,local_authority,country/england,,,E09000013,local_authority/Hammersmith and Fulham +local_authority/E09000014,Haringey,local_authority,country/england,,,E09000014,local_authority/Haringey +local_authority/E09000015,Harrow,local_authority,country/england,,,E09000015,local_authority/Harrow +local_authority/E09000016,Havering,local_authority,country/england,,,E09000016,local_authority/Havering +local_authority/E09000017,Hillingdon,local_authority,country/england,,,E09000017,local_authority/Hillingdon +local_authority/E09000018,Hounslow,local_authority,country/england,,,E09000018,local_authority/Hounslow +local_authority/E09000019,Islington,local_authority,country/england,,,E09000019,local_authority/Islington +local_authority/E09000020,Kensington and Chelsea,local_authority,country/england,,,E09000020,local_authority/Kensington and Chelsea +local_authority/E09000021,Kingston upon Thames,local_authority,country/england,,,E09000021,local_authority/Kingston upon Thames +local_authority/E09000022,Lambeth,local_authority,country/england,,,E09000022,local_authority/Lambeth +local_authority/E09000023,Lewisham,local_authority,country/england,,,E09000023,local_authority/Lewisham +local_authority/E09000024,Merton,local_authority,country/england,,,E09000024,local_authority/Merton +local_authority/E09000025,Newham,local_authority,country/england,,,E09000025,local_authority/Newham +local_authority/E09000026,Redbridge,local_authority,country/england,,,E09000026,local_authority/Redbridge +local_authority/E09000027,Richmond upon Thames,local_authority,country/england,,,E09000027,local_authority/Richmond upon Thames +local_authority/E09000028,Southwark,local_authority,country/england,,,E09000028,local_authority/Southwark +local_authority/E09000029,Sutton,local_authority,country/england,,,E09000029,local_authority/Sutton +local_authority/E09000030,Tower Hamlets,local_authority,country/england,,,E09000030,local_authority/Tower Hamlets +local_authority/E09000031,Waltham Forest,local_authority,country/england,,,E09000031,local_authority/Waltham Forest +local_authority/E09000032,Wandsworth,local_authority,country/england,,,E09000032,local_authority/Wandsworth +local_authority/E09000033,Westminster,local_authority,country/england,,,E09000033,local_authority/Westminster +local_authority/N09000001,Antrim and Newtownabbey,local_authority,country/northern_ireland,,,N09000001,local_authority/Antrim and Newtownabbey +local_authority/N09000002,"Armagh City, Banbridge and Craigavon",local_authority,country/northern_ireland,,,N09000002,"local_authority/Armagh City, Banbridge and Craigavon" +local_authority/N09000003,Belfast,local_authority,country/northern_ireland,,,N09000003,local_authority/Belfast +local_authority/N09000004,Causeway Coast and Glens,local_authority,country/northern_ireland,,,N09000004,local_authority/Causeway Coast and Glens +local_authority/N09000005,Derry City and Strabane,local_authority,country/northern_ireland,,,N09000005,local_authority/Derry City and Strabane +local_authority/N09000006,Fermanagh and Omagh,local_authority,country/northern_ireland,,,N09000006,local_authority/Fermanagh and Omagh +local_authority/N09000007,Lisburn and Castlereagh,local_authority,country/northern_ireland,,,N09000007,local_authority/Lisburn and Castlereagh +local_authority/N09000008,Mid and East Antrim,local_authority,country/northern_ireland,,,N09000008,local_authority/Mid and East Antrim +local_authority/N09000009,Mid Ulster,local_authority,country/northern_ireland,,,N09000009,local_authority/Mid Ulster +local_authority/N09000010,"Newry, Mourne and Down",local_authority,country/northern_ireland,,,N09000010,"local_authority/Newry, Mourne and Down" +local_authority/N09000011,Ards and North Down,local_authority,country/northern_ireland,,,N09000011,local_authority/Ards and North Down +local_authority/S12000005,Clackmannanshire,local_authority,country/scotland,,,S12000005,local_authority/Clackmannanshire +local_authority/S12000006,Dumfries and Galloway,local_authority,country/scotland,,,S12000006,local_authority/Dumfries and Galloway +local_authority/S12000008,East Ayrshire,local_authority,country/scotland,,,S12000008,local_authority/East Ayrshire +local_authority/S12000010,East Lothian,local_authority,country/scotland,,,S12000010,local_authority/East Lothian +local_authority/S12000011,East Renfrewshire,local_authority,country/scotland,,,S12000011,local_authority/East Renfrewshire +local_authority/S12000013,Na h-Eileanan Siar,local_authority,country/scotland,,,S12000013,local_authority/Na h-Eileanan Siar +local_authority/S12000014,Falkirk,local_authority,country/scotland,,,S12000014,local_authority/Falkirk +local_authority/S12000017,Highland,local_authority,country/scotland,,,S12000017,local_authority/Highland +local_authority/S12000018,Inverclyde,local_authority,country/scotland,,,S12000018,local_authority/Inverclyde +local_authority/S12000019,Midlothian,local_authority,country/scotland,,,S12000019,local_authority/Midlothian +local_authority/S12000020,Moray,local_authority,country/scotland,,,S12000020,local_authority/Moray +local_authority/S12000021,North Ayrshire,local_authority,country/scotland,,,S12000021,local_authority/North Ayrshire +local_authority/S12000023,Orkney Islands,local_authority,country/scotland,,,S12000023,local_authority/Orkney Islands +local_authority/S12000026,Scottish Borders,local_authority,country/scotland,,,S12000026,local_authority/Scottish Borders +local_authority/S12000027,Shetland Islands,local_authority,country/scotland,,,S12000027,local_authority/Shetland Islands +local_authority/S12000028,South Ayrshire,local_authority,country/scotland,,,S12000028,local_authority/South Ayrshire +local_authority/S12000029,South Lanarkshire,local_authority,country/scotland,,,S12000029,local_authority/South Lanarkshire +local_authority/S12000030,Stirling,local_authority,country/scotland,,,S12000030,local_authority/Stirling +local_authority/S12000033,Aberdeen City,local_authority,country/scotland,,,S12000033,local_authority/Aberdeen City +local_authority/S12000034,Aberdeenshire,local_authority,country/scotland,,,S12000034,local_authority/Aberdeenshire +local_authority/S12000035,Argyll and Bute,local_authority,country/scotland,,,S12000035,local_authority/Argyll and Bute +local_authority/S12000036,City of Edinburgh,local_authority,country/scotland,,,S12000036,local_authority/City of Edinburgh +local_authority/S12000038,Renfrewshire,local_authority,country/scotland,,,S12000038,local_authority/Renfrewshire +local_authority/S12000039,West Dunbartonshire,local_authority,country/scotland,,,S12000039,local_authority/West Dunbartonshire +local_authority/S12000040,West Lothian,local_authority,country/scotland,,,S12000040,local_authority/West Lothian +local_authority/S12000041,Angus,local_authority,country/scotland,,,S12000041,local_authority/Angus +local_authority/S12000042,Dundee City,local_authority,country/scotland,,,S12000042,local_authority/Dundee City +local_authority/S12000045,East Dunbartonshire,local_authority,country/scotland,,,S12000045,local_authority/East Dunbartonshire +local_authority/S12000047,Fife,local_authority,country/scotland,,,S12000047,local_authority/Fife +local_authority/S12000048,Perth and Kinross,local_authority,country/scotland,,,S12000048,local_authority/Perth and Kinross +local_authority/S12000049,Glasgow City,local_authority,country/scotland,,,S12000049,local_authority/Glasgow City +local_authority/S12000050,North Lanarkshire,local_authority,country/scotland,,,S12000050,local_authority/North Lanarkshire +local_authority/W06000001,Isle of Anglesey,local_authority,country/wales,,,W06000001,local_authority/Isle of Anglesey +local_authority/W06000002,Gwynedd,local_authority,country/wales,,,W06000002,local_authority/Gwynedd +local_authority/W06000003,Conwy,local_authority,country/wales,,,W06000003,local_authority/Conwy +local_authority/W06000004,Denbighshire,local_authority,country/wales,,,W06000004,local_authority/Denbighshire +local_authority/W06000005,Flintshire,local_authority,country/wales,,,W06000005,local_authority/Flintshire +local_authority/W06000006,Wrexham,local_authority,country/wales,,,W06000006,local_authority/Wrexham +local_authority/W06000008,Ceredigion,local_authority,country/wales,,,W06000008,local_authority/Ceredigion +local_authority/W06000009,Pembrokeshire,local_authority,country/wales,,,W06000009,local_authority/Pembrokeshire +local_authority/W06000010,Carmarthenshire,local_authority,country/wales,,,W06000010,local_authority/Carmarthenshire +local_authority/W06000011,Swansea,local_authority,country/wales,,,W06000011,local_authority/Swansea +local_authority/W06000012,Neath Port Talbot,local_authority,country/wales,,,W06000012,local_authority/Neath Port Talbot +local_authority/W06000013,Bridgend,local_authority,country/wales,,,W06000013,local_authority/Bridgend +local_authority/W06000014,Vale of Glamorgan,local_authority,country/wales,,,W06000014,local_authority/Vale of Glamorgan +local_authority/W06000015,Cardiff,local_authority,country/wales,,,W06000015,local_authority/Cardiff +local_authority/W06000016,Rhondda Cynon Taf,local_authority,country/wales,,,W06000016,local_authority/Rhondda Cynon Taf +local_authority/W06000018,Caerphilly,local_authority,country/wales,,,W06000018,local_authority/Caerphilly +local_authority/W06000019,Blaenau Gwent,local_authority,country/wales,,,W06000019,local_authority/Blaenau Gwent +local_authority/W06000020,Torfaen,local_authority,country/wales,,,W06000020,local_authority/Torfaen +local_authority/W06000021,Monmouthshire,local_authority,country/wales,,,W06000021,local_authority/Monmouthshire +local_authority/W06000022,Newport,local_authority,country/wales,,,W06000022,local_authority/Newport +local_authority/W06000023,Powys,local_authority,country/wales,,,W06000023,local_authority/Powys +local_authority/W06000024,Merthyr Tydfil,local_authority,country/wales,,,W06000024,local_authority/Merthyr Tydfil diff --git a/policyengine_api/data/v2/catalog/region_reference_files/us.csv b/policyengine_api/data/v2/catalog/region_reference_files/us.csv new file mode 100644 index 000000000..a8a3b3166 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_reference_files/us.csv @@ -0,0 +1,822 @@ +code,label,region_type,parent_code,state_code,state_name,filter_value,v1_code +us,United States,national,,,,,us +state/ak,Alaska,state,us,AK,Alaska,2,state/ak +state/al,Alabama,state,us,AL,Alabama,1,state/al +state/ar,Arkansas,state,us,AR,Arkansas,5,state/ar +state/az,Arizona,state,us,AZ,Arizona,4,state/az +state/ca,California,state,us,CA,California,6,state/ca +state/co,Colorado,state,us,CO,Colorado,8,state/co +state/ct,Connecticut,state,us,CT,Connecticut,9,state/ct +state/dc,District of Columbia,state,us,DC,District of Columbia,11,state/dc +state/de,Delaware,state,us,DE,Delaware,10,state/de +state/fl,Florida,state,us,FL,Florida,12,state/fl +state/ga,Georgia,state,us,GA,Georgia,13,state/ga +state/hi,Hawaii,state,us,HI,Hawaii,15,state/hi +state/ia,Iowa,state,us,IA,Iowa,19,state/ia +state/id,Idaho,state,us,ID,Idaho,16,state/id +state/il,Illinois,state,us,IL,Illinois,17,state/il +state/in,Indiana,state,us,IN,Indiana,18,state/in +state/ks,Kansas,state,us,KS,Kansas,20,state/ks +state/ky,Kentucky,state,us,KY,Kentucky,21,state/ky +state/la,Louisiana,state,us,LA,Louisiana,22,state/la +state/ma,Massachusetts,state,us,MA,Massachusetts,25,state/ma +state/md,Maryland,state,us,MD,Maryland,24,state/md +state/me,Maine,state,us,ME,Maine,23,state/me +state/mi,Michigan,state,us,MI,Michigan,26,state/mi +state/mn,Minnesota,state,us,MN,Minnesota,27,state/mn +state/mo,Missouri,state,us,MO,Missouri,29,state/mo +state/ms,Mississippi,state,us,MS,Mississippi,28,state/ms +state/mt,Montana,state,us,MT,Montana,30,state/mt +state/nc,North Carolina,state,us,NC,North Carolina,37,state/nc +state/nd,North Dakota,state,us,ND,North Dakota,38,state/nd +state/ne,Nebraska,state,us,NE,Nebraska,31,state/ne +state/nh,New Hampshire,state,us,NH,New Hampshire,33,state/nh +state/nj,New Jersey,state,us,NJ,New Jersey,34,state/nj +state/nm,New Mexico,state,us,NM,New Mexico,35,state/nm +state/nv,Nevada,state,us,NV,Nevada,32,state/nv +state/ny,New York,state,us,NY,New York,36,state/ny +state/oh,Ohio,state,us,OH,Ohio,39,state/oh +state/ok,Oklahoma,state,us,OK,Oklahoma,40,state/ok +state/or,Oregon,state,us,OR,Oregon,41,state/or +state/pa,Pennsylvania,state,us,PA,Pennsylvania,42,state/pa +state/ri,Rhode Island,state,us,RI,Rhode Island,44,state/ri +state/sc,South Carolina,state,us,SC,South Carolina,45,state/sc +state/sd,South Dakota,state,us,SD,South Dakota,46,state/sd +state/tn,Tennessee,state,us,TN,Tennessee,47,state/tn +state/tx,Texas,state,us,TX,Texas,48,state/tx +state/ut,Utah,state,us,UT,Utah,49,state/ut +state/va,Virginia,state,us,VA,Virginia,51,state/va +state/vt,Vermont,state,us,VT,Vermont,50,state/vt +state/wa,Washington,state,us,WA,Washington,53,state/wa +state/wi,Wisconsin,state,us,WI,Wisconsin,55,state/wi +state/wv,West Virginia,state,us,WV,West Virginia,54,state/wv +state/wy,Wyoming,state,us,WY,Wyoming,56,state/wy +congressional_district/AK-01,Alaska's at-large congressional district,congressional_district,state/ak,AK,Alaska,200,congressional_district/AK-01 +congressional_district/AL-01,Alabama's 1st congressional district,congressional_district,state/al,AL,Alabama,101,congressional_district/AL-01 +congressional_district/AL-02,Alabama's 2nd congressional district,congressional_district,state/al,AL,Alabama,102,congressional_district/AL-02 +congressional_district/AL-03,Alabama's 3rd congressional district,congressional_district,state/al,AL,Alabama,103,congressional_district/AL-03 +congressional_district/AL-04,Alabama's 4th congressional district,congressional_district,state/al,AL,Alabama,104,congressional_district/AL-04 +congressional_district/AL-05,Alabama's 5th congressional district,congressional_district,state/al,AL,Alabama,105,congressional_district/AL-05 +congressional_district/AL-06,Alabama's 6th congressional district,congressional_district,state/al,AL,Alabama,106,congressional_district/AL-06 +congressional_district/AL-07,Alabama's 7th congressional district,congressional_district,state/al,AL,Alabama,107,congressional_district/AL-07 +congressional_district/AR-01,Arkansas's 1st congressional district,congressional_district,state/ar,AR,Arkansas,501,congressional_district/AR-01 +congressional_district/AR-02,Arkansas's 2nd congressional district,congressional_district,state/ar,AR,Arkansas,502,congressional_district/AR-02 +congressional_district/AR-03,Arkansas's 3rd congressional district,congressional_district,state/ar,AR,Arkansas,503,congressional_district/AR-03 +congressional_district/AR-04,Arkansas's 4th congressional district,congressional_district,state/ar,AR,Arkansas,504,congressional_district/AR-04 +congressional_district/AZ-01,Arizona's 1st congressional district,congressional_district,state/az,AZ,Arizona,401,congressional_district/AZ-01 +congressional_district/AZ-02,Arizona's 2nd congressional district,congressional_district,state/az,AZ,Arizona,402,congressional_district/AZ-02 +congressional_district/AZ-03,Arizona's 3rd congressional district,congressional_district,state/az,AZ,Arizona,403,congressional_district/AZ-03 +congressional_district/AZ-04,Arizona's 4th congressional district,congressional_district,state/az,AZ,Arizona,404,congressional_district/AZ-04 +congressional_district/AZ-05,Arizona's 5th congressional district,congressional_district,state/az,AZ,Arizona,405,congressional_district/AZ-05 +congressional_district/AZ-06,Arizona's 6th congressional district,congressional_district,state/az,AZ,Arizona,406,congressional_district/AZ-06 +congressional_district/AZ-07,Arizona's 7th congressional district,congressional_district,state/az,AZ,Arizona,407,congressional_district/AZ-07 +congressional_district/AZ-08,Arizona's 8th congressional district,congressional_district,state/az,AZ,Arizona,408,congressional_district/AZ-08 +congressional_district/AZ-09,Arizona's 9th congressional district,congressional_district,state/az,AZ,Arizona,409,congressional_district/AZ-09 +congressional_district/CA-01,California's 1st congressional district,congressional_district,state/ca,CA,California,601,congressional_district/CA-01 +congressional_district/CA-02,California's 2nd congressional district,congressional_district,state/ca,CA,California,602,congressional_district/CA-02 +congressional_district/CA-03,California's 3rd congressional district,congressional_district,state/ca,CA,California,603,congressional_district/CA-03 +congressional_district/CA-04,California's 4th congressional district,congressional_district,state/ca,CA,California,604,congressional_district/CA-04 +congressional_district/CA-05,California's 5th congressional district,congressional_district,state/ca,CA,California,605,congressional_district/CA-05 +congressional_district/CA-06,California's 6th congressional district,congressional_district,state/ca,CA,California,606,congressional_district/CA-06 +congressional_district/CA-07,California's 7th congressional district,congressional_district,state/ca,CA,California,607,congressional_district/CA-07 +congressional_district/CA-08,California's 8th congressional district,congressional_district,state/ca,CA,California,608,congressional_district/CA-08 +congressional_district/CA-09,California's 9th congressional district,congressional_district,state/ca,CA,California,609,congressional_district/CA-09 +congressional_district/CA-10,California's 10th congressional district,congressional_district,state/ca,CA,California,610,congressional_district/CA-10 +congressional_district/CA-11,California's 11th congressional district,congressional_district,state/ca,CA,California,611,congressional_district/CA-11 +congressional_district/CA-12,California's 12th congressional district,congressional_district,state/ca,CA,California,612,congressional_district/CA-12 +congressional_district/CA-13,California's 13th congressional district,congressional_district,state/ca,CA,California,613,congressional_district/CA-13 +congressional_district/CA-14,California's 14th congressional district,congressional_district,state/ca,CA,California,614,congressional_district/CA-14 +congressional_district/CA-15,California's 15th congressional district,congressional_district,state/ca,CA,California,615,congressional_district/CA-15 +congressional_district/CA-16,California's 16th congressional district,congressional_district,state/ca,CA,California,616,congressional_district/CA-16 +congressional_district/CA-17,California's 17th congressional district,congressional_district,state/ca,CA,California,617,congressional_district/CA-17 +congressional_district/CA-18,California's 18th congressional district,congressional_district,state/ca,CA,California,618,congressional_district/CA-18 +congressional_district/CA-19,California's 19th congressional district,congressional_district,state/ca,CA,California,619,congressional_district/CA-19 +congressional_district/CA-20,California's 20th congressional district,congressional_district,state/ca,CA,California,620,congressional_district/CA-20 +congressional_district/CA-21,California's 21st congressional district,congressional_district,state/ca,CA,California,621,congressional_district/CA-21 +congressional_district/CA-22,California's 22nd congressional district,congressional_district,state/ca,CA,California,622,congressional_district/CA-22 +congressional_district/CA-23,California's 23rd congressional district,congressional_district,state/ca,CA,California,623,congressional_district/CA-23 +congressional_district/CA-24,California's 24th congressional district,congressional_district,state/ca,CA,California,624,congressional_district/CA-24 +congressional_district/CA-25,California's 25th congressional district,congressional_district,state/ca,CA,California,625,congressional_district/CA-25 +congressional_district/CA-26,California's 26th congressional district,congressional_district,state/ca,CA,California,626,congressional_district/CA-26 +congressional_district/CA-27,California's 27th congressional district,congressional_district,state/ca,CA,California,627,congressional_district/CA-27 +congressional_district/CA-28,California's 28th congressional district,congressional_district,state/ca,CA,California,628,congressional_district/CA-28 +congressional_district/CA-29,California's 29th congressional district,congressional_district,state/ca,CA,California,629,congressional_district/CA-29 +congressional_district/CA-30,California's 30th congressional district,congressional_district,state/ca,CA,California,630,congressional_district/CA-30 +congressional_district/CA-31,California's 31st congressional district,congressional_district,state/ca,CA,California,631,congressional_district/CA-31 +congressional_district/CA-32,California's 32nd congressional district,congressional_district,state/ca,CA,California,632,congressional_district/CA-32 +congressional_district/CA-33,California's 33rd congressional district,congressional_district,state/ca,CA,California,633,congressional_district/CA-33 +congressional_district/CA-34,California's 34th congressional district,congressional_district,state/ca,CA,California,634,congressional_district/CA-34 +congressional_district/CA-35,California's 35th congressional district,congressional_district,state/ca,CA,California,635,congressional_district/CA-35 +congressional_district/CA-36,California's 36th congressional district,congressional_district,state/ca,CA,California,636,congressional_district/CA-36 +congressional_district/CA-37,California's 37th congressional district,congressional_district,state/ca,CA,California,637,congressional_district/CA-37 +congressional_district/CA-38,California's 38th congressional district,congressional_district,state/ca,CA,California,638,congressional_district/CA-38 +congressional_district/CA-39,California's 39th congressional district,congressional_district,state/ca,CA,California,639,congressional_district/CA-39 +congressional_district/CA-40,California's 40th congressional district,congressional_district,state/ca,CA,California,640,congressional_district/CA-40 +congressional_district/CA-41,California's 41st congressional district,congressional_district,state/ca,CA,California,641,congressional_district/CA-41 +congressional_district/CA-42,California's 42nd congressional district,congressional_district,state/ca,CA,California,642,congressional_district/CA-42 +congressional_district/CA-43,California's 43rd congressional district,congressional_district,state/ca,CA,California,643,congressional_district/CA-43 +congressional_district/CA-44,California's 44th congressional district,congressional_district,state/ca,CA,California,644,congressional_district/CA-44 +congressional_district/CA-45,California's 45th congressional district,congressional_district,state/ca,CA,California,645,congressional_district/CA-45 +congressional_district/CA-46,California's 46th congressional district,congressional_district,state/ca,CA,California,646,congressional_district/CA-46 +congressional_district/CA-47,California's 47th congressional district,congressional_district,state/ca,CA,California,647,congressional_district/CA-47 +congressional_district/CA-48,California's 48th congressional district,congressional_district,state/ca,CA,California,648,congressional_district/CA-48 +congressional_district/CA-49,California's 49th congressional district,congressional_district,state/ca,CA,California,649,congressional_district/CA-49 +congressional_district/CA-50,California's 50th congressional district,congressional_district,state/ca,CA,California,650,congressional_district/CA-50 +congressional_district/CA-51,California's 51st congressional district,congressional_district,state/ca,CA,California,651,congressional_district/CA-51 +congressional_district/CA-52,California's 52nd congressional district,congressional_district,state/ca,CA,California,652,congressional_district/CA-52 +congressional_district/CO-01,Colorado's 1st congressional district,congressional_district,state/co,CO,Colorado,801,congressional_district/CO-01 +congressional_district/CO-02,Colorado's 2nd congressional district,congressional_district,state/co,CO,Colorado,802,congressional_district/CO-02 +congressional_district/CO-03,Colorado's 3rd congressional district,congressional_district,state/co,CO,Colorado,803,congressional_district/CO-03 +congressional_district/CO-04,Colorado's 4th congressional district,congressional_district,state/co,CO,Colorado,804,congressional_district/CO-04 +congressional_district/CO-05,Colorado's 5th congressional district,congressional_district,state/co,CO,Colorado,805,congressional_district/CO-05 +congressional_district/CO-06,Colorado's 6th congressional district,congressional_district,state/co,CO,Colorado,806,congressional_district/CO-06 +congressional_district/CO-07,Colorado's 7th congressional district,congressional_district,state/co,CO,Colorado,807,congressional_district/CO-07 +congressional_district/CO-08,Colorado's 8th congressional district,congressional_district,state/co,CO,Colorado,808,congressional_district/CO-08 +congressional_district/CT-01,Connecticut's 1st congressional district,congressional_district,state/ct,CT,Connecticut,901,congressional_district/CT-01 +congressional_district/CT-02,Connecticut's 2nd congressional district,congressional_district,state/ct,CT,Connecticut,902,congressional_district/CT-02 +congressional_district/CT-03,Connecticut's 3rd congressional district,congressional_district,state/ct,CT,Connecticut,903,congressional_district/CT-03 +congressional_district/CT-04,Connecticut's 4th congressional district,congressional_district,state/ct,CT,Connecticut,904,congressional_district/CT-04 +congressional_district/CT-05,Connecticut's 5th congressional district,congressional_district,state/ct,CT,Connecticut,905,congressional_district/CT-05 +congressional_district/DC-01,District of Columbia's at-large congressional district,congressional_district,state/dc,DC,District of Columbia,1100,congressional_district/DC-01 +congressional_district/DE-01,Delaware's at-large congressional district,congressional_district,state/de,DE,Delaware,1000,congressional_district/DE-01 +congressional_district/FL-01,Florida's 1st congressional district,congressional_district,state/fl,FL,Florida,1201,congressional_district/FL-01 +congressional_district/FL-02,Florida's 2nd congressional district,congressional_district,state/fl,FL,Florida,1202,congressional_district/FL-02 +congressional_district/FL-03,Florida's 3rd congressional district,congressional_district,state/fl,FL,Florida,1203,congressional_district/FL-03 +congressional_district/FL-04,Florida's 4th congressional district,congressional_district,state/fl,FL,Florida,1204,congressional_district/FL-04 +congressional_district/FL-05,Florida's 5th congressional district,congressional_district,state/fl,FL,Florida,1205,congressional_district/FL-05 +congressional_district/FL-06,Florida's 6th congressional district,congressional_district,state/fl,FL,Florida,1206,congressional_district/FL-06 +congressional_district/FL-07,Florida's 7th congressional district,congressional_district,state/fl,FL,Florida,1207,congressional_district/FL-07 +congressional_district/FL-08,Florida's 8th congressional district,congressional_district,state/fl,FL,Florida,1208,congressional_district/FL-08 +congressional_district/FL-09,Florida's 9th congressional district,congressional_district,state/fl,FL,Florida,1209,congressional_district/FL-09 +congressional_district/FL-10,Florida's 10th congressional district,congressional_district,state/fl,FL,Florida,1210,congressional_district/FL-10 +congressional_district/FL-11,Florida's 11th congressional district,congressional_district,state/fl,FL,Florida,1211,congressional_district/FL-11 +congressional_district/FL-12,Florida's 12th congressional district,congressional_district,state/fl,FL,Florida,1212,congressional_district/FL-12 +congressional_district/FL-13,Florida's 13th congressional district,congressional_district,state/fl,FL,Florida,1213,congressional_district/FL-13 +congressional_district/FL-14,Florida's 14th congressional district,congressional_district,state/fl,FL,Florida,1214,congressional_district/FL-14 +congressional_district/FL-15,Florida's 15th congressional district,congressional_district,state/fl,FL,Florida,1215,congressional_district/FL-15 +congressional_district/FL-16,Florida's 16th congressional district,congressional_district,state/fl,FL,Florida,1216,congressional_district/FL-16 +congressional_district/FL-17,Florida's 17th congressional district,congressional_district,state/fl,FL,Florida,1217,congressional_district/FL-17 +congressional_district/FL-18,Florida's 18th congressional district,congressional_district,state/fl,FL,Florida,1218,congressional_district/FL-18 +congressional_district/FL-19,Florida's 19th congressional district,congressional_district,state/fl,FL,Florida,1219,congressional_district/FL-19 +congressional_district/FL-20,Florida's 20th congressional district,congressional_district,state/fl,FL,Florida,1220,congressional_district/FL-20 +congressional_district/FL-21,Florida's 21st congressional district,congressional_district,state/fl,FL,Florida,1221,congressional_district/FL-21 +congressional_district/FL-22,Florida's 22nd congressional district,congressional_district,state/fl,FL,Florida,1222,congressional_district/FL-22 +congressional_district/FL-23,Florida's 23rd congressional district,congressional_district,state/fl,FL,Florida,1223,congressional_district/FL-23 +congressional_district/FL-24,Florida's 24th congressional district,congressional_district,state/fl,FL,Florida,1224,congressional_district/FL-24 +congressional_district/FL-25,Florida's 25th congressional district,congressional_district,state/fl,FL,Florida,1225,congressional_district/FL-25 +congressional_district/FL-26,Florida's 26th congressional district,congressional_district,state/fl,FL,Florida,1226,congressional_district/FL-26 +congressional_district/FL-27,Florida's 27th congressional district,congressional_district,state/fl,FL,Florida,1227,congressional_district/FL-27 +congressional_district/FL-28,Florida's 28th congressional district,congressional_district,state/fl,FL,Florida,1228,congressional_district/FL-28 +congressional_district/GA-01,Georgia's 1st congressional district,congressional_district,state/ga,GA,Georgia,1301,congressional_district/GA-01 +congressional_district/GA-02,Georgia's 2nd congressional district,congressional_district,state/ga,GA,Georgia,1302,congressional_district/GA-02 +congressional_district/GA-03,Georgia's 3rd congressional district,congressional_district,state/ga,GA,Georgia,1303,congressional_district/GA-03 +congressional_district/GA-04,Georgia's 4th congressional district,congressional_district,state/ga,GA,Georgia,1304,congressional_district/GA-04 +congressional_district/GA-05,Georgia's 5th congressional district,congressional_district,state/ga,GA,Georgia,1305,congressional_district/GA-05 +congressional_district/GA-06,Georgia's 6th congressional district,congressional_district,state/ga,GA,Georgia,1306,congressional_district/GA-06 +congressional_district/GA-07,Georgia's 7th congressional district,congressional_district,state/ga,GA,Georgia,1307,congressional_district/GA-07 +congressional_district/GA-08,Georgia's 8th congressional district,congressional_district,state/ga,GA,Georgia,1308,congressional_district/GA-08 +congressional_district/GA-09,Georgia's 9th congressional district,congressional_district,state/ga,GA,Georgia,1309,congressional_district/GA-09 +congressional_district/GA-10,Georgia's 10th congressional district,congressional_district,state/ga,GA,Georgia,1310,congressional_district/GA-10 +congressional_district/GA-11,Georgia's 11th congressional district,congressional_district,state/ga,GA,Georgia,1311,congressional_district/GA-11 +congressional_district/GA-12,Georgia's 12th congressional district,congressional_district,state/ga,GA,Georgia,1312,congressional_district/GA-12 +congressional_district/GA-13,Georgia's 13th congressional district,congressional_district,state/ga,GA,Georgia,1313,congressional_district/GA-13 +congressional_district/GA-14,Georgia's 14th congressional district,congressional_district,state/ga,GA,Georgia,1314,congressional_district/GA-14 +congressional_district/HI-01,Hawaii's 1st congressional district,congressional_district,state/hi,HI,Hawaii,1501,congressional_district/HI-01 +congressional_district/HI-02,Hawaii's 2nd congressional district,congressional_district,state/hi,HI,Hawaii,1502,congressional_district/HI-02 +congressional_district/IA-01,Iowa's 1st congressional district,congressional_district,state/ia,IA,Iowa,1901,congressional_district/IA-01 +congressional_district/IA-02,Iowa's 2nd congressional district,congressional_district,state/ia,IA,Iowa,1902,congressional_district/IA-02 +congressional_district/IA-03,Iowa's 3rd congressional district,congressional_district,state/ia,IA,Iowa,1903,congressional_district/IA-03 +congressional_district/IA-04,Iowa's 4th congressional district,congressional_district,state/ia,IA,Iowa,1904,congressional_district/IA-04 +congressional_district/ID-01,Idaho's 1st congressional district,congressional_district,state/id,ID,Idaho,1601,congressional_district/ID-01 +congressional_district/ID-02,Idaho's 2nd congressional district,congressional_district,state/id,ID,Idaho,1602,congressional_district/ID-02 +congressional_district/IL-01,Illinois's 1st congressional district,congressional_district,state/il,IL,Illinois,1701,congressional_district/IL-01 +congressional_district/IL-02,Illinois's 2nd congressional district,congressional_district,state/il,IL,Illinois,1702,congressional_district/IL-02 +congressional_district/IL-03,Illinois's 3rd congressional district,congressional_district,state/il,IL,Illinois,1703,congressional_district/IL-03 +congressional_district/IL-04,Illinois's 4th congressional district,congressional_district,state/il,IL,Illinois,1704,congressional_district/IL-04 +congressional_district/IL-05,Illinois's 5th congressional district,congressional_district,state/il,IL,Illinois,1705,congressional_district/IL-05 +congressional_district/IL-06,Illinois's 6th congressional district,congressional_district,state/il,IL,Illinois,1706,congressional_district/IL-06 +congressional_district/IL-07,Illinois's 7th congressional district,congressional_district,state/il,IL,Illinois,1707,congressional_district/IL-07 +congressional_district/IL-08,Illinois's 8th congressional district,congressional_district,state/il,IL,Illinois,1708,congressional_district/IL-08 +congressional_district/IL-09,Illinois's 9th congressional district,congressional_district,state/il,IL,Illinois,1709,congressional_district/IL-09 +congressional_district/IL-10,Illinois's 10th congressional district,congressional_district,state/il,IL,Illinois,1710,congressional_district/IL-10 +congressional_district/IL-11,Illinois's 11th congressional district,congressional_district,state/il,IL,Illinois,1711,congressional_district/IL-11 +congressional_district/IL-12,Illinois's 12th congressional district,congressional_district,state/il,IL,Illinois,1712,congressional_district/IL-12 +congressional_district/IL-13,Illinois's 13th congressional district,congressional_district,state/il,IL,Illinois,1713,congressional_district/IL-13 +congressional_district/IL-14,Illinois's 14th congressional district,congressional_district,state/il,IL,Illinois,1714,congressional_district/IL-14 +congressional_district/IL-15,Illinois's 15th congressional district,congressional_district,state/il,IL,Illinois,1715,congressional_district/IL-15 +congressional_district/IL-16,Illinois's 16th congressional district,congressional_district,state/il,IL,Illinois,1716,congressional_district/IL-16 +congressional_district/IL-17,Illinois's 17th congressional district,congressional_district,state/il,IL,Illinois,1717,congressional_district/IL-17 +congressional_district/IN-01,Indiana's 1st congressional district,congressional_district,state/in,IN,Indiana,1801,congressional_district/IN-01 +congressional_district/IN-02,Indiana's 2nd congressional district,congressional_district,state/in,IN,Indiana,1802,congressional_district/IN-02 +congressional_district/IN-03,Indiana's 3rd congressional district,congressional_district,state/in,IN,Indiana,1803,congressional_district/IN-03 +congressional_district/IN-04,Indiana's 4th congressional district,congressional_district,state/in,IN,Indiana,1804,congressional_district/IN-04 +congressional_district/IN-05,Indiana's 5th congressional district,congressional_district,state/in,IN,Indiana,1805,congressional_district/IN-05 +congressional_district/IN-06,Indiana's 6th congressional district,congressional_district,state/in,IN,Indiana,1806,congressional_district/IN-06 +congressional_district/IN-07,Indiana's 7th congressional district,congressional_district,state/in,IN,Indiana,1807,congressional_district/IN-07 +congressional_district/IN-08,Indiana's 8th congressional district,congressional_district,state/in,IN,Indiana,1808,congressional_district/IN-08 +congressional_district/IN-09,Indiana's 9th congressional district,congressional_district,state/in,IN,Indiana,1809,congressional_district/IN-09 +congressional_district/KS-01,Kansas's 1st congressional district,congressional_district,state/ks,KS,Kansas,2001,congressional_district/KS-01 +congressional_district/KS-02,Kansas's 2nd congressional district,congressional_district,state/ks,KS,Kansas,2002,congressional_district/KS-02 +congressional_district/KS-03,Kansas's 3rd congressional district,congressional_district,state/ks,KS,Kansas,2003,congressional_district/KS-03 +congressional_district/KS-04,Kansas's 4th congressional district,congressional_district,state/ks,KS,Kansas,2004,congressional_district/KS-04 +congressional_district/KY-01,Kentucky's 1st congressional district,congressional_district,state/ky,KY,Kentucky,2101,congressional_district/KY-01 +congressional_district/KY-02,Kentucky's 2nd congressional district,congressional_district,state/ky,KY,Kentucky,2102,congressional_district/KY-02 +congressional_district/KY-03,Kentucky's 3rd congressional district,congressional_district,state/ky,KY,Kentucky,2103,congressional_district/KY-03 +congressional_district/KY-04,Kentucky's 4th congressional district,congressional_district,state/ky,KY,Kentucky,2104,congressional_district/KY-04 +congressional_district/KY-05,Kentucky's 5th congressional district,congressional_district,state/ky,KY,Kentucky,2105,congressional_district/KY-05 +congressional_district/KY-06,Kentucky's 6th congressional district,congressional_district,state/ky,KY,Kentucky,2106,congressional_district/KY-06 +congressional_district/LA-01,Louisiana's 1st congressional district,congressional_district,state/la,LA,Louisiana,2201,congressional_district/LA-01 +congressional_district/LA-02,Louisiana's 2nd congressional district,congressional_district,state/la,LA,Louisiana,2202,congressional_district/LA-02 +congressional_district/LA-03,Louisiana's 3rd congressional district,congressional_district,state/la,LA,Louisiana,2203,congressional_district/LA-03 +congressional_district/LA-04,Louisiana's 4th congressional district,congressional_district,state/la,LA,Louisiana,2204,congressional_district/LA-04 +congressional_district/LA-05,Louisiana's 5th congressional district,congressional_district,state/la,LA,Louisiana,2205,congressional_district/LA-05 +congressional_district/LA-06,Louisiana's 6th congressional district,congressional_district,state/la,LA,Louisiana,2206,congressional_district/LA-06 +congressional_district/MA-01,Massachusetts's 1st congressional district,congressional_district,state/ma,MA,Massachusetts,2501,congressional_district/MA-01 +congressional_district/MA-02,Massachusetts's 2nd congressional district,congressional_district,state/ma,MA,Massachusetts,2502,congressional_district/MA-02 +congressional_district/MA-03,Massachusetts's 3rd congressional district,congressional_district,state/ma,MA,Massachusetts,2503,congressional_district/MA-03 +congressional_district/MA-04,Massachusetts's 4th congressional district,congressional_district,state/ma,MA,Massachusetts,2504,congressional_district/MA-04 +congressional_district/MA-05,Massachusetts's 5th congressional district,congressional_district,state/ma,MA,Massachusetts,2505,congressional_district/MA-05 +congressional_district/MA-06,Massachusetts's 6th congressional district,congressional_district,state/ma,MA,Massachusetts,2506,congressional_district/MA-06 +congressional_district/MA-07,Massachusetts's 7th congressional district,congressional_district,state/ma,MA,Massachusetts,2507,congressional_district/MA-07 +congressional_district/MA-08,Massachusetts's 8th congressional district,congressional_district,state/ma,MA,Massachusetts,2508,congressional_district/MA-08 +congressional_district/MA-09,Massachusetts's 9th congressional district,congressional_district,state/ma,MA,Massachusetts,2509,congressional_district/MA-09 +congressional_district/MD-01,Maryland's 1st congressional district,congressional_district,state/md,MD,Maryland,2401,congressional_district/MD-01 +congressional_district/MD-02,Maryland's 2nd congressional district,congressional_district,state/md,MD,Maryland,2402,congressional_district/MD-02 +congressional_district/MD-03,Maryland's 3rd congressional district,congressional_district,state/md,MD,Maryland,2403,congressional_district/MD-03 +congressional_district/MD-04,Maryland's 4th congressional district,congressional_district,state/md,MD,Maryland,2404,congressional_district/MD-04 +congressional_district/MD-05,Maryland's 5th congressional district,congressional_district,state/md,MD,Maryland,2405,congressional_district/MD-05 +congressional_district/MD-06,Maryland's 6th congressional district,congressional_district,state/md,MD,Maryland,2406,congressional_district/MD-06 +congressional_district/MD-07,Maryland's 7th congressional district,congressional_district,state/md,MD,Maryland,2407,congressional_district/MD-07 +congressional_district/MD-08,Maryland's 8th congressional district,congressional_district,state/md,MD,Maryland,2408,congressional_district/MD-08 +congressional_district/ME-01,Maine's 1st congressional district,congressional_district,state/me,ME,Maine,2301,congressional_district/ME-01 +congressional_district/ME-02,Maine's 2nd congressional district,congressional_district,state/me,ME,Maine,2302,congressional_district/ME-02 +congressional_district/MI-01,Michigan's 1st congressional district,congressional_district,state/mi,MI,Michigan,2601,congressional_district/MI-01 +congressional_district/MI-02,Michigan's 2nd congressional district,congressional_district,state/mi,MI,Michigan,2602,congressional_district/MI-02 +congressional_district/MI-03,Michigan's 3rd congressional district,congressional_district,state/mi,MI,Michigan,2603,congressional_district/MI-03 +congressional_district/MI-04,Michigan's 4th congressional district,congressional_district,state/mi,MI,Michigan,2604,congressional_district/MI-04 +congressional_district/MI-05,Michigan's 5th congressional district,congressional_district,state/mi,MI,Michigan,2605,congressional_district/MI-05 +congressional_district/MI-06,Michigan's 6th congressional district,congressional_district,state/mi,MI,Michigan,2606,congressional_district/MI-06 +congressional_district/MI-07,Michigan's 7th congressional district,congressional_district,state/mi,MI,Michigan,2607,congressional_district/MI-07 +congressional_district/MI-08,Michigan's 8th congressional district,congressional_district,state/mi,MI,Michigan,2608,congressional_district/MI-08 +congressional_district/MI-09,Michigan's 9th congressional district,congressional_district,state/mi,MI,Michigan,2609,congressional_district/MI-09 +congressional_district/MI-10,Michigan's 10th congressional district,congressional_district,state/mi,MI,Michigan,2610,congressional_district/MI-10 +congressional_district/MI-11,Michigan's 11th congressional district,congressional_district,state/mi,MI,Michigan,2611,congressional_district/MI-11 +congressional_district/MI-12,Michigan's 12th congressional district,congressional_district,state/mi,MI,Michigan,2612,congressional_district/MI-12 +congressional_district/MI-13,Michigan's 13th congressional district,congressional_district,state/mi,MI,Michigan,2613,congressional_district/MI-13 +congressional_district/MN-01,Minnesota's 1st congressional district,congressional_district,state/mn,MN,Minnesota,2701,congressional_district/MN-01 +congressional_district/MN-02,Minnesota's 2nd congressional district,congressional_district,state/mn,MN,Minnesota,2702,congressional_district/MN-02 +congressional_district/MN-03,Minnesota's 3rd congressional district,congressional_district,state/mn,MN,Minnesota,2703,congressional_district/MN-03 +congressional_district/MN-04,Minnesota's 4th congressional district,congressional_district,state/mn,MN,Minnesota,2704,congressional_district/MN-04 +congressional_district/MN-05,Minnesota's 5th congressional district,congressional_district,state/mn,MN,Minnesota,2705,congressional_district/MN-05 +congressional_district/MN-06,Minnesota's 6th congressional district,congressional_district,state/mn,MN,Minnesota,2706,congressional_district/MN-06 +congressional_district/MN-07,Minnesota's 7th congressional district,congressional_district,state/mn,MN,Minnesota,2707,congressional_district/MN-07 +congressional_district/MN-08,Minnesota's 8th congressional district,congressional_district,state/mn,MN,Minnesota,2708,congressional_district/MN-08 +congressional_district/MO-01,Missouri's 1st congressional district,congressional_district,state/mo,MO,Missouri,2901,congressional_district/MO-01 +congressional_district/MO-02,Missouri's 2nd congressional district,congressional_district,state/mo,MO,Missouri,2902,congressional_district/MO-02 +congressional_district/MO-03,Missouri's 3rd congressional district,congressional_district,state/mo,MO,Missouri,2903,congressional_district/MO-03 +congressional_district/MO-04,Missouri's 4th congressional district,congressional_district,state/mo,MO,Missouri,2904,congressional_district/MO-04 +congressional_district/MO-05,Missouri's 5th congressional district,congressional_district,state/mo,MO,Missouri,2905,congressional_district/MO-05 +congressional_district/MO-06,Missouri's 6th congressional district,congressional_district,state/mo,MO,Missouri,2906,congressional_district/MO-06 +congressional_district/MO-07,Missouri's 7th congressional district,congressional_district,state/mo,MO,Missouri,2907,congressional_district/MO-07 +congressional_district/MO-08,Missouri's 8th congressional district,congressional_district,state/mo,MO,Missouri,2908,congressional_district/MO-08 +congressional_district/MS-01,Mississippi's 1st congressional district,congressional_district,state/ms,MS,Mississippi,2801,congressional_district/MS-01 +congressional_district/MS-02,Mississippi's 2nd congressional district,congressional_district,state/ms,MS,Mississippi,2802,congressional_district/MS-02 +congressional_district/MS-03,Mississippi's 3rd congressional district,congressional_district,state/ms,MS,Mississippi,2803,congressional_district/MS-03 +congressional_district/MS-04,Mississippi's 4th congressional district,congressional_district,state/ms,MS,Mississippi,2804,congressional_district/MS-04 +congressional_district/MT-01,Montana's 1st congressional district,congressional_district,state/mt,MT,Montana,3001,congressional_district/MT-01 +congressional_district/MT-02,Montana's 2nd congressional district,congressional_district,state/mt,MT,Montana,3002,congressional_district/MT-02 +congressional_district/NC-01,North Carolina's 1st congressional district,congressional_district,state/nc,NC,North Carolina,3701,congressional_district/NC-01 +congressional_district/NC-02,North Carolina's 2nd congressional district,congressional_district,state/nc,NC,North Carolina,3702,congressional_district/NC-02 +congressional_district/NC-03,North Carolina's 3rd congressional district,congressional_district,state/nc,NC,North Carolina,3703,congressional_district/NC-03 +congressional_district/NC-04,North Carolina's 4th congressional district,congressional_district,state/nc,NC,North Carolina,3704,congressional_district/NC-04 +congressional_district/NC-05,North Carolina's 5th congressional district,congressional_district,state/nc,NC,North Carolina,3705,congressional_district/NC-05 +congressional_district/NC-06,North Carolina's 6th congressional district,congressional_district,state/nc,NC,North Carolina,3706,congressional_district/NC-06 +congressional_district/NC-07,North Carolina's 7th congressional district,congressional_district,state/nc,NC,North Carolina,3707,congressional_district/NC-07 +congressional_district/NC-08,North Carolina's 8th congressional district,congressional_district,state/nc,NC,North Carolina,3708,congressional_district/NC-08 +congressional_district/NC-09,North Carolina's 9th congressional district,congressional_district,state/nc,NC,North Carolina,3709,congressional_district/NC-09 +congressional_district/NC-10,North Carolina's 10th congressional district,congressional_district,state/nc,NC,North Carolina,3710,congressional_district/NC-10 +congressional_district/NC-11,North Carolina's 11th congressional district,congressional_district,state/nc,NC,North Carolina,3711,congressional_district/NC-11 +congressional_district/NC-12,North Carolina's 12th congressional district,congressional_district,state/nc,NC,North Carolina,3712,congressional_district/NC-12 +congressional_district/NC-13,North Carolina's 13th congressional district,congressional_district,state/nc,NC,North Carolina,3713,congressional_district/NC-13 +congressional_district/NC-14,North Carolina's 14th congressional district,congressional_district,state/nc,NC,North Carolina,3714,congressional_district/NC-14 +congressional_district/ND-01,North Dakota's at-large congressional district,congressional_district,state/nd,ND,North Dakota,3800,congressional_district/ND-01 +congressional_district/NE-01,Nebraska's 1st congressional district,congressional_district,state/ne,NE,Nebraska,3101,congressional_district/NE-01 +congressional_district/NE-02,Nebraska's 2nd congressional district,congressional_district,state/ne,NE,Nebraska,3102,congressional_district/NE-02 +congressional_district/NE-03,Nebraska's 3rd congressional district,congressional_district,state/ne,NE,Nebraska,3103,congressional_district/NE-03 +congressional_district/NH-01,New Hampshire's 1st congressional district,congressional_district,state/nh,NH,New Hampshire,3301,congressional_district/NH-01 +congressional_district/NH-02,New Hampshire's 2nd congressional district,congressional_district,state/nh,NH,New Hampshire,3302,congressional_district/NH-02 +congressional_district/NJ-01,New Jersey's 1st congressional district,congressional_district,state/nj,NJ,New Jersey,3401,congressional_district/NJ-01 +congressional_district/NJ-02,New Jersey's 2nd congressional district,congressional_district,state/nj,NJ,New Jersey,3402,congressional_district/NJ-02 +congressional_district/NJ-03,New Jersey's 3rd congressional district,congressional_district,state/nj,NJ,New Jersey,3403,congressional_district/NJ-03 +congressional_district/NJ-04,New Jersey's 4th congressional district,congressional_district,state/nj,NJ,New Jersey,3404,congressional_district/NJ-04 +congressional_district/NJ-05,New Jersey's 5th congressional district,congressional_district,state/nj,NJ,New Jersey,3405,congressional_district/NJ-05 +congressional_district/NJ-06,New Jersey's 6th congressional district,congressional_district,state/nj,NJ,New Jersey,3406,congressional_district/NJ-06 +congressional_district/NJ-07,New Jersey's 7th congressional district,congressional_district,state/nj,NJ,New Jersey,3407,congressional_district/NJ-07 +congressional_district/NJ-08,New Jersey's 8th congressional district,congressional_district,state/nj,NJ,New Jersey,3408,congressional_district/NJ-08 +congressional_district/NJ-09,New Jersey's 9th congressional district,congressional_district,state/nj,NJ,New Jersey,3409,congressional_district/NJ-09 +congressional_district/NJ-10,New Jersey's 10th congressional district,congressional_district,state/nj,NJ,New Jersey,3410,congressional_district/NJ-10 +congressional_district/NJ-11,New Jersey's 11th congressional district,congressional_district,state/nj,NJ,New Jersey,3411,congressional_district/NJ-11 +congressional_district/NJ-12,New Jersey's 12th congressional district,congressional_district,state/nj,NJ,New Jersey,3412,congressional_district/NJ-12 +congressional_district/NM-01,New Mexico's 1st congressional district,congressional_district,state/nm,NM,New Mexico,3501,congressional_district/NM-01 +congressional_district/NM-02,New Mexico's 2nd congressional district,congressional_district,state/nm,NM,New Mexico,3502,congressional_district/NM-02 +congressional_district/NM-03,New Mexico's 3rd congressional district,congressional_district,state/nm,NM,New Mexico,3503,congressional_district/NM-03 +congressional_district/NV-01,Nevada's 1st congressional district,congressional_district,state/nv,NV,Nevada,3201,congressional_district/NV-01 +congressional_district/NV-02,Nevada's 2nd congressional district,congressional_district,state/nv,NV,Nevada,3202,congressional_district/NV-02 +congressional_district/NV-03,Nevada's 3rd congressional district,congressional_district,state/nv,NV,Nevada,3203,congressional_district/NV-03 +congressional_district/NV-04,Nevada's 4th congressional district,congressional_district,state/nv,NV,Nevada,3204,congressional_district/NV-04 +congressional_district/NY-01,New York's 1st congressional district,congressional_district,state/ny,NY,New York,3601,congressional_district/NY-01 +congressional_district/NY-02,New York's 2nd congressional district,congressional_district,state/ny,NY,New York,3602,congressional_district/NY-02 +congressional_district/NY-03,New York's 3rd congressional district,congressional_district,state/ny,NY,New York,3603,congressional_district/NY-03 +congressional_district/NY-04,New York's 4th congressional district,congressional_district,state/ny,NY,New York,3604,congressional_district/NY-04 +congressional_district/NY-05,New York's 5th congressional district,congressional_district,state/ny,NY,New York,3605,congressional_district/NY-05 +congressional_district/NY-06,New York's 6th congressional district,congressional_district,state/ny,NY,New York,3606,congressional_district/NY-06 +congressional_district/NY-07,New York's 7th congressional district,congressional_district,state/ny,NY,New York,3607,congressional_district/NY-07 +congressional_district/NY-08,New York's 8th congressional district,congressional_district,state/ny,NY,New York,3608,congressional_district/NY-08 +congressional_district/NY-09,New York's 9th congressional district,congressional_district,state/ny,NY,New York,3609,congressional_district/NY-09 +congressional_district/NY-10,New York's 10th congressional district,congressional_district,state/ny,NY,New York,3610,congressional_district/NY-10 +congressional_district/NY-11,New York's 11th congressional district,congressional_district,state/ny,NY,New York,3611,congressional_district/NY-11 +congressional_district/NY-12,New York's 12th congressional district,congressional_district,state/ny,NY,New York,3612,congressional_district/NY-12 +congressional_district/NY-13,New York's 13th congressional district,congressional_district,state/ny,NY,New York,3613,congressional_district/NY-13 +congressional_district/NY-14,New York's 14th congressional district,congressional_district,state/ny,NY,New York,3614,congressional_district/NY-14 +congressional_district/NY-15,New York's 15th congressional district,congressional_district,state/ny,NY,New York,3615,congressional_district/NY-15 +congressional_district/NY-16,New York's 16th congressional district,congressional_district,state/ny,NY,New York,3616,congressional_district/NY-16 +congressional_district/NY-17,New York's 17th congressional district,congressional_district,state/ny,NY,New York,3617,congressional_district/NY-17 +congressional_district/NY-18,New York's 18th congressional district,congressional_district,state/ny,NY,New York,3618,congressional_district/NY-18 +congressional_district/NY-19,New York's 19th congressional district,congressional_district,state/ny,NY,New York,3619,congressional_district/NY-19 +congressional_district/NY-20,New York's 20th congressional district,congressional_district,state/ny,NY,New York,3620,congressional_district/NY-20 +congressional_district/NY-21,New York's 21st congressional district,congressional_district,state/ny,NY,New York,3621,congressional_district/NY-21 +congressional_district/NY-22,New York's 22nd congressional district,congressional_district,state/ny,NY,New York,3622,congressional_district/NY-22 +congressional_district/NY-23,New York's 23rd congressional district,congressional_district,state/ny,NY,New York,3623,congressional_district/NY-23 +congressional_district/NY-24,New York's 24th congressional district,congressional_district,state/ny,NY,New York,3624,congressional_district/NY-24 +congressional_district/NY-25,New York's 25th congressional district,congressional_district,state/ny,NY,New York,3625,congressional_district/NY-25 +congressional_district/NY-26,New York's 26th congressional district,congressional_district,state/ny,NY,New York,3626,congressional_district/NY-26 +congressional_district/OH-01,Ohio's 1st congressional district,congressional_district,state/oh,OH,Ohio,3901,congressional_district/OH-01 +congressional_district/OH-02,Ohio's 2nd congressional district,congressional_district,state/oh,OH,Ohio,3902,congressional_district/OH-02 +congressional_district/OH-03,Ohio's 3rd congressional district,congressional_district,state/oh,OH,Ohio,3903,congressional_district/OH-03 +congressional_district/OH-04,Ohio's 4th congressional district,congressional_district,state/oh,OH,Ohio,3904,congressional_district/OH-04 +congressional_district/OH-05,Ohio's 5th congressional district,congressional_district,state/oh,OH,Ohio,3905,congressional_district/OH-05 +congressional_district/OH-06,Ohio's 6th congressional district,congressional_district,state/oh,OH,Ohio,3906,congressional_district/OH-06 +congressional_district/OH-07,Ohio's 7th congressional district,congressional_district,state/oh,OH,Ohio,3907,congressional_district/OH-07 +congressional_district/OH-08,Ohio's 8th congressional district,congressional_district,state/oh,OH,Ohio,3908,congressional_district/OH-08 +congressional_district/OH-09,Ohio's 9th congressional district,congressional_district,state/oh,OH,Ohio,3909,congressional_district/OH-09 +congressional_district/OH-10,Ohio's 10th congressional district,congressional_district,state/oh,OH,Ohio,3910,congressional_district/OH-10 +congressional_district/OH-11,Ohio's 11th congressional district,congressional_district,state/oh,OH,Ohio,3911,congressional_district/OH-11 +congressional_district/OH-12,Ohio's 12th congressional district,congressional_district,state/oh,OH,Ohio,3912,congressional_district/OH-12 +congressional_district/OH-13,Ohio's 13th congressional district,congressional_district,state/oh,OH,Ohio,3913,congressional_district/OH-13 +congressional_district/OH-14,Ohio's 14th congressional district,congressional_district,state/oh,OH,Ohio,3914,congressional_district/OH-14 +congressional_district/OH-15,Ohio's 15th congressional district,congressional_district,state/oh,OH,Ohio,3915,congressional_district/OH-15 +congressional_district/OK-01,Oklahoma's 1st congressional district,congressional_district,state/ok,OK,Oklahoma,4001,congressional_district/OK-01 +congressional_district/OK-02,Oklahoma's 2nd congressional district,congressional_district,state/ok,OK,Oklahoma,4002,congressional_district/OK-02 +congressional_district/OK-03,Oklahoma's 3rd congressional district,congressional_district,state/ok,OK,Oklahoma,4003,congressional_district/OK-03 +congressional_district/OK-04,Oklahoma's 4th congressional district,congressional_district,state/ok,OK,Oklahoma,4004,congressional_district/OK-04 +congressional_district/OK-05,Oklahoma's 5th congressional district,congressional_district,state/ok,OK,Oklahoma,4005,congressional_district/OK-05 +congressional_district/OR-01,Oregon's 1st congressional district,congressional_district,state/or,OR,Oregon,4101,congressional_district/OR-01 +congressional_district/OR-02,Oregon's 2nd congressional district,congressional_district,state/or,OR,Oregon,4102,congressional_district/OR-02 +congressional_district/OR-03,Oregon's 3rd congressional district,congressional_district,state/or,OR,Oregon,4103,congressional_district/OR-03 +congressional_district/OR-04,Oregon's 4th congressional district,congressional_district,state/or,OR,Oregon,4104,congressional_district/OR-04 +congressional_district/OR-05,Oregon's 5th congressional district,congressional_district,state/or,OR,Oregon,4105,congressional_district/OR-05 +congressional_district/OR-06,Oregon's 6th congressional district,congressional_district,state/or,OR,Oregon,4106,congressional_district/OR-06 +congressional_district/PA-01,Pennsylvania's 1st congressional district,congressional_district,state/pa,PA,Pennsylvania,4201,congressional_district/PA-01 +congressional_district/PA-02,Pennsylvania's 2nd congressional district,congressional_district,state/pa,PA,Pennsylvania,4202,congressional_district/PA-02 +congressional_district/PA-03,Pennsylvania's 3rd congressional district,congressional_district,state/pa,PA,Pennsylvania,4203,congressional_district/PA-03 +congressional_district/PA-04,Pennsylvania's 4th congressional district,congressional_district,state/pa,PA,Pennsylvania,4204,congressional_district/PA-04 +congressional_district/PA-05,Pennsylvania's 5th congressional district,congressional_district,state/pa,PA,Pennsylvania,4205,congressional_district/PA-05 +congressional_district/PA-06,Pennsylvania's 6th congressional district,congressional_district,state/pa,PA,Pennsylvania,4206,congressional_district/PA-06 +congressional_district/PA-07,Pennsylvania's 7th congressional district,congressional_district,state/pa,PA,Pennsylvania,4207,congressional_district/PA-07 +congressional_district/PA-08,Pennsylvania's 8th congressional district,congressional_district,state/pa,PA,Pennsylvania,4208,congressional_district/PA-08 +congressional_district/PA-09,Pennsylvania's 9th congressional district,congressional_district,state/pa,PA,Pennsylvania,4209,congressional_district/PA-09 +congressional_district/PA-10,Pennsylvania's 10th congressional district,congressional_district,state/pa,PA,Pennsylvania,4210,congressional_district/PA-10 +congressional_district/PA-11,Pennsylvania's 11th congressional district,congressional_district,state/pa,PA,Pennsylvania,4211,congressional_district/PA-11 +congressional_district/PA-12,Pennsylvania's 12th congressional district,congressional_district,state/pa,PA,Pennsylvania,4212,congressional_district/PA-12 +congressional_district/PA-13,Pennsylvania's 13th congressional district,congressional_district,state/pa,PA,Pennsylvania,4213,congressional_district/PA-13 +congressional_district/PA-14,Pennsylvania's 14th congressional district,congressional_district,state/pa,PA,Pennsylvania,4214,congressional_district/PA-14 +congressional_district/PA-15,Pennsylvania's 15th congressional district,congressional_district,state/pa,PA,Pennsylvania,4215,congressional_district/PA-15 +congressional_district/PA-16,Pennsylvania's 16th congressional district,congressional_district,state/pa,PA,Pennsylvania,4216,congressional_district/PA-16 +congressional_district/PA-17,Pennsylvania's 17th congressional district,congressional_district,state/pa,PA,Pennsylvania,4217,congressional_district/PA-17 +congressional_district/RI-01,Rhode Island's 1st congressional district,congressional_district,state/ri,RI,Rhode Island,4401,congressional_district/RI-01 +congressional_district/RI-02,Rhode Island's 2nd congressional district,congressional_district,state/ri,RI,Rhode Island,4402,congressional_district/RI-02 +congressional_district/SC-01,South Carolina's 1st congressional district,congressional_district,state/sc,SC,South Carolina,4501,congressional_district/SC-01 +congressional_district/SC-02,South Carolina's 2nd congressional district,congressional_district,state/sc,SC,South Carolina,4502,congressional_district/SC-02 +congressional_district/SC-03,South Carolina's 3rd congressional district,congressional_district,state/sc,SC,South Carolina,4503,congressional_district/SC-03 +congressional_district/SC-04,South Carolina's 4th congressional district,congressional_district,state/sc,SC,South Carolina,4504,congressional_district/SC-04 +congressional_district/SC-05,South Carolina's 5th congressional district,congressional_district,state/sc,SC,South Carolina,4505,congressional_district/SC-05 +congressional_district/SC-06,South Carolina's 6th congressional district,congressional_district,state/sc,SC,South Carolina,4506,congressional_district/SC-06 +congressional_district/SC-07,South Carolina's 7th congressional district,congressional_district,state/sc,SC,South Carolina,4507,congressional_district/SC-07 +congressional_district/SD-01,South Dakota's at-large congressional district,congressional_district,state/sd,SD,South Dakota,4600,congressional_district/SD-01 +congressional_district/TN-01,Tennessee's 1st congressional district,congressional_district,state/tn,TN,Tennessee,4701,congressional_district/TN-01 +congressional_district/TN-02,Tennessee's 2nd congressional district,congressional_district,state/tn,TN,Tennessee,4702,congressional_district/TN-02 +congressional_district/TN-03,Tennessee's 3rd congressional district,congressional_district,state/tn,TN,Tennessee,4703,congressional_district/TN-03 +congressional_district/TN-04,Tennessee's 4th congressional district,congressional_district,state/tn,TN,Tennessee,4704,congressional_district/TN-04 +congressional_district/TN-05,Tennessee's 5th congressional district,congressional_district,state/tn,TN,Tennessee,4705,congressional_district/TN-05 +congressional_district/TN-06,Tennessee's 6th congressional district,congressional_district,state/tn,TN,Tennessee,4706,congressional_district/TN-06 +congressional_district/TN-07,Tennessee's 7th congressional district,congressional_district,state/tn,TN,Tennessee,4707,congressional_district/TN-07 +congressional_district/TN-08,Tennessee's 8th congressional district,congressional_district,state/tn,TN,Tennessee,4708,congressional_district/TN-08 +congressional_district/TN-09,Tennessee's 9th congressional district,congressional_district,state/tn,TN,Tennessee,4709,congressional_district/TN-09 +congressional_district/TX-01,Texas's 1st congressional district,congressional_district,state/tx,TX,Texas,4801,congressional_district/TX-01 +congressional_district/TX-02,Texas's 2nd congressional district,congressional_district,state/tx,TX,Texas,4802,congressional_district/TX-02 +congressional_district/TX-03,Texas's 3rd congressional district,congressional_district,state/tx,TX,Texas,4803,congressional_district/TX-03 +congressional_district/TX-04,Texas's 4th congressional district,congressional_district,state/tx,TX,Texas,4804,congressional_district/TX-04 +congressional_district/TX-05,Texas's 5th congressional district,congressional_district,state/tx,TX,Texas,4805,congressional_district/TX-05 +congressional_district/TX-06,Texas's 6th congressional district,congressional_district,state/tx,TX,Texas,4806,congressional_district/TX-06 +congressional_district/TX-07,Texas's 7th congressional district,congressional_district,state/tx,TX,Texas,4807,congressional_district/TX-07 +congressional_district/TX-08,Texas's 8th congressional district,congressional_district,state/tx,TX,Texas,4808,congressional_district/TX-08 +congressional_district/TX-09,Texas's 9th congressional district,congressional_district,state/tx,TX,Texas,4809,congressional_district/TX-09 +congressional_district/TX-10,Texas's 10th congressional district,congressional_district,state/tx,TX,Texas,4810,congressional_district/TX-10 +congressional_district/TX-11,Texas's 11th congressional district,congressional_district,state/tx,TX,Texas,4811,congressional_district/TX-11 +congressional_district/TX-12,Texas's 12th congressional district,congressional_district,state/tx,TX,Texas,4812,congressional_district/TX-12 +congressional_district/TX-13,Texas's 13th congressional district,congressional_district,state/tx,TX,Texas,4813,congressional_district/TX-13 +congressional_district/TX-14,Texas's 14th congressional district,congressional_district,state/tx,TX,Texas,4814,congressional_district/TX-14 +congressional_district/TX-15,Texas's 15th congressional district,congressional_district,state/tx,TX,Texas,4815,congressional_district/TX-15 +congressional_district/TX-16,Texas's 16th congressional district,congressional_district,state/tx,TX,Texas,4816,congressional_district/TX-16 +congressional_district/TX-17,Texas's 17th congressional district,congressional_district,state/tx,TX,Texas,4817,congressional_district/TX-17 +congressional_district/TX-18,Texas's 18th congressional district,congressional_district,state/tx,TX,Texas,4818,congressional_district/TX-18 +congressional_district/TX-19,Texas's 19th congressional district,congressional_district,state/tx,TX,Texas,4819,congressional_district/TX-19 +congressional_district/TX-20,Texas's 20th congressional district,congressional_district,state/tx,TX,Texas,4820,congressional_district/TX-20 +congressional_district/TX-21,Texas's 21st congressional district,congressional_district,state/tx,TX,Texas,4821,congressional_district/TX-21 +congressional_district/TX-22,Texas's 22nd congressional district,congressional_district,state/tx,TX,Texas,4822,congressional_district/TX-22 +congressional_district/TX-23,Texas's 23rd congressional district,congressional_district,state/tx,TX,Texas,4823,congressional_district/TX-23 +congressional_district/TX-24,Texas's 24th congressional district,congressional_district,state/tx,TX,Texas,4824,congressional_district/TX-24 +congressional_district/TX-25,Texas's 25th congressional district,congressional_district,state/tx,TX,Texas,4825,congressional_district/TX-25 +congressional_district/TX-26,Texas's 26th congressional district,congressional_district,state/tx,TX,Texas,4826,congressional_district/TX-26 +congressional_district/TX-27,Texas's 27th congressional district,congressional_district,state/tx,TX,Texas,4827,congressional_district/TX-27 +congressional_district/TX-28,Texas's 28th congressional district,congressional_district,state/tx,TX,Texas,4828,congressional_district/TX-28 +congressional_district/TX-29,Texas's 29th congressional district,congressional_district,state/tx,TX,Texas,4829,congressional_district/TX-29 +congressional_district/TX-30,Texas's 30th congressional district,congressional_district,state/tx,TX,Texas,4830,congressional_district/TX-30 +congressional_district/TX-31,Texas's 31st congressional district,congressional_district,state/tx,TX,Texas,4831,congressional_district/TX-31 +congressional_district/TX-32,Texas's 32nd congressional district,congressional_district,state/tx,TX,Texas,4832,congressional_district/TX-32 +congressional_district/TX-33,Texas's 33rd congressional district,congressional_district,state/tx,TX,Texas,4833,congressional_district/TX-33 +congressional_district/TX-34,Texas's 34th congressional district,congressional_district,state/tx,TX,Texas,4834,congressional_district/TX-34 +congressional_district/TX-35,Texas's 35th congressional district,congressional_district,state/tx,TX,Texas,4835,congressional_district/TX-35 +congressional_district/TX-36,Texas's 36th congressional district,congressional_district,state/tx,TX,Texas,4836,congressional_district/TX-36 +congressional_district/TX-37,Texas's 37th congressional district,congressional_district,state/tx,TX,Texas,4837,congressional_district/TX-37 +congressional_district/TX-38,Texas's 38th congressional district,congressional_district,state/tx,TX,Texas,4838,congressional_district/TX-38 +congressional_district/UT-01,Utah's 1st congressional district,congressional_district,state/ut,UT,Utah,4901,congressional_district/UT-01 +congressional_district/UT-02,Utah's 2nd congressional district,congressional_district,state/ut,UT,Utah,4902,congressional_district/UT-02 +congressional_district/UT-03,Utah's 3rd congressional district,congressional_district,state/ut,UT,Utah,4903,congressional_district/UT-03 +congressional_district/UT-04,Utah's 4th congressional district,congressional_district,state/ut,UT,Utah,4904,congressional_district/UT-04 +congressional_district/VA-01,Virginia's 1st congressional district,congressional_district,state/va,VA,Virginia,5101,congressional_district/VA-01 +congressional_district/VA-02,Virginia's 2nd congressional district,congressional_district,state/va,VA,Virginia,5102,congressional_district/VA-02 +congressional_district/VA-03,Virginia's 3rd congressional district,congressional_district,state/va,VA,Virginia,5103,congressional_district/VA-03 +congressional_district/VA-04,Virginia's 4th congressional district,congressional_district,state/va,VA,Virginia,5104,congressional_district/VA-04 +congressional_district/VA-05,Virginia's 5th congressional district,congressional_district,state/va,VA,Virginia,5105,congressional_district/VA-05 +congressional_district/VA-06,Virginia's 6th congressional district,congressional_district,state/va,VA,Virginia,5106,congressional_district/VA-06 +congressional_district/VA-07,Virginia's 7th congressional district,congressional_district,state/va,VA,Virginia,5107,congressional_district/VA-07 +congressional_district/VA-08,Virginia's 8th congressional district,congressional_district,state/va,VA,Virginia,5108,congressional_district/VA-08 +congressional_district/VA-09,Virginia's 9th congressional district,congressional_district,state/va,VA,Virginia,5109,congressional_district/VA-09 +congressional_district/VA-10,Virginia's 10th congressional district,congressional_district,state/va,VA,Virginia,5110,congressional_district/VA-10 +congressional_district/VA-11,Virginia's 11th congressional district,congressional_district,state/va,VA,Virginia,5111,congressional_district/VA-11 +congressional_district/VT-01,Vermont's at-large congressional district,congressional_district,state/vt,VT,Vermont,5000,congressional_district/VT-01 +congressional_district/WA-01,Washington's 1st congressional district,congressional_district,state/wa,WA,Washington,5301,congressional_district/WA-01 +congressional_district/WA-02,Washington's 2nd congressional district,congressional_district,state/wa,WA,Washington,5302,congressional_district/WA-02 +congressional_district/WA-03,Washington's 3rd congressional district,congressional_district,state/wa,WA,Washington,5303,congressional_district/WA-03 +congressional_district/WA-04,Washington's 4th congressional district,congressional_district,state/wa,WA,Washington,5304,congressional_district/WA-04 +congressional_district/WA-05,Washington's 5th congressional district,congressional_district,state/wa,WA,Washington,5305,congressional_district/WA-05 +congressional_district/WA-06,Washington's 6th congressional district,congressional_district,state/wa,WA,Washington,5306,congressional_district/WA-06 +congressional_district/WA-07,Washington's 7th congressional district,congressional_district,state/wa,WA,Washington,5307,congressional_district/WA-07 +congressional_district/WA-08,Washington's 8th congressional district,congressional_district,state/wa,WA,Washington,5308,congressional_district/WA-08 +congressional_district/WA-09,Washington's 9th congressional district,congressional_district,state/wa,WA,Washington,5309,congressional_district/WA-09 +congressional_district/WA-10,Washington's 10th congressional district,congressional_district,state/wa,WA,Washington,5310,congressional_district/WA-10 +congressional_district/WI-01,Wisconsin's 1st congressional district,congressional_district,state/wi,WI,Wisconsin,5501,congressional_district/WI-01 +congressional_district/WI-02,Wisconsin's 2nd congressional district,congressional_district,state/wi,WI,Wisconsin,5502,congressional_district/WI-02 +congressional_district/WI-03,Wisconsin's 3rd congressional district,congressional_district,state/wi,WI,Wisconsin,5503,congressional_district/WI-03 +congressional_district/WI-04,Wisconsin's 4th congressional district,congressional_district,state/wi,WI,Wisconsin,5504,congressional_district/WI-04 +congressional_district/WI-05,Wisconsin's 5th congressional district,congressional_district,state/wi,WI,Wisconsin,5505,congressional_district/WI-05 +congressional_district/WI-06,Wisconsin's 6th congressional district,congressional_district,state/wi,WI,Wisconsin,5506,congressional_district/WI-06 +congressional_district/WI-07,Wisconsin's 7th congressional district,congressional_district,state/wi,WI,Wisconsin,5507,congressional_district/WI-07 +congressional_district/WI-08,Wisconsin's 8th congressional district,congressional_district,state/wi,WI,Wisconsin,5508,congressional_district/WI-08 +congressional_district/WV-01,West Virginia's 1st congressional district,congressional_district,state/wv,WV,West Virginia,5401,congressional_district/WV-01 +congressional_district/WV-02,West Virginia's 2nd congressional district,congressional_district,state/wv,WV,West Virginia,5402,congressional_district/WV-02 +congressional_district/WY-01,Wyoming's at-large congressional district,congressional_district,state/wy,WY,Wyoming,5600,congressional_district/WY-01 +place/AK-03000,Anchorage,place,state/ak,AK,Alaska,,place/AK-03000 +place/AL-07000,Birmingham,place,state/al,AL,Alabama,,place/AL-07000 +place/AL-37000,Huntsville,place,state/al,AL,Alabama,,place/AL-37000 +place/AL-50000,Mobile,place,state/al,AL,Alabama,,place/AL-50000 +place/AL-51000,Montgomery,place,state/al,AL,Alabama,,place/AL-51000 +place/AL-77256,Tuscaloosa,place,state/al,AL,Alabama,,place/AL-77256 +place/AR-23290,Fayetteville,place,state/ar,AR,Arkansas,,place/AR-23290 +place/AR-41000,Little Rock,place,state/ar,AR,Arkansas,,place/AR-41000 +place/AZ-07940,Buckeye,place,state/az,AZ,Arizona,,place/AZ-07940 +place/AZ-12000,Chandler,place,state/az,AZ,Arizona,,place/AZ-12000 +place/AZ-27400,Gilbert,place,state/az,AZ,Arizona,,place/AZ-27400 +place/AZ-27820,Glendale,place,state/az,AZ,Arizona,,place/AZ-27820 +place/AZ-28380,Goodyear,place,state/az,AZ,Arizona,,place/AZ-28380 +place/AZ-46000,Mesa,place,state/az,AZ,Arizona,,place/AZ-46000 +place/AZ-54050,Peoria,place,state/az,AZ,Arizona,,place/AZ-54050 +place/AZ-55000,Phoenix,place,state/az,AZ,Arizona,,place/AZ-55000 +place/AZ-65000,Scottsdale,place,state/az,AZ,Arizona,,place/AZ-65000 +place/AZ-71510,Surprise,place,state/az,AZ,Arizona,,place/AZ-71510 +place/AZ-73000,Tempe,place,state/az,AZ,Arizona,,place/AZ-73000 +place/AZ-77000,Tucson,place,state/az,AZ,Arizona,,place/AZ-77000 +place/AZ-85540,Yuma,place,state/az,AZ,Arizona,,place/AZ-85540 +place/CA-02000,Anaheim,place,state/ca,CA,California,,place/CA-02000 +place/CA-02252,Antioch,place,state/ca,CA,California,,place/CA-02252 +place/CA-03526,Bakersfield,place,state/ca,CA,California,,place/CA-03526 +place/CA-06000,Berkeley,place,state/ca,CA,California,,place/CA-06000 +place/CA-08954,Burbank,place,state/ca,CA,California,,place/CA-08954 +place/CA-11194,Carlsbad,place,state/ca,CA,California,,place/CA-11194 +place/CA-13014,Chico,place,state/ca,CA,California,,place/CA-13014 +place/CA-13392,Chula Vista,place,state/ca,CA,California,,place/CA-13392 +place/CA-14218,Clovis,place,state/ca,CA,California,,place/CA-14218 +place/CA-16000,Concord,place,state/ca,CA,California,,place/CA-16000 +place/CA-16350,Corona,place,state/ca,CA,California,,place/CA-16350 +place/CA-16532,Costa Mesa,place,state/ca,CA,California,,place/CA-16532 +place/CA-19766,Downey,place,state/ca,CA,California,,place/CA-19766 +place/CA-21712,El Cajon,place,state/ca,CA,California,,place/CA-21712 +place/CA-22020,Elk Grove,place,state/ca,CA,California,,place/CA-22020 +place/CA-22230,El Monte,place,state/ca,CA,California,,place/CA-22230 +place/CA-22804,Escondido,place,state/ca,CA,California,,place/CA-22804 +place/CA-23182,Fairfield,place,state/ca,CA,California,,place/CA-23182 +place/CA-24680,Fontana,place,state/ca,CA,California,,place/CA-24680 +place/CA-26000,Fremont,place,state/ca,CA,California,,place/CA-26000 +place/CA-27000,Fresno,place,state/ca,CA,California,,place/CA-27000 +place/CA-28000,Fullerton,place,state/ca,CA,California,,place/CA-28000 +place/CA-29000,Garden Grove,place,state/ca,CA,California,,place/CA-29000 +place/CA-30000,Glendale,place,state/ca,CA,California,,place/CA-30000 +place/CA-33000,Hayward,place,state/ca,CA,California,,place/CA-33000 +place/CA-33434,Hesperia,place,state/ca,CA,California,,place/CA-33434 +place/CA-36000,Huntington Beach,place,state/ca,CA,California,,place/CA-36000 +place/CA-36546,Inglewood,place,state/ca,CA,California,,place/CA-36546 +place/CA-36770,Irvine,place,state/ca,CA,California,,place/CA-36770 +place/CA-37692,Jurupa Valley,place,state/ca,CA,California,,place/CA-37692 +place/CA-40130,Lancaster,place,state/ca,CA,California,,place/CA-40130 +place/CA-43000,Long Beach,place,state/ca,CA,California,,place/CA-43000 +place/CA-44000,Los Angeles,place,state/ca,CA,California,,place/CA-44000 +place/CA-46842,Menifee,place,state/ca,CA,California,,place/CA-46842 +place/CA-48354,Modesto,place,state/ca,CA,California,,place/CA-48354 +place/CA-49270,Moreno Valley,place,state/ca,CA,California,,place/CA-49270 +place/CA-50076,Murrieta,place,state/ca,CA,California,,place/CA-50076 +place/CA-53000,Oakland,place,state/ca,CA,California,,place/CA-53000 +place/CA-53322,Oceanside,place,state/ca,CA,California,,place/CA-53322 +place/CA-53896,Ontario,place,state/ca,CA,California,,place/CA-53896 +place/CA-53980,Orange,place,state/ca,CA,California,,place/CA-53980 +place/CA-54652,Oxnard,place,state/ca,CA,California,,place/CA-54652 +place/CA-55156,Palmdale,place,state/ca,CA,California,,place/CA-55156 +place/CA-56000,Pasadena,place,state/ca,CA,California,,place/CA-56000 +place/CA-58072,Pomona,place,state/ca,CA,California,,place/CA-58072 +place/CA-59451,Rancho Cucamonga,place,state/ca,CA,California,,place/CA-59451 +place/CA-60466,Rialto,place,state/ca,CA,California,,place/CA-60466 +place/CA-60620,Richmond,place,state/ca,CA,California,,place/CA-60620 +place/CA-62000,Riverside,place,state/ca,CA,California,,place/CA-62000 +place/CA-62938,Roseville,place,state/ca,CA,California,,place/CA-62938 +place/CA-64000,Sacramento,place,state/ca,CA,California,,place/CA-64000 +place/CA-64224,Salinas,place,state/ca,CA,California,,place/CA-64224 +place/CA-65000,San Bernardino,place,state/ca,CA,California,,place/CA-65000 +place/CA-65042,Ventura,place,state/ca,CA,California,,place/CA-65042 +place/CA-66000,San Diego,place,state/ca,CA,California,,place/CA-66000 +place/CA-67000,San Francisco,place,state/ca,CA,California,,place/CA-67000 +place/CA-68000,San Jose,place,state/ca,CA,California,,place/CA-68000 +place/CA-68252,San Mateo,place,state/ca,CA,California,,place/CA-68252 +place/CA-69000,Santa Ana,place,state/ca,CA,California,,place/CA-69000 +place/CA-69084,Santa Clara,place,state/ca,CA,California,,place/CA-69084 +place/CA-69088,Santa Clarita,place,state/ca,CA,California,,place/CA-69088 +place/CA-69196,Santa Maria,place,state/ca,CA,California,,place/CA-69196 +place/CA-70098,Santa Rosa,place,state/ca,CA,California,,place/CA-70098 +place/CA-72016,Simi Valley,place,state/ca,CA,California,,place/CA-72016 +place/CA-75000,Stockton,place,state/ca,CA,California,,place/CA-75000 +place/CA-77000,Sunnyvale,place,state/ca,CA,California,,place/CA-77000 +place/CA-78120,Temecula,place,state/ca,CA,California,,place/CA-78120 +place/CA-78582,Thousand Oaks,place,state/ca,CA,California,,place/CA-78582 +place/CA-80000,Torrance,place,state/ca,CA,California,,place/CA-80000 +place/CA-81554,Vacaville,place,state/ca,CA,California,,place/CA-81554 +place/CA-81666,Vallejo,place,state/ca,CA,California,,place/CA-81666 +place/CA-82590,Victorville,place,state/ca,CA,California,,place/CA-82590 +place/CA-82954,Visalia,place,state/ca,CA,California,,place/CA-82954 +place/CA-84200,West Covina,place,state/ca,CA,California,,place/CA-84200 +place/CO-03455,Arvada,place,state/co,CO,Colorado,,place/CO-03455 +place/CO-04000,Aurora,place,state/co,CO,Colorado,,place/CO-04000 +place/CO-07850,Boulder,place,state/co,CO,Colorado,,place/CO-07850 +place/CO-12815,Centennial,place,state/co,CO,Colorado,,place/CO-12815 +place/CO-16000,Colorado Springs,place,state/co,CO,Colorado,,place/CO-16000 +place/CO-20000,Denver,place,state/co,CO,Colorado,,place/CO-20000 +place/CO-27425,Fort Collins,place,state/co,CO,Colorado,,place/CO-27425 +place/CO-32155,Greeley,place,state/co,CO,Colorado,,place/CO-32155 +place/CO-43000,Lakewood,place,state/co,CO,Colorado,,place/CO-43000 +place/CO-62000,Pueblo,place,state/co,CO,Colorado,,place/CO-62000 +place/CO-77290,Thornton,place,state/co,CO,Colorado,,place/CO-77290 +place/CO-83835,Westminster,place,state/co,CO,Colorado,,place/CO-83835 +place/CT-08000,Bridgeport,place,state/ct,CT,Connecticut,,place/CT-08000 +place/CT-37000,Hartford,place,state/ct,CT,Connecticut,,place/CT-37000 +place/CT-52000,New Haven,place,state/ct,CT,Connecticut,,place/CT-52000 +place/CT-73000,Stamford,place,state/ct,CT,Connecticut,,place/CT-73000 +place/CT-80000,Waterbury,place,state/ct,CT,Connecticut,,place/CT-80000 +place/DC-50000,Washington,place,state/dc,DC,District of Columbia,,place/DC-50000 +place/FL-10275,Cape Coral,place,state/fl,FL,Florida,,place/FL-10275 +place/FL-12875,Clearwater,place,state/fl,FL,Florida,,place/FL-12875 +place/FL-14400,Coral Springs,place,state/fl,FL,Florida,,place/FL-14400 +place/FL-16475,Davie,place,state/fl,FL,Florida,,place/FL-16475 +place/FL-24000,Fort Lauderdale,place,state/fl,FL,Florida,,place/FL-24000 +place/FL-25175,Gainesville,place,state/fl,FL,Florida,,place/FL-25175 +place/FL-30000,Hialeah,place,state/fl,FL,Florida,,place/FL-30000 +place/FL-32000,Hollywood,place,state/fl,FL,Florida,,place/FL-32000 +place/FL-35000,Jacksonville,place,state/fl,FL,Florida,,place/FL-35000 +place/FL-38250,Lakeland,place,state/fl,FL,Florida,,place/FL-38250 +place/FL-45000,Miami,place,state/fl,FL,Florida,,place/FL-45000 +place/FL-45060,Miami Gardens,place,state/fl,FL,Florida,,place/FL-45060 +place/FL-45975,Miramar,place,state/fl,FL,Florida,,place/FL-45975 +place/FL-53000,Orlando,place,state/fl,FL,Florida,,place/FL-53000 +place/FL-54000,Palm Bay,place,state/fl,FL,Florida,,place/FL-54000 +place/FL-54200,Palm Coast,place,state/fl,FL,Florida,,place/FL-54200 +place/FL-55775,Pembroke Pines,place,state/fl,FL,Florida,,place/FL-55775 +place/FL-58050,Pompano Beach,place,state/fl,FL,Florida,,place/FL-58050 +place/FL-58715,Port St. Lucie,place,state/fl,FL,Florida,,place/FL-58715 +place/FL-63000,St. Petersburg,place,state/fl,FL,Florida,,place/FL-63000 +place/FL-70600,Tallahassee,place,state/fl,FL,Florida,,place/FL-70600 +place/FL-71000,Tampa,place,state/fl,FL,Florida,,place/FL-71000 +place/FL-76600,West Palm Beach,place,state/fl,FL,Florida,,place/FL-76600 +place/GA-03440,Athens-Clarke County,place,state/ga,GA,Georgia,,place/GA-03440 +place/GA-04000,Atlanta,place,state/ga,GA,Georgia,,place/GA-04000 +place/GA-04204,Augusta-Richmond County,place,state/ga,GA,Georgia,,place/GA-04204 +place/GA-19000,Columbus,place,state/ga,GA,Georgia,,place/GA-19000 +place/GA-49008,Macon-Bibb County,place,state/ga,GA,Georgia,,place/GA-49008 +place/GA-68516,Sandy Springs,place,state/ga,GA,Georgia,,place/GA-68516 +place/GA-69000,Savannah,place,state/ga,GA,Georgia,,place/GA-69000 +place/GA-72122,South Fulton,place,state/ga,GA,Georgia,,place/GA-72122 +place/HI-71550,Urban Honolulu,place,state/hi,HI,Hawaii,,place/HI-71550 +place/IA-12000,Cedar Rapids,place,state/ia,IA,Iowa,,place/IA-12000 +place/IA-19000,Davenport,place,state/ia,IA,Iowa,,place/IA-19000 +place/IA-21000,Des Moines,place,state/ia,IA,Iowa,,place/IA-21000 +place/ID-08830,Boise City,place,state/id,ID,Idaho,,place/ID-08830 +place/ID-52120,Meridian,place,state/id,ID,Idaho,,place/ID-52120 +place/ID-56260,Nampa,place,state/id,ID,Idaho,,place/ID-56260 +place/IL-03012,Aurora,place,state/il,IL,Illinois,,place/IL-03012 +place/IL-14000,Chicago,place,state/il,IL,Illinois,,place/IL-14000 +place/IL-23074,Elgin,place,state/il,IL,Illinois,,place/IL-23074 +place/IL-38570,Joliet,place,state/il,IL,Illinois,,place/IL-38570 +place/IL-51622,Naperville,place,state/il,IL,Illinois,,place/IL-51622 +place/IL-59000,Peoria,place,state/il,IL,Illinois,,place/IL-59000 +place/IL-65000,Rockford,place,state/il,IL,Illinois,,place/IL-65000 +place/IL-72000,Springfield,place,state/il,IL,Illinois,,place/IL-72000 +place/IN-10342,Carmel,place,state/in,IN,Indiana,,place/IN-10342 +place/IN-22000,Evansville,place,state/in,IN,Indiana,,place/IN-22000 +place/IN-23278,Fishers,place,state/in,IN,Indiana,,place/IN-23278 +place/IN-25000,Fort Wayne,place,state/in,IN,Indiana,,place/IN-25000 +place/IN-36003,Indianapolis,place,state/in,IN,Indiana,,place/IN-36003 +place/IN-71000,South Bend,place,state/in,IN,Indiana,,place/IN-71000 +place/KS-36000,Kansas City,place,state/ks,KS,Kansas,,place/KS-36000 +place/KS-52575,Olathe,place,state/ks,KS,Kansas,,place/KS-52575 +place/KS-53775,Overland Park,place,state/ks,KS,Kansas,,place/KS-53775 +place/KS-71000,Topeka,place,state/ks,KS,Kansas,,place/KS-71000 +place/KS-79000,Wichita,place,state/ks,KS,Kansas,,place/KS-79000 +place/KY-46027,Lexington-Fayette,place,state/ky,KY,Kentucky,,place/KY-46027 +place/KY-48006,Louisville/Jefferson County,place,state/ky,KY,Kentucky,,place/KY-48006 +place/LA-05000,Baton Rouge,place,state/la,LA,Louisiana,,place/LA-05000 +place/LA-40735,Lafayette,place,state/la,LA,Louisiana,,place/LA-40735 +place/LA-55000,New Orleans,place,state/la,LA,Louisiana,,place/LA-55000 +place/LA-70000,Shreveport,place,state/la,LA,Louisiana,,place/LA-70000 +place/MA-07000,Boston,place,state/ma,MA,Massachusetts,,place/MA-07000 +place/MA-09000,Brockton,place,state/ma,MA,Massachusetts,,place/MA-09000 +place/MA-11000,Cambridge,place,state/ma,MA,Massachusetts,,place/MA-11000 +place/MA-37000,Lowell,place,state/ma,MA,Massachusetts,,place/MA-37000 +place/MA-37490,Lynn,place,state/ma,MA,Massachusetts,,place/MA-37490 +place/MA-45000,New Bedford,place,state/ma,MA,Massachusetts,,place/MA-45000 +place/MA-55745,Quincy,place,state/ma,MA,Massachusetts,,place/MA-55745 +place/MA-67000,Springfield,place,state/ma,MA,Massachusetts,,place/MA-67000 +place/MA-82000,Worcester,place,state/ma,MA,Massachusetts,,place/MA-82000 +place/MD-04000,Baltimore,place,state/md,MD,Maryland,,place/MD-04000 +place/MI-03000,Ann Arbor,place,state/mi,MI,Michigan,,place/MI-03000 +place/MI-21000,Dearborn,place,state/mi,MI,Michigan,,place/MI-21000 +place/MI-22000,Detroit,place,state/mi,MI,Michigan,,place/MI-22000 +place/MI-34000,Grand Rapids,place,state/mi,MI,Michigan,,place/MI-34000 +place/MI-46000,Lansing,place,state/mi,MI,Michigan,,place/MI-46000 +place/MI-76460,Sterling Heights,place,state/mi,MI,Michigan,,place/MI-76460 +place/MI-84000,Warren,place,state/mi,MI,Michigan,,place/MI-84000 +place/MN-43000,Minneapolis,place,state/mn,MN,Minnesota,,place/MN-43000 +place/MN-54880,Rochester,place,state/mn,MN,Minnesota,,place/MN-54880 +place/MN-58000,St. Paul,place,state/mn,MN,Minnesota,,place/MN-58000 +place/MO-15670,Columbia,place,state/mo,MO,Missouri,,place/MO-15670 +place/MO-35000,Independence,place,state/mo,MO,Missouri,,place/MO-35000 +place/MO-38000,Kansas City,place,state/mo,MO,Missouri,,place/MO-38000 +place/MO-41348,Lee's Summit,place,state/mo,MO,Missouri,,place/MO-41348 +place/MO-65000,St. Louis,place,state/mo,MO,Missouri,,place/MO-65000 +place/MO-70000,Springfield,place,state/mo,MO,Missouri,,place/MO-70000 +place/MS-36000,Jackson,place,state/ms,MS,Mississippi,,place/MS-36000 +place/MT-06550,Billings,place,state/mt,MT,Montana,,place/MT-06550 +place/NC-10740,Cary,place,state/nc,NC,North Carolina,,place/NC-10740 +place/NC-12000,Charlotte,place,state/nc,NC,North Carolina,,place/NC-12000 +place/NC-14100,Concord,place,state/nc,NC,North Carolina,,place/NC-14100 +place/NC-19000,Durham,place,state/nc,NC,North Carolina,,place/NC-19000 +place/NC-22920,Fayetteville,place,state/nc,NC,North Carolina,,place/NC-22920 +place/NC-28000,Greensboro,place,state/nc,NC,North Carolina,,place/NC-28000 +place/NC-31400,High Point,place,state/nc,NC,North Carolina,,place/NC-31400 +place/NC-55000,Raleigh,place,state/nc,NC,North Carolina,,place/NC-55000 +place/NC-74440,Wilmington,place,state/nc,NC,North Carolina,,place/NC-74440 +place/NC-75000,Winston-Salem,place,state/nc,NC,North Carolina,,place/NC-75000 +place/ND-25700,Fargo,place,state/nd,ND,North Dakota,,place/ND-25700 +place/NE-28000,Lincoln,place,state/ne,NE,Nebraska,,place/NE-28000 +place/NE-37000,Omaha,place,state/ne,NE,Nebraska,,place/NE-37000 +place/NH-45140,Manchester,place,state/nh,NH,New Hampshire,,place/NH-45140 +place/NJ-21000,Elizabeth,place,state/nj,NJ,New Jersey,,place/NJ-21000 +place/NJ-36000,Jersey City,place,state/nj,NJ,New Jersey,,place/NJ-36000 +place/NJ-51000,Newark,place,state/nj,NJ,New Jersey,,place/NJ-51000 +place/NJ-57000,Paterson,place,state/nj,NJ,New Jersey,,place/NJ-57000 +place/NM-02000,Albuquerque,place,state/nm,NM,New Mexico,,place/NM-02000 +place/NM-39380,Las Cruces,place,state/nm,NM,New Mexico,,place/NM-39380 +place/NM-63460,Rio Rancho,place,state/nm,NM,New Mexico,,place/NM-63460 +place/NV-31900,Henderson,place,state/nv,NV,Nevada,,place/NV-31900 +place/NV-40000,Las Vegas,place,state/nv,NV,Nevada,,place/NV-40000 +place/NV-51800,North Las Vegas,place,state/nv,NV,Nevada,,place/NV-51800 +place/NV-60600,Reno,place,state/nv,NV,Nevada,,place/NV-60600 +place/NV-68400,Sparks,place,state/nv,NV,Nevada,,place/NV-68400 +place/NY-01000,Albany,place,state/ny,NY,New York,,place/NY-01000 +place/NY-11000,Buffalo,place,state/ny,NY,New York,,place/NY-11000 +place/NY-51000,New York City,place,state/ny,NY,New York,,place/NY-51000 +place/NY-63000,Rochester,place,state/ny,NY,New York,,place/NY-63000 +place/NY-73000,Syracuse,place,state/ny,NY,New York,,place/NY-73000 +place/NY-84000,Yonkers,place,state/ny,NY,New York,,place/NY-84000 +place/OH-01000,Akron,place,state/oh,OH,Ohio,,place/OH-01000 +place/OH-15000,Cincinnati,place,state/oh,OH,Ohio,,place/OH-15000 +place/OH-16000,Cleveland,place,state/oh,OH,Ohio,,place/OH-16000 +place/OH-18000,Columbus,place,state/oh,OH,Ohio,,place/OH-18000 +place/OH-21000,Dayton,place,state/oh,OH,Ohio,,place/OH-21000 +place/OH-77000,Toledo,place,state/oh,OH,Ohio,,place/OH-77000 +place/OK-09050,Broken Arrow,place,state/ok,OK,Oklahoma,,place/OK-09050 +place/OK-52500,Norman,place,state/ok,OK,Oklahoma,,place/OK-52500 +place/OK-55000,Oklahoma City,place,state/ok,OK,Oklahoma,,place/OK-55000 +place/OK-75000,Tulsa,place,state/ok,OK,Oklahoma,,place/OK-75000 +place/OR-05800,Bend,place,state/or,OR,Oregon,,place/OR-05800 +place/OR-23850,Eugene,place,state/or,OR,Oregon,,place/OR-23850 +place/OR-31250,Gresham,place,state/or,OR,Oregon,,place/OR-31250 +place/OR-34100,Hillsboro,place,state/or,OR,Oregon,,place/OR-34100 +place/OR-59000,Portland,place,state/or,OR,Oregon,,place/OR-59000 +place/OR-64900,Salem,place,state/or,OR,Oregon,,place/OR-64900 +place/PA-02000,Allentown,place,state/pa,PA,Pennsylvania,,place/PA-02000 +place/PA-60000,Philadelphia,place,state/pa,PA,Pennsylvania,,place/PA-60000 +place/PA-61000,Pittsburgh,place,state/pa,PA,Pennsylvania,,place/PA-61000 +place/RI-59000,Providence,place,state/ri,RI,Rhode Island,,place/RI-59000 +place/SC-13330,Charleston,place,state/sc,SC,South Carolina,,place/SC-13330 +place/SC-16000,Columbia,place,state/sc,SC,South Carolina,,place/SC-16000 +place/SC-50875,North Charleston,place,state/sc,SC,South Carolina,,place/SC-50875 +place/SD-59020,Sioux Falls,place,state/sd,SD,South Dakota,,place/SD-59020 +place/TN-14000,Chattanooga,place,state/tn,TN,Tennessee,,place/TN-14000 +place/TN-15160,Clarksville,place,state/tn,TN,Tennessee,,place/TN-15160 +place/TN-40000,Knoxville,place,state/tn,TN,Tennessee,,place/TN-40000 +place/TN-48000,Memphis,place,state/tn,TN,Tennessee,,place/TN-48000 +place/TN-51560,Murfreesboro,place,state/tn,TN,Tennessee,,place/TN-51560 +place/TN-52006,Nashville-Davidson,place,state/tn,TN,Tennessee,,place/TN-52006 +place/TX-01000,Abilene,place,state/tx,TX,Texas,,place/TX-01000 +place/TX-01924,Allen,place,state/tx,TX,Texas,,place/TX-01924 +place/TX-03000,Amarillo,place,state/tx,TX,Texas,,place/TX-03000 +place/TX-04000,Arlington,place,state/tx,TX,Texas,,place/TX-04000 +place/TX-05000,Austin,place,state/tx,TX,Texas,,place/TX-05000 +place/TX-07000,Beaumont,place,state/tx,TX,Texas,,place/TX-07000 +place/TX-10768,Brownsville,place,state/tx,TX,Texas,,place/TX-10768 +place/TX-13024,Carrollton,place,state/tx,TX,Texas,,place/TX-13024 +place/TX-15976,College Station,place,state/tx,TX,Texas,,place/TX-15976 +place/TX-16432,Conroe,place,state/tx,TX,Texas,,place/TX-16432 +place/TX-17000,Corpus Christi,place,state/tx,TX,Texas,,place/TX-17000 +place/TX-19000,Dallas,place,state/tx,TX,Texas,,place/TX-19000 +place/TX-19972,Denton,place,state/tx,TX,Texas,,place/TX-19972 +place/TX-22660,Edinburg,place,state/tx,TX,Texas,,place/TX-22660 +place/TX-24000,El Paso,place,state/tx,TX,Texas,,place/TX-24000 +place/TX-27000,Fort Worth,place,state/tx,TX,Texas,,place/TX-27000 +place/TX-27684,Frisco,place,state/tx,TX,Texas,,place/TX-27684 +place/TX-29000,Garland,place,state/tx,TX,Texas,,place/TX-29000 +place/TX-30464,Grand Prairie,place,state/tx,TX,Texas,,place/TX-30464 +place/TX-35000,Houston,place,state/tx,TX,Texas,,place/TX-35000 +place/TX-37000,Irving,place,state/tx,TX,Texas,,place/TX-37000 +place/TX-39148,Killeen,place,state/tx,TX,Texas,,place/TX-39148 +place/TX-41464,Laredo,place,state/tx,TX,Texas,,place/TX-41464 +place/TX-41980,League City,place,state/tx,TX,Texas,,place/TX-41980 +place/TX-42508,Lewisville,place,state/tx,TX,Texas,,place/TX-42508 +place/TX-45000,Lubbock,place,state/tx,TX,Texas,,place/TX-45000 +place/TX-45384,McAllen,place,state/tx,TX,Texas,,place/TX-45384 +place/TX-45744,McKinney,place,state/tx,TX,Texas,,place/TX-45744 +place/TX-47892,Mesquite,place,state/tx,TX,Texas,,place/TX-47892 +place/TX-48072,Midland,place,state/tx,TX,Texas,,place/TX-48072 +place/TX-50820,New Braunfels,place,state/tx,TX,Texas,,place/TX-50820 +place/TX-53388,Odessa,place,state/tx,TX,Texas,,place/TX-53388 +place/TX-56000,Pasadena,place,state/tx,TX,Texas,,place/TX-56000 +place/TX-56348,Pearland,place,state/tx,TX,Texas,,place/TX-56348 +place/TX-58016,Plano,place,state/tx,TX,Texas,,place/TX-58016 +place/TX-61796,Richardson,place,state/tx,TX,Texas,,place/TX-61796 +place/TX-63500,Round Rock,place,state/tx,TX,Texas,,place/TX-63500 +place/TX-65000,San Antonio,place,state/tx,TX,Texas,,place/TX-65000 +place/TX-70808,Sugar Land,place,state/tx,TX,Texas,,place/TX-70808 +place/TX-74144,Tyler,place,state/tx,TX,Texas,,place/TX-74144 +place/TX-76000,Waco,place,state/tx,TX,Texas,,place/TX-76000 +place/TX-79000,Wichita Falls,place,state/tx,TX,Texas,,place/TX-79000 +place/UT-62470,Provo,place,state/ut,UT,Utah,,place/UT-62470 +place/UT-65330,St. George,place,state/ut,UT,Utah,,place/UT-65330 +place/UT-67000,Salt Lake City,place,state/ut,UT,Utah,,place/UT-67000 +place/UT-82950,West Jordan,place,state/ut,UT,Utah,,place/UT-82950 +place/UT-83470,West Valley City,place,state/ut,UT,Utah,,place/UT-83470 +place/VA-01000,Alexandria,place,state/va,VA,Virginia,,place/VA-01000 +place/VA-16000,Chesapeake,place,state/va,VA,Virginia,,place/VA-16000 +place/VA-35000,Hampton,place,state/va,VA,Virginia,,place/VA-35000 +place/VA-56000,Newport News,place,state/va,VA,Virginia,,place/VA-56000 +place/VA-57000,Norfolk,place,state/va,VA,Virginia,,place/VA-57000 +place/VA-67000,Richmond,place,state/va,VA,Virginia,,place/VA-67000 +place/VA-76432,Suffolk,place,state/va,VA,Virginia,,place/VA-76432 +place/VA-82000,Virginia Beach,place,state/va,VA,Virginia,,place/VA-82000 +place/WA-05210,Bellevue,place,state/wa,WA,Washington,,place/WA-05210 +place/WA-22640,Everett,place,state/wa,WA,Washington,,place/WA-22640 +place/WA-35415,Kent,place,state/wa,WA,Washington,,place/WA-35415 +place/WA-57745,Renton,place,state/wa,WA,Washington,,place/WA-57745 +place/WA-63000,Seattle,place,state/wa,WA,Washington,,place/WA-63000 +place/WA-67000,Spokane,place,state/wa,WA,Washington,,place/WA-67000 +place/WA-67167,Spokane Valley,place,state/wa,WA,Washington,,place/WA-67167 +place/WA-70000,Tacoma,place,state/wa,WA,Washington,,place/WA-70000 +place/WA-74060,Vancouver,place,state/wa,WA,Washington,,place/WA-74060 +place/WI-31000,Green Bay,place,state/wi,WI,Wisconsin,,place/WI-31000 +place/WI-48000,Madison,place,state/wi,WI,Wisconsin,,place/WI-48000 +place/WI-53000,Milwaukee,place,state/wi,WI,Wisconsin,,place/WI-53000 diff --git a/policyengine_api/data/v2/catalog/region_references.py b/policyengine_api/data/v2/catalog/region_references.py new file mode 100644 index 000000000..421e73622 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_references.py @@ -0,0 +1,259 @@ +"""The Public API's checked-in region reference files, one per country. + +Each file lists every region of one country: its code, label, type, parent, +US state, the value that selects its households in its dataset, and the v1 +identifier it replaces. Catalog publication reads regions from these files, +never from PolicyEngine.py's region registry. +""" + +from __future__ import annotations + +import csv +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + +REGION_REFERENCE_DIRECTORY = Path(__file__).parent / "region_reference_files" +REGION_REFERENCE_FIELDS = ( + "code", + "label", + "region_type", + "parent_code", + "state_code", + "state_name", + "filter_value", + "v1_code", +) +SUPPORTED_REGION_TYPES = frozenset( + { + "national", + "country", + "state", + "congressional_district", + "constituency", + "local_authority", + "city", + "place", + } +) +# The catalog's region columns hold at most these many characters. +MAX_REGION_CODE_LENGTH = 255 +MAX_REGION_LABEL_LENGTH = 255 +MAX_REGION_STATE_CODE_LENGTH = 16 +MAX_REGION_STATE_NAME_LENGTH = 128 +MAX_REGION_FILTER_VALUE_LENGTH = 255 + + +@dataclass(frozen=True, slots=True) +class RegionReference: + """One region as its country's reference file lists it. + + ``filter_value`` is the text that selects the region's households: a FIPS + number or district GEOID for US states and districts, the stored region + names of a UK nation, or the ONS code of a UK constituency or local + authority (its area code in the local-area dataset). The national region and regions + that cannot run yet have none. ``v1_code`` is the v1 region identifier this + region replaces, if v1 had one. + """ + + code: str + label: str + region_type: str + parent_code: str | None + state_code: str | None + state_name: str | None + filter_value: str | None + v1_code: str | None + + +def _text( + row: Mapping[str, str | None], + field: str, + *, + code: str, + maximum: int, + required: bool, +) -> str | None: + value = row.get(field) or "" + if value != value.strip(): + msg = f"region {code!r} {field} has surrounding whitespace" + raise CatalogExtractionError(msg) + if not value: + if required: + msg = f"region {code!r} {field} must be a non-empty string" + raise CatalogExtractionError(msg) + return None + if len(value) > maximum: + msg = f"region {code!r} {field} exceeds {maximum} characters" + raise CatalogExtractionError(msg) + return value + + +def _reference(country_id: str, row: Mapping[str, str | None]) -> RegionReference: + code = row.get("code") or "" + if not code or code != code.strip() or len(code) > MAX_REGION_CODE_LENGTH: + msg = f"{country_id} region reference file has an invalid code {code!r}" + raise CatalogExtractionError(msg) + region_type = row.get("region_type") or "" + if region_type not in SUPPORTED_REGION_TYPES: + msg = f"region {code!r} has unsupported type {region_type!r}" + raise CatalogExtractionError(msg) + if region_type != "national" and not code.startswith(f"{region_type}/"): + msg = f"region {code!r} does not start with its type {region_type + '/'!r}" + raise CatalogExtractionError(msg) + return RegionReference( + code=code, + label=_text( + row, "label", code=code, maximum=MAX_REGION_LABEL_LENGTH, required=True + ) + or "", + region_type=region_type, + parent_code=_text( + row, + "parent_code", + code=code, + maximum=MAX_REGION_CODE_LENGTH, + required=False, + ), + state_code=_text( + row, + "state_code", + code=code, + maximum=MAX_REGION_STATE_CODE_LENGTH, + required=False, + ), + state_name=_text( + row, + "state_name", + code=code, + maximum=MAX_REGION_STATE_NAME_LENGTH, + required=False, + ), + filter_value=_text( + row, + "filter_value", + code=code, + maximum=MAX_REGION_FILTER_VALUE_LENGTH, + required=False, + ), + v1_code=_text( + row, "v1_code", code=code, maximum=MAX_REGION_CODE_LENGTH, required=False + ), + ) + + +def _repeated(values: Iterable[str | None]) -> list[str]: + return sorted( + value + for value, count in Counter(value for value in values if value).items() + if count > 1 + ) + + +def _check_parents_present( + country_id: str, + references: tuple[RegionReference, ...], +) -> None: + """Require one parentless national region and a parent for every other region.""" + if not any(reference.code == country_id for reference in references): + msg = ( + f"{country_id} region reference file has no national region {country_id!r}" + ) + raise CatalogExtractionError(msg) + for reference in references: + if reference.region_type == "national": + if reference.code != country_id: + msg = f"national region {reference.code!r} must be named {country_id!r}" + raise CatalogExtractionError(msg) + if reference.parent_code is not None: + msg = f"national region {reference.code!r} must have no parent" + raise CatalogExtractionError(msg) + elif reference.parent_code is None: + msg = f"region {reference.code!r} has no parent" + raise CatalogExtractionError(msg) + + +def _check_ancestry( + country_id: str, + references: tuple[RegionReference, ...], +) -> None: + """Require every region's parents to exist and lead to the national region.""" + parents = {reference.code: reference.parent_code for reference in references} + for reference in references: + visited = {reference.code} + child = reference.code + ancestor = reference.parent_code + while ancestor is not None and ancestor != country_id: + if ancestor not in parents: + raise CatalogExtractionError( + _unknown_ancestor_message(reference.code, child, ancestor) + ) + if ancestor in visited: + msg = f"region {reference.code!r} has a parent cycle" + raise CatalogExtractionError(msg) + visited.add(ancestor) + child, ancestor = ancestor, parents[ancestor] + + +def _unknown_ancestor_message(code: str, child: str, ancestor: str) -> str: + """Name the missing region and, if it is further up, the region it parents.""" + if child == code: + return f"region {code!r} names an unknown parent {ancestor!r}" + return ( + f"region {code!r} descends from {child!r}, whose parent {ancestor!r} is unknown" + ) + + +def parse_region_references( + country_id: str, + rows: Iterable[Mapping[str, str | None]], +) -> tuple[RegionReference, ...]: + """Validate one country's reference rows and return them in file order. + + Codes and v1 identifiers must be unique, every region's code must start + with its type, and every region must descend from the national region. + """ + references = tuple(_reference(country_id, row) for row in rows) + repeated_codes = _repeated(reference.code for reference in references) + if repeated_codes: + msg = ( + f"{country_id} region reference file repeats region codes: {repeated_codes}" + ) + raise CatalogExtractionError(msg) + repeated_v1_codes = _repeated(reference.v1_code for reference in references) + if repeated_v1_codes: + msg = ( + f"{country_id} region reference file repeats v1 identifiers: " + f"{repeated_v1_codes}" + ) + raise CatalogExtractionError(msg) + _check_parents_present(country_id, references) + _check_ancestry(country_id, references) + return references + + +def load_region_references( + country_id: str, + *, + directory: Path = REGION_REFERENCE_DIRECTORY, +) -> tuple[RegionReference, ...]: + """Read and validate the checked-in reference file of one country.""" + path = directory / f"{country_id}.csv" + if not path.is_file(): + msg = f"{country_id} has no region reference file" + raise CatalogExtractionError(msg) + with path.open(encoding="utf-8", newline="") as handle: + reader = csv.DictReader(handle) + if tuple(reader.fieldnames or ()) != REGION_REFERENCE_FIELDS: + msg = ( + f"{country_id} region reference file columns must be " + f"{', '.join(REGION_REFERENCE_FIELDS)}" + ) + raise CatalogExtractionError(msg) + return parse_region_references(country_id, reader) diff --git a/policyengine_api/data/v2/catalog/region_scoping.py b/policyengine_api/data/v2/catalog/region_scoping.py new file mode 100644 index 000000000..aa11d8cc7 --- /dev/null +++ b/policyengine_api/data/v2/catalog/region_scoping.py @@ -0,0 +1,324 @@ +"""Give each reference region its dataset and filter from the certified data release. + +A scoping rule per country and region type names the release's dataset role +(a key of the release manifest's ``region_datasets``) and the household column +whose value selects the region's households in that dataset. Region types +without a rule cannot run yet, so publication leaves them out and counts them. +""" + +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.region_references import RegionReference + +ROW_FILTER = "row_filter" +NATIONAL_DATASET_ROLE = "national" + + +@dataclass(frozen=True, slots=True) +class RegionScopingRule: + """How regions of one type find their dataset and select their households. + + ``filter_field`` is the household column a row filter compares with the + region's filter value. When the release names no dataset for the role, + the regions use the default dataset, unless ``requires_release_dataset``: + then only the routed dataset carries the column, and scoping fails. + """ + + dataset_role: str + filter_strategy: str | None = None + filter_field: str | None = None + requires_release_dataset: bool = False + + +REGION_SCOPING_RULES: Mapping[tuple[str, str], RegionScopingRule] = { + ("us", "national"): RegionScopingRule(NATIONAL_DATASET_ROLE), + ("us", "state"): RegionScopingRule("state", ROW_FILTER, "state_fips"), + ("us", "congressional_district"): RegionScopingRule( + "congressional_district", ROW_FILTER, "congressional_district_geoid" + ), + ("uk", "national"): RegionScopingRule(NATIONAL_DATASET_ROLE), + ("uk", "country"): RegionScopingRule(NATIONAL_DATASET_ROLE, ROW_FILTER, "region"), + # The UK national file has no area codes; the release routes constituencies + # and local authorities to a local-area dataset that carries them. + ("uk", "constituency"): RegionScopingRule( + "constituency", + ROW_FILTER, + "constituency_code_oa", + requires_release_dataset=True, + ), + ("uk", "local_authority"): RegionScopingRule( + "local_authority", + ROW_FILTER, + "la_code_oa", + requires_release_dataset=True, + ), +} + + +@dataclass(frozen=True, slots=True) +class ScopedRegion: + """A reference region with its dataset and the filter selecting its households.""" + + reference: RegionReference + dataset_name: str + filter_strategy: str | None + filter_field: str | None + + +@dataclass(frozen=True, slots=True) +class RegionScoping: + """Every runnable region of one country, and what the others fell back to. + + ``fallback_counts`` counts, by region type, regions whose dataset role the + release does not name, so they use the default dataset; ``omitted_counts`` + counts the regions left out because their type cannot run yet. + """ + + regions: tuple[ScopedRegion, ...] + dataset_names: frozenset[str] + certified_artifacts: Mapping[str, CertifiedArtifact] + fallback_counts: Mapping[str, int] + omitted_counts: Mapping[str, int] + + +def _manifest_text(value: object, *, description: str) -> str: + if not isinstance(value, str) or not value: + msg = f"{description} must be a non-empty string" + raise CatalogExtractionError(msg) + return value + + +@runtime_checkable +class ReleaseManifest(Protocol): + """What scoping reads from a PolicyEngine.py country release manifest. + + The manifest is an untyped public object, so each value is checked where + it is read. + """ + + @property + def datasets(self) -> object: + """The certified artifacts by name.""" + ... + + @property + def region_datasets(self) -> object: + """The dataset path of each region type.""" + ... + + @property + def data_package(self) -> object: + """The repository and revision artifacts default to.""" + ... + + +def certified_artifacts( + country_id: str, + release_manifest: ReleaseManifest, +) -> dict[str, CertifiedArtifact]: + """Return the release manifest's artifacts, with the data package's defaults. + + An artifact without its own repository, repository type or revision is in + the data package's repository at the release revision. + """ + data_package = release_manifest.data_package + default_repo_id = getattr(data_package, "repo_id", None) + default_repo_type = getattr(data_package, "repo_type", None) or "model" + default_revision = getattr( + data_package, "release_manifest_revision", None + ) or getattr(data_package, "version", None) + datasets = release_manifest.datasets + if not isinstance(datasets, Mapping): + msg = f"{country_id} release manifest has no dataset map" + raise CatalogExtractionError(msg) + artifacts: dict[str, CertifiedArtifact] = {} + for name, reference in datasets.items(): + description = f"{country_id} release artifact {name}" + artifacts[name] = CertifiedArtifact( + name=_manifest_text(name, description=f"{country_id} release artifact"), + repo_id=_manifest_text( + getattr(reference, "repo_id", None) or default_repo_id, + description=f"{description} repository", + ), + repo_type=_manifest_text( + getattr(reference, "repo_type", None) or default_repo_type, + description=f"{description} repository type", + ), + path=_manifest_text( + getattr(reference, "path", None), description=f"{description} path" + ), + revision=_manifest_text( + getattr(reference, "revision", None) or default_revision, + description=f"{description} revision", + ), + sha256=getattr(reference, "sha256", None), + ) + return artifacts + + +def _role_datasets( + country_id: str, + release_manifest: ReleaseManifest, + artifacts: Mapping[str, CertifiedArtifact], +) -> dict[str, str]: + """Map each dataset role the release names to its certified dataset's name.""" + region_datasets = release_manifest.region_datasets + if not isinstance(region_datasets, Mapping): + msg = f"{country_id} release manifest has no region dataset map" + raise CatalogExtractionError(msg) + names_by_path = {artifact.path: name for name, artifact in artifacts.items()} + role_datasets: dict[str, str] = {} + for role, template in region_datasets.items(): + path = _manifest_text( + getattr(template, "path_template", None), + description=f"{country_id} {role} region dataset", + ) + if "{" in path: + msg = ( + f"{country_id} {role} regions name a per-region dataset template " + f"{path!r}, which the catalog does not support" + ) + raise CatalogExtractionError(msg) + if path not in names_by_path: + msg = ( + f"{country_id} release names no certified dataset at path {path!r} " + f"for {role} regions" + ) + raise CatalogExtractionError(msg) + role_datasets[role] = names_by_path[path] + return role_datasets + + +def _checked_filter_value(reference: RegionReference, rule: RegionScopingRule) -> None: + """Require a filter value exactly when the rule filters, and valid list values.""" + value = reference.filter_value + if rule.filter_strategy is None: + if value is not None: + msg = ( + f"region {reference.code!r} has a filter value, but a national region " + "selects every household" + ) + raise CatalogExtractionError(msg) + return + if value is None: + msg = f"region {reference.code!r} has no filter value" + raise CatalogExtractionError(msg) + if value.startswith(("[", "{")): + try: + members = json.loads(value, parse_constant=_reject_constant) + except ValueError: + members = None + if not ( + isinstance(members, list) + and members + and all(isinstance(member, (str, int, float, bool)) for member in members) + ): + msg = f"region {reference.code!r} has an unsupported filter value" + raise CatalogExtractionError(msg) + + +def _reject_constant(constant: str) -> object: + msg = f"{constant} is not a filter value" + raise ValueError(msg) + + +def scope_regions( + country_id: str, + references: tuple[RegionReference, ...], + release_manifest: object, + *, + default_dataset: str, +) -> RegionScoping: + """Give every runnable reference region its certified dataset and filter. + + ``default_dataset`` is the release's own default. The release must certify + it, and the national region must use it. A region whose type has no + scoping rule is left out, and a runnable region may not sit under a region + that is left out. + """ + if not isinstance(release_manifest, ReleaseManifest): + msg = ( + f"{country_id} release manifest lacks datasets, region_datasets or " + "data_package" + ) + raise CatalogExtractionError(msg) + artifacts = certified_artifacts(country_id, release_manifest) + if default_dataset not in artifacts: + msg = f"{country_id} release does not certify {default_dataset}" + raise CatalogExtractionError(msg) + role_datasets = _role_datasets(country_id, release_manifest, artifacts) + national_dataset = role_datasets.get(NATIONAL_DATASET_ROLE, default_dataset) + if national_dataset != default_dataset: + msg = ( + f"{country_id} release routes national regions to {national_dataset}, " + f"not to its default dataset {default_dataset}" + ) + raise CatalogExtractionError(msg) + + omitted = { + reference.code: reference.region_type + for reference in references + if (country_id, reference.region_type) not in REGION_SCOPING_RULES + } + fallback_counts: Counter[str] = Counter() + regions: list[ScopedRegion] = [] + for reference in references: + if reference.code in omitted: + continue + if reference.parent_code in omitted: + msg = ( + f"region {reference.code!r} cannot be published because its parent " + f"{reference.parent_code!r} is left out" + ) + raise CatalogExtractionError(msg) + rule = REGION_SCOPING_RULES[country_id, reference.region_type] + _checked_filter_value(reference, rule) + dataset_name = role_datasets.get(rule.dataset_role) + if dataset_name is None: + if rule.requires_release_dataset: + msg = ( + f"the {country_id} release routes no dataset to " + f"{rule.dataset_role} regions, so region {reference.code!r} " + "cannot select its households" + ) + raise CatalogExtractionError(msg) + dataset_name = default_dataset + if rule.dataset_role != NATIONAL_DATASET_ROLE: + fallback_counts[reference.region_type] += 1 + regions.append( + ScopedRegion( + reference=reference, + dataset_name=dataset_name, + filter_strategy=rule.filter_strategy, + filter_field=rule.filter_field, + ) + ) + return RegionScoping( + regions=tuple(regions), + dataset_names=frozenset( + {default_dataset, *(region.dataset_name for region in regions)} + ), + certified_artifacts=artifacts, + fallback_counts=dict(sorted(fallback_counts.items())), + omitted_counts=dict(sorted(Counter(omitted.values()).items())), + ) + + +def filtered_artifacts(scoping: RegionScoping) -> tuple[CertifiedArtifact, ...]: + """Return, by name, the datasets the regions' row filters read.""" + names = { + region.dataset_name + for region in scoping.regions + if region.filter_strategy == ROW_FILTER + } + return tuple(scoping.certified_artifacts[name] for name in sorted(names)) diff --git a/policyengine_api/data/v2/catalog/spm_availability.py b/policyengine_api/data/v2/catalog/spm_availability.py new file mode 100644 index 000000000..398cb2a41 --- /dev/null +++ b/policyengine_api/data/v2/catalog/spm_availability.py @@ -0,0 +1,193 @@ +"""Record whether each published catalog version offers SPM settings. + +The simulation service's public ``GET /versions`` lists, under +``spm_capabilities``, the certified SPM capability of each PolicyEngine.py +bundle it runs: its contract version and pinned defaults. The Public API +already checks economy requests' SPM selections against it. A US catalog +version offers SPM when the service reports a canonical capability for its +PolicyEngine.py version; other countries never do. + +Publication records it after the catalog itself, for the installed +PolicyEngine.py version's catalog versions that have none yet, so a version +published before this existed gets it on the next publication run of that +PolicyEngine.py version; versions of other PolicyEngine.py versions keep none. +A bundle pins its SPM release, so a recorded value never changes. A US version +the service does not serve yet (its ``policyengine`` routes do not name it) +is left unrecorded rather than recorded as unavailable, and the next +publication run tries again; ``/v2/model-catalog`` offers no SPM settings for +it meanwhile. So is every version when the service's versions cannot be +read: an unreadable answer never fails the publication. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Any + +import httpx +import sqlalchemy as sa +from pydantic import ValidationError + +from policyengine_api.data.v2.catalog.publication_types import ( + CatalogPublicationError, +) +from policyengine_api.data.v2.models import TaxBenefitModel, TaxBenefitModelVersion +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from sqlalchemy import Connection, Engine + + from policyengine_api.data.v2.catalog.records import NormalizedCatalog + +LOGGER = logging.getLogger(__name__) +SIMULATION_ENTRYPOINT_URL = "SIMULATION_ENTRYPOINT_URL" +SPM_COUNTRY_ID = "us" +SPM_CONTRACT_VERSION = "canonical-spm-v1" +VERSIONS_TIMEOUT_SECONDS = 30.0 +UNAVAILABLE: dict[str, Any] = {"available": False} +MODELS = TaxBenefitModel.__table__ +MODEL_VERSIONS = TaxBenefitModelVersion.__table__ + + +def _served(policyengine_version: str, versions: dict[str, Any]) -> bool: + """Return whether the service's ``policyengine`` routes name the version.""" + routes = versions.get("policyengine") + if not isinstance(routes, dict): + return False + return policyengine_version != "latest" and ( + policyengine_version in routes or routes.get("latest") == policyengine_version + ) + + +def spm_availability( + country_id: str, policyengine_version: str, versions: object +) -> dict[str, Any] | None: + """Return ``{available, settings_schema, defaults}`` for one catalog version. + + Only a US version whose PolicyEngine.py version has a canonical capability + with pinned defaults (forecast artifact and scenario) offers SPM; any other + country, and a US version the service serves without one, is + ``{"available": False}``. A US version the service does not serve yet is + None: not known. + """ + if country_id != SPM_COUNTRY_ID: + return dict(UNAVAILABLE) + if not isinstance(versions, dict) or not _served(policyengine_version, versions): + return None + capabilities = versions.get("spm_capabilities") + capability = ( + capabilities.get(policyengine_version) + if isinstance(capabilities, dict) + else None + ) + if ( + not isinstance(capability, dict) + or set(capability) != {"contract_version", "defaults"} + or capability["contract_version"] != SPM_CONTRACT_VERSION + ): + return dict(UNAVAILABLE) + try: + defaults = SPMSelection.model_validate(capability["defaults"]) + except ValidationError: + return dict(UNAVAILABLE) + if not defaults.forecast_content_sha256 or not defaults.scenario: + return dict(UNAVAILABLE) + return { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": { + name: getattr(defaults, name) for name in SPMSelection.model_fields + }, + } + + +def simulation_entrypoint_url(environ: Mapping[str, str]) -> str: + """Return the simulation service's base URL; raise when it is not set.""" + url = environ.get(SIMULATION_ENTRYPOINT_URL, "").strip().rstrip("/") + if not url: + msg = f"{SIMULATION_ENTRYPOINT_URL} is required to record SPM availability" + raise V2ConfigurationError(msg) + return url + + +def fetch_simulation_versions( + base_url: str, *, client: httpx.Client | None = None +) -> object: + """Read the simulation service's public ``GET /versions`` answer.""" + url = f"{base_url.rstrip('/')}/versions" + try: + if client is None: + response = httpx.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + else: + response = client.get(url, timeout=VERSIONS_TIMEOUT_SECONDS) + response.raise_for_status() + return response.json() + except (httpx.HTTPError, ValueError) as error: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) from error + + +def record_spm_availability( + connection: Connection, + *, + model_name: str, + version: str, + availability: Mapping[str, Any], +) -> bool: + """Store a catalog version's SPM availability unless it already has one. + + Return whether the version was changed. + """ + model_ids = sa.select(MODELS.c.id).where(MODELS.c.name == model_name) + result = connection.execute( + sa.update(MODEL_VERSIONS) + .where( + MODEL_VERSIONS.c.model_id.in_(model_ids), + MODEL_VERSIONS.c.version == version, + MODEL_VERSIONS.c.spm_availability.is_(None), + ) + .values(spm_availability=dict(availability)) + ) + return bool(result.rowcount) + + +def record_published_spm_availability( + engine: Engine, + catalog: NormalizedCatalog, + environ: Mapping[str, str] | None = None, + *, + fetch_versions: Callable[[str], object] = fetch_simulation_versions, +) -> None: + """Record each published country's SPM availability for the catalog version. + + A US version the service does not serve yet stays unrecorded, and so does + every version when the service's versions cannot be read; the next + publication run tries again. Raise V2ConfigurationError without + ``SIMULATION_ENTRYPOINT_URL``. + """ + url = simulation_entrypoint_url(os.environ if environ is None else environ) + try: + versions = fetch_versions(url) + except CatalogPublicationError as error: + LOGGER.warning( + "SPM availability was not recorded; the next publication run " + "tries again: %s", + error, + ) + return + with engine.begin() as connection: + for country in catalog.countries: + availability = spm_availability( + country.country_id, catalog.policyengine_version, versions + ) + if availability is not None: + record_spm_availability( + connection, + model_name=country.model.name, + version=country.model_version.version, + availability=availability, + ) diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py index 07b88ad1a..f9df94461 100644 --- a/policyengine_api/data/v2/database.py +++ b/policyengine_api/data/v2/database.py @@ -8,6 +8,7 @@ from threading import Lock from typing import TYPE_CHECKING +from sqlalchemy import event from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -28,7 +29,7 @@ @dataclass class _V2EngineState: - """The process-owned v2 engine and Session factory. + """The process-owned v2 engine and Session factories. ``engine_pid`` is the ID of the process that created the engine, and ``engine_url_fingerprint`` identifies its target. ``_state_lock`` guards @@ -39,6 +40,7 @@ class _V2EngineState: engine_pid: int | None = None engine_url_fingerprint: tuple[object, ...] | None = None session_factory: sessionmaker[Session] | None = None + read_session_factory: sessionmaker[Session] | None = None _state_lock = Lock() @@ -82,6 +84,36 @@ def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) +class ReadOnlySessionError(RuntimeError): + """Raised when code tries to write rows through a v2 read session.""" + + +def _refuse_flush(*_flush_arguments: object) -> None: + msg = "v2 read sessions do not write rows" + raise ReadOnlySessionError(msg) + + +def build_v2_read_session_factory(engine: Engine) -> sessionmaker[Session]: + """Build a factory of Sessions for v2 metadata reads, run without a transaction. + + The sessions share ``engine``'s pool in autocommit mode, so psycopg sends + each statement on its own: no BEGIN before the first read and no ROLLBACK + when the session closes. Only the metadata reader uses them: each of its + reads is one statement on a published catalog version, which never + changes, so no common snapshot is needed. The sessions refuse to flush + added or changed rows; statements passed directly to ``execute`` are not + checked, so they must stay reads. The pool restores the ordinary mode + when a connection returns. + """ + factory = sessionmaker( + bind=engine.execution_options(isolation_level="AUTOCOMMIT"), + class_=Session, + expire_on_commit=False, + ) + event.listen(factory, "before_flush", _refuse_flush) + return factory + + def _discard_inherited_state(current_pid: int) -> None: """Drop pool state inherited through a process fork without touching parent FDs.""" if _engine_state.engine is not None: @@ -90,6 +122,7 @@ def _discard_inherited_state(current_pid: int) -> None: _engine_state.engine_pid = current_pid _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None def get_v2_engine(settings: V2DatabaseSettings | None = None) -> Engine: @@ -130,6 +163,20 @@ def get_v2_session_factory( return _engine_state.session_factory +def get_v2_read_session_factory( + settings: V2DatabaseSettings | None = None, +) -> sessionmaker[Session]: + """Return the process-owned read-only Session factory without opening one. + + See ``build_v2_read_session_factory``. + """ + engine = get_v2_engine(settings) + with _state_lock: + if _engine_state.read_session_factory is None: + _engine_state.read_session_factory = build_v2_read_session_factory(engine) + return _engine_state.read_session_factory + + def close_v2_database() -> None: """Dispose this process's v2 pool and forget its lazy configuration.""" with _state_lock: @@ -139,6 +186,7 @@ def close_v2_database() -> None: _engine_state.engine_pid = None _engine_state.engine_url_fingerprint = None _engine_state.session_factory = None + _engine_state.read_session_factory = None atexit.register(close_v2_database) diff --git a/policyengine_api/data/v2/metadata_validation.py b/policyengine_api/data/v2/metadata_validation.py index 7445269f1..ab60ab93a 100644 --- a/policyengine_api/data/v2/metadata_validation.py +++ b/policyengine_api/data/v2/metadata_validation.py @@ -3,8 +3,8 @@ from collections.abc import Iterable # These names belong only to the existing v1 database domain. Names that are -# intentionally shared between v1 and v2, such as `simulations` and -# `user_policies`, are excluded from this targeted rejection set. +# intentionally shared between v1 and v2, such as `simulations`, are excluded +# from this targeted rejection set. V1_ONLY_TABLES = frozenset( { "analysis", diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..77e019ec6 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "aeb5f3e8f662" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index c63655bb6..8089fc90a 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -31,6 +31,7 @@ Stage12RunStatus, Stage12SimulationRole, ) +from policyengine_api.data.v2.models.countries import Country # noqa: E402 from policyengine_api.data.v2.models.household_mappings import ( # noqa: E402 LegacyHouseholdMapping, ) @@ -40,6 +41,8 @@ HouseholdJobStatus, ) from policyengine_api.data.v2.models.metadata import ( # noqa: E402 + CatalogEntity, + CatalogEnumValue, Dataset, DatasetVersion, Parameter, @@ -99,9 +102,12 @@ "AggregateOutput", "AggregateType", "BudgetSummary", + "CatalogEntity", + "CatalogEnumValue", "ChangeAggregate", "CongressionalDistrictImpact", "ConstituencyImpact", + "Country", "Dataset", "DatasetVersion", "DecileImpact", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index 2c5cd21b1..a0fdfcdf1 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -51,10 +51,6 @@ class UserHouseholdAssociation(TimestampedModel, table=True): "country_id", name="uq_user_household_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_household_associations_country", - ), sa.CheckConstraint( "household_id IS NOT NULL OR legacy_household_id IS NOT NULL", name="ck_user_household_associations_household_ref", @@ -91,7 +87,11 @@ class UserHouseholdAssociation(TimestampedModel, table=True): ) household_id: UUID | None = Field(default=None, index=True) legacy_household_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) @@ -106,29 +106,25 @@ class UserPolicy(TimestampedModel, table=True): policy that users have saved cannot be deleted. """ - __tablename__ = "user_policies" + __tablename__ = "user_policy_associations" __table_args__ = ( sa.UniqueConstraint( "id", "country_id", - name="uq_user_policies_id_country", - ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_policies_country", + name="uq_user_policy_associations_id_country", ), sa.CheckConstraint( "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", - name="ck_user_policies_policy_ref", + name="ck_user_policy_associations_policy_ref", ), sa.CheckConstraint( "baseline IS NULL OR baseline IN ('current-law', 'current-policy')", - name="ck_user_policies_baseline", + name="ck_user_policy_associations_baseline", ), sa.ForeignKeyConstraint( ["policy_id", "country_id"], ["policies.id", "policies.country_id"], - name="fk_user_policies_policy_country", + name="fk_user_policy_associations_policy_country", ondelete="RESTRICT", ), sa.Index( @@ -157,7 +153,11 @@ class UserPolicy(TimestampedModel, table=True): ) policy_id: UUID | None = Field(default=None, index=True) legacy_policy_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) baseline: str | None = Field(default=None, max_length=16) @@ -185,10 +185,6 @@ class UserSimulationAssociation(TimestampedModel, table=True): "country_id", name="uq_user_simulation_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_simulation_associations_country", - ), sa.CheckConstraint( "simulation_id IS NOT NULL OR legacy_simulation_id IS NOT NULL", name="ck_user_simulation_associations_simulation_ref", @@ -214,7 +210,11 @@ class UserSimulationAssociation(TimestampedModel, table=True): ) simulation_id: UUID | None = Field(default=None, index=True) legacy_simulation_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) # Retired; see the module docstring. @@ -239,10 +239,6 @@ class UserReportAssociation(TimestampedModel, table=True): "country_id", name="uq_user_report_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_report_associations_country", - ), sa.CheckConstraint( "report_id IS NOT NULL OR legacy_report_id IS NOT NULL", name="ck_user_report_associations_report_ref", @@ -273,12 +269,16 @@ class UserReportAssociation(TimestampedModel, table=True): ) report_id: UUID | None = Field(default=None, index=True) legacy_report_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) user_policy_id: UUID | None = Field( default=None, - foreign_key="user_policies.id", + foreign_key="user_policy_associations.id", ondelete="SET NULL", index=True, ) @@ -286,7 +286,7 @@ class UserReportAssociation(TimestampedModel, table=True): # comparison was not current law; its name is the baseline's label. baseline_user_policy_id: UUID | None = Field( default=None, - foreign_key="user_policies.id", + foreign_key="user_policy_associations.id", ondelete="SET NULL", index=True, ) diff --git a/policyengine_api/data/v2/models/countries.py b/policyengine_api/data/v2/models/countries.py new file mode 100644 index 000000000..fc3ae8677 --- /dev/null +++ b/policyengine_api/data/v2/models/countries.py @@ -0,0 +1,37 @@ +"""Canonical SQLModel table recording what each served country supports. + +Which countries exist is decided in code: ``policyengine_api.supported_countries`` +lists them, and this table must hold exactly one row per listed country. A +row records the calculations its country supports: a household-only country +has ``household_enabled`` set and ``society_wide_enabled`` unset. + +``users.primary_country`` and the ``country_id`` of households, policies and +the four user association tables reference ``countries.id``. Other v2 country +columns do not: the mapping tables that pair v1 records with their v2 copies +keep their US-and-UK checks, and the simulation-domain tables, the comparison +run tables and the retired association ``country`` columns store unconstrained +text. +""" + +import sqlalchemy as sa +from sqlmodel import Field, SQLModel + + +class Country(SQLModel, table=True): + """One served country and the calculations it supports. + + A society-wide country must also be household capable. A country that a + user, household, policy or user association names cannot be deleted. + """ + + __tablename__ = "countries" + __table_args__ = ( + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name="ck_countries_society_wide_requires_household", + ), + ) + + id: str = Field(primary_key=True, max_length=2) + household_enabled: bool + society_wide_enabled: bool diff --git a/policyengine_api/data/v2/models/households.py b/policyengine_api/data/v2/models/households.py index 78e76b2ab..181266d9d 100644 --- a/policyengine_api/data/v2/models/households.py +++ b/policyengine_api/data/v2/models/households.py @@ -53,10 +53,6 @@ class Household(TimestampedModel, table=True): "content_hash", name="uq_households_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_households_country", - ), sa.CheckConstraint( "default_year IS NULL OR default_year BETWEEN 1900 AND 2200", name="ck_households_default_year", @@ -78,7 +74,11 @@ class Household(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) default_year: int | None = None household_data: dict[str, Any] = Field( sa_type=sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 8d91864df..5779d65b9 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -7,6 +7,7 @@ import sqlalchemy as sa from sqlmodel import Field, Relationship +from typing_extensions import TypedDict from policyengine_api.data.v2.models.base import ( IdentifiedModel, @@ -36,6 +37,28 @@ class RegionType(StrEnum): PLACE = "place" +class CatalogEntity(TypedDict): + """One entity of a catalog version's ``entities`` column. + + ``is_person`` marks the person entity; each version has exactly one. + """ + + key: str + plural: str + is_person: bool + + +class CatalogEnumValue(TypedDict): + """One value of a variable's ``enum_values`` column. + + ``value`` is the stored name and ``label`` the label the country package + declares, kept as declared (usually a string). + """ + + value: str + label: str | int | float | bool | None + + class TaxBenefitModel(TimestampedModel, table=True): """A country's tax-benefit model, unique by name. @@ -85,6 +108,16 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): description: str | None = None current_law_id: int metadata_time_periods: list[int] = Field(sa_type=sa.JSON) + # Catalog format 2 fields. Versions published before format 2 keep them + # null. + catalog_format: int | None = None + entities: list[CatalogEntity] | None = Field(default=None, sa_type=sa.JSON) + basic_inputs: list[str] | None = Field(default=None, sa_type=sa.JSON) + # Whether this version offers SPM settings, with their schema and the + # certified defaults, as the simulation service reports its bundle's SPM + # support: {available, settings_schema, defaults}. Recorded after the + # catalog is published, so a version briefly has none. + spm_availability: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) model: TaxBenefitModel = Relationship(back_populates="versions") variables: list["Variable"] = Relationship( @@ -111,8 +144,9 @@ class Region(TimestampedModel, table=True): """A region in one model version, unique by code within that version. Its default dataset must belong to the same version. Filtered regions - (``requires_filter``) also name the variable and value that select their - households. + (``requires_filter``) also name how they select their households: a + ``row_filter`` keeps the households whose ``filter_field`` column matches + ``filter_value``. """ __tablename__ = "regions" @@ -276,6 +310,19 @@ class Variable(IdentifiedModel, table=True): default_value: Any = Field(default=None, sa_type=sa.JSON) adds: list[str] | None = Field(default=None, sa_type=sa.JSON) subtracts: list[str] | None = Field(default=None, sa_type=sa.JSON) + # Catalog format 2 fields; null in versions published before it. A + # parameter path names the parameter whose dated value lists the variables + # this one adds or subtracts. + is_input_variable: bool | None = None + hidden_input: bool | None = None + module_name: str | None = Field(default=None, max_length=512) + adds_parameter_path: str | None = Field(default=None, max_length=512) + subtracts_parameter_path: str | None = Field(default=None, max_length=512) + enum_values: list[CatalogEnumValue] | None = Field(default=None, sa_type=sa.JSON) + # The unit (for example ``currency-USD``) and the period a value covers + # (``year``, ``month`` or ``eternity``), as the country package declares. + unit: str | None = Field(default=None, max_length=128) + definition_period: str | None = Field(default=None, max_length=32) tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", @@ -336,6 +383,9 @@ class Parameter(IdentifiedModel, table=True): description: str | None = None data_type: str | None = Field(default=None, max_length=128) unit: str | None = Field(default=None, max_length=128) + # Catalog format 2 fields; null in versions published before it. + economy: bool | None = None + household: bool | None = None tax_benefit_model_version_id: UUID = Field( foreign_key="tax_benefit_model_versions.id", ondelete="CASCADE", diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index 97e4a619e..f0da6e8cc 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -40,10 +40,6 @@ class Policy(TimestampedModel, table=True): "content_hash", name="uq_policies_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_policies_country", - ), sa.CheckConstraint( "canonicalization_version > 0", name="ck_policies_canonicalization_version", @@ -64,7 +60,11 @@ class Policy(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index 7cdbc91b9..69aef5642 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -104,7 +104,7 @@ class LegacyUserPolicyMapping(TimestampedModel, table=True): ), sa.ForeignKeyConstraint( ["user_policy_id", "country_id"], - ["user_policies.id", "user_policies.country_id"], + ["user_policy_associations.id", "user_policy_associations.country_id"], name="fk_legacy_user_policy_mappings_association_country", ondelete="CASCADE", ), diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index 85c0ea19a..f3b987123 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -44,10 +44,6 @@ class User(IdentifiedModel, table=True): __tablename__ = "users" __table_args__ = ( sa.UniqueConstraint("email", name="uq_users_email"), - sa.CheckConstraint( - "primary_country IN ('us', 'uk')", - name="ck_users_primary_country", - ), sa.CheckConstraint( "user_type IN ('unauthenticated', 'standard')", name="ck_users_user_type", @@ -61,7 +57,12 @@ class User(IdentifiedModel, table=True): first_name: str | None = Field(default=None, max_length=255) last_name: str | None = Field(default=None, max_length=255) email: str | None = Field(default=None, max_length=320, index=True) - primary_country: str | None = Field(default=None, max_length=2) + primary_country: str | None = Field( + default=None, + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) # The server default keeps inserts by earlier Public API revisions, which # do not map this column, creating standard users. user_type: str = Field( diff --git a/policyengine_api/data/v2/policy_migration_qualification.py b/policyengine_api/data/v2/policy_migration_qualification.py index beb03db24..d353226b1 100644 --- a/policyengine_api/data/v2/policy_migration_qualification.py +++ b/policyengine_api/data/v2/policy_migration_qualification.py @@ -10,12 +10,12 @@ from alembic.config import Config from alembic.migration import MigrationContext from alembic.script import ScriptDirectory -from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy import Connection, Engine, func, inspect, select, table from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.pool import NullPool from sqlmodel import create_engine -from policyengine_api.data.v2.models import ParameterValue, Policy, UserPolicy +from policyengine_api.data.v2.models import ParameterValue, Policy from policyengine_api.data.v2.settings import ( V2ConfigurationError, V2DatabaseSettings, @@ -26,6 +26,10 @@ from collections.abc import Callable, Collection, Mapping POLICY_MIGRATION_REVISION = "711ec2f0a5a5" +# The saved-policy table's name while the policy revision is pending. A later +# revision renames it to ``user_policy_associations``, so counting through +# the current model would query a table such a database does not have yet. +SAVED_POLICY_TABLE_AT_POLICY_REVISION = "user_policies" V2_ALEMBIC_CONFIG = "alembic-v2.ini" @@ -123,8 +127,15 @@ def table_exists(name: str) -> bool: if table_exists("parameter_values") else 0, user_policies=( - int(executor.scalar(select(func.count()).select_from(UserPolicy)) or 0) - if table_exists("user_policies") + int( + executor.scalar( + select(func.count()).select_from( + table(SAVED_POLICY_TABLE_AT_POLICY_REVISION) + ) + ) + or 0 + ) + if table_exists(SAVED_POLICY_TABLE_AT_POLICY_REVISION) else 0 ), ) diff --git a/policyengine_api/data/v2/privilege_reconciliation.py b/policyengine_api/data/v2/privilege_reconciliation.py index d9d0cacec..33af3896b 100644 --- a/policyengine_api/data/v2/privilege_reconciliation.py +++ b/policyengine_api/data/v2/privilege_reconciliation.py @@ -6,8 +6,10 @@ - ``apply`` reads the managed roles' current grants, grants every declared privilege a role lacks, and reports privileges the manifest doesn't declare. - It never revokes, and it changes nothing when a declared role or table is - missing. Running it again grants nothing more. + It never revokes, and it changes nothing when a declared role, table or view + is missing. A declared table must exist as a table and a declared temporary + view as a view; a relation of the other kind counts as missing. Running it + again grants nothing more. - ``verify`` checks every declared privilege with ``has_table_privilege`` (rule ``api.v2.deployed-privileges-match``) and fails on any that is missing. @@ -42,6 +44,7 @@ ) from policyengine_api.data.v2.privileges import ( PERSISTENT_ENVIRONMENTS, + TEMPORARY_VIEW_TABLES, TablePrivilege, declared_privileges, ) @@ -56,6 +59,8 @@ GITHUB_STEP_SUMMARY = "GITHUB_STEP_SUMMARY" SCHEMA = "public" SAFE_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") +# The manifest's temporary views; every other declared name is a table. +DECLARED_VIEWS: frozenset[str] = frozenset(TEMPORARY_VIEW_TABLES) class PrivilegeReconciliationError(RuntimeError): @@ -94,11 +99,20 @@ class ReconciliationPlan: undeclared: tuple[PrivilegeEntry, ...] missing_roles: tuple[str, ...] missing_tables: tuple[str, ...] + missing_views: tuple[str, ...] = () @property def can_apply(self) -> bool: - """Whether every declared role and table exists, so grants may run.""" - return not self.missing_roles and not self.missing_tables + """Whether every declared role, table and view exists, so grants may run.""" + return not (self.missing_roles or self.missing_tables or self.missing_views) + + +@dataclass(frozen=True) +class PublicRelations: + """The tables and views in the public schema, by name.""" + + tables: frozenset[str] + views: frozenset[str] def plan_reconciliation( @@ -106,21 +120,30 @@ def plan_reconciliation( granted: Mapping[str, Mapping[str, Collection[str]]], *, existing_roles: Collection[str], - existing_tables: Collection[str], + existing_relations: PublicRelations, + declared_views: Collection[str] = (), ) -> ReconciliationPlan: """Compare declared privileges with what the database grants. ``granted`` holds the managed roles' direct grants, by role then table. + Names in ``declared_views`` must exist as views, and every other declared + name as a table. """ missing_roles = tuple(sorted(set(declared) - set(existing_roles))) - declared_tables = {table for tables in declared.values() for table in tables} - missing_tables = tuple(sorted(declared_tables - set(existing_tables))) + declared_names = {table for tables in declared.values() for table in tables} + missing_views = tuple( + sorted((declared_names & set(declared_views)) - existing_relations.views) + ) + missing_tables = tuple( + sorted(declared_names - set(declared_views) - existing_relations.tables) + ) + missing_relations = {*missing_tables, *missing_views} grants = sorted( PrivilegeEntry(role, table, privilege.value) for role, tables in declared.items() if role not in missing_roles for table, privileges in tables.items() - if table not in missing_tables + if table not in missing_relations for privilege in privileges if privilege.value not in granted.get(role, {}).get(table, ()) ) @@ -140,6 +163,7 @@ def plan_reconciliation( undeclared=tuple(undeclared), missing_roles=missing_roles, missing_tables=missing_tables, + missing_views=missing_views, ) @@ -214,6 +238,12 @@ def render_summary( + ", ".join(f"`{table}`" for table in plan.missing_tables), "", ] + if plan.missing_views: + lines += [ + "Declared views missing from the database (nothing was granted): " + + ", ".join(f"`{view}`" for view in plan.missing_views), + "", + ] grant_heading = "Granted" if applied else "Would grant" lines += [f"### {grant_heading}", ""] if plan.grants: @@ -270,15 +300,17 @@ def render_summary( "SELECT rolname FROM pg_roles WHERE rolname IN :roles" ).bindparams(bindparam("roles", expanding=True)) -_PUBLIC_TABLES = text( +_PUBLIC_RELATIONS = text( """ - SELECT table_class.relname - FROM pg_class AS table_class - JOIN pg_namespace AS table_namespace - ON table_namespace.oid = table_class.relnamespace - WHERE table_namespace.nspname = :schema AND table_class.relkind IN ('r', 'p') + SELECT relation_class.relname, relation_class.relkind + FROM pg_class AS relation_class + JOIN pg_namespace AS relation_namespace + ON relation_namespace.oid = relation_class.relnamespace + WHERE relation_namespace.nspname = :schema + AND relation_class.relkind IN ('r', 'p', 'v') """ ) +_TABLE_KINDS = frozenset({"r", "p"}) _DEFAULT_PRIVILEGES = text( """ @@ -311,9 +343,17 @@ def read_existing_roles(connection: Connection, roles: Collection[str]) -> set[s return set(connection.execute(_EXISTING_ROLES, {"roles": list(roles)}).scalars()) -def read_public_tables(connection: Connection) -> set[str]: - """Return every table in the public schema.""" - return set(connection.execute(_PUBLIC_TABLES, {"schema": SCHEMA}).scalars()) +def read_public_relations(connection: Connection) -> PublicRelations: + """Return the tables and the views in the public schema. + + The manifest declares privileges on temporary views as it does on tables, + and ``GRANT ... ON TABLE`` grants on a view too. + """ + rows = connection.execute(_PUBLIC_RELATIONS, {"schema": SCHEMA}).all() + return PublicRelations( + tables=frozenset(name for name, kind in rows if kind in _TABLE_KINDS), + views=frozenset(name for name, kind in rows if kind == "v"), + ) def read_granted_privileges( @@ -363,17 +403,22 @@ def create_missing_roles(connection: Connection, roles: Iterable[str]) -> list[s def apply_declared_privileges( connection: Connection, declared: Mapping[str, Mapping[str, frozenset[TablePrivilege]]], + *, + declared_views: Collection[str] = DECLARED_VIEWS, ) -> tuple[ReconciliationPlan, list[DefaultPrivilege]]: """Grant every declared privilege a role lacks; never revoke. - Grants nothing when a declared role or table is missing. + Grants nothing when a declared role, table or view is missing, including a + declared view that exists as a table or a declared table that exists as a + view. """ roles = list(declared) plan = plan_reconciliation( declared, read_granted_privileges(connection, roles), existing_roles=read_existing_roles(connection, roles), - existing_tables=read_public_tables(connection), + existing_relations=read_public_relations(connection), + declared_views=declared_views, ) if plan.can_apply: for statement in grant_statements( @@ -386,16 +431,24 @@ def apply_declared_privileges( def check_declared_privileges( connection: Connection, declared: Mapping[str, Mapping[str, frozenset[TablePrivilege]]], + *, + declared_views: Collection[str] = DECLARED_VIEWS, ) -> list[PrivilegeEntry]: """Return every declared privilege a role doesn't effectively hold. - A missing role or table counts as missing every privilege declared for it. + A missing role, table or view counts as missing every privilege declared + for it; so does a declared view that exists as a table, or the reverse. """ existing_roles = read_existing_roles(connection, list(declared)) - existing_tables = read_public_tables(connection) + relations = read_public_relations(connection) + + def exists_as_declared(name: str) -> bool: + if name in declared_views: + return name in relations.views + return name in relations.tables def has_privilege(role: str, table: str, privilege: str) -> bool: - if role not in existing_roles or table not in existing_tables: + if role not in existing_roles or not exists_as_declared(table): return False return bool( connection.execute( diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 8f8c34515..66ea50fef 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -10,7 +10,9 @@ ``api.v2.runtime-privileges-tested-as-runtime-role``). Every table in the v2 SQLModel metadata, plus ``alembic_version``, has an entry -for every managed role, so "no access" is a decision, not an omission. +for every managed role, so "no access" is a decision, not an omission. A +temporary view under a renamed table's earlier name (``TEMPORARY_VIEW_TABLES``) +is declared with its table's privileges. Roles and what decides their privileges: @@ -132,10 +134,13 @@ def _access( # Stage 10 saved-policy mirror. "legacy_user_mappings": _access(runtime=READ_WRITE), "legacy_user_policy_mappings": _access(runtime=READ_WRITE), + # Served countries; the migration that creates the table inserts its rows, + # and the Public API reads them. + "countries": _access(runtime=READ), # Users and their saved work (Stage 12A). "user_household_associations": _access(runtime=READ_WRITE), "user_identities": _access(runtime=READ_WRITE), - "user_policies": _access(runtime=READ_WRITE), + "user_policy_associations": _access(runtime=READ_WRITE), "user_report_associations": _access(runtime=READ_WRITE), "user_simulation_associations": _access(runtime=READ_WRITE), "users": _access(runtime=READ_WRITE), @@ -145,6 +150,17 @@ def _access( } +# Temporary views under a renamed table's earlier name, each mapped to the +# table it shows. The Public API release that still serves during a deploy, or +# after a rollback to it, reads and writes the table through the view, so every +# role holds on the view what it holds on the table. A later revision drops +# each view, and its entry here goes with it. +TEMPORARY_VIEW_TABLES: Final[dict[str, str]] = { + # Revision fa7fa1035444 renamed user_policies to user_policy_associations. + "user_policies": "user_policy_associations", +} + + class InvalidDeploymentEnvironmentError(ValueError): """Raised for a deployment environment that can't name a Stage 12 role.""" @@ -167,12 +183,23 @@ def stage12_role(deployment_environment: str) -> str: def declared_privileges( deployment_environment: str, ) -> dict[str, dict[str, frozenset[TablePrivilege]]]: - """Return each managed role's declared privileges, keyed by role then table.""" + """Return each managed role's declared privileges, keyed by role then table. + + Each temporary view is declared with its table's privileges. + """ stage12 = stage12_role(deployment_environment) + access_by_relation = TABLE_ACCESS | { + view: TABLE_ACCESS[table] for view, table in TEMPORARY_VIEW_TABLES.items() + } return { - RUNTIME_ROLE: {table: access.runtime for table, access in TABLE_ACCESS.items()}, + RUNTIME_ROLE: { + relation: access.runtime for relation, access in access_by_relation.items() + }, CATALOG_WRITER_ROLE: { - table: access.catalog_writer for table, access in TABLE_ACCESS.items() + relation: access.catalog_writer + for relation, access in access_by_relation.items() + }, + stage12: { + relation: access.stage12 for relation, access in access_by_relation.items() }, - stage12: {table: access.stage12 for table, access in TABLE_ACCESS.items()}, } diff --git a/policyengine_api/data/v2/user_association_migration_qualification.py b/policyengine_api/data/v2/user_association_migration_qualification.py index 9d7aad2a5..fac2b0d00 100644 --- a/policyengine_api/data/v2/user_association_migration_qualification.py +++ b/policyengine_api/data/v2/user_association_migration_qualification.py @@ -24,14 +24,13 @@ from alembic.config import Config from alembic.migration import MigrationContext from alembic.script import ScriptDirectory -from sqlalchemy import Connection, Engine, func, inspect, select +from sqlalchemy import Connection, Engine, func, inspect, select, table from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.pool import NullPool from sqlmodel import create_engine from policyengine_api.data.v2.models import ( UserHouseholdAssociation, - UserPolicy, UserReportAssociation, UserSimulationAssociation, ) @@ -44,7 +43,14 @@ if TYPE_CHECKING: from collections.abc import Callable, Collection, Mapping + from sqlalchemy.sql.expression import TableClause + USER_ASSOCIATION_MIGRATION_REVISION = "fbb7488ebe8a" +# The saved-policy table's name at this revision, and the name a later +# revision gives it. The command runs before every upgrade, so it counts the +# table under whichever name the database has. +SAVED_POLICY_TABLE_AT_RESHAPE = "user_policies" +SAVED_POLICY_TABLE_AFTER_RENAME = "user_policy_associations" V2_ALEMBIC_CONFIG = "alembic-v2.ini" @@ -122,13 +128,20 @@ def read_user_association_counts( ) -> UserAssociationCounts: """Count every row in the four user-association tables.""" - def count(model: type, table_name: str) -> int: + def count(source: type | TableClause, table_name: str) -> int: if existing_tables is not None and table_name not in existing_tables: return 0 - return int(executor.scalar(select(func.count()).select_from(model)) or 0) + return int(executor.scalar(select(func.count()).select_from(source)) or 0) + + saved_policy_table = ( + SAVED_POLICY_TABLE_AFTER_RENAME + if existing_tables is not None + and SAVED_POLICY_TABLE_AFTER_RENAME in existing_tables + else SAVED_POLICY_TABLE_AT_RESHAPE + ) return UserAssociationCounts( - user_policies=count(UserPolicy, "user_policies"), + user_policies=count(table(saved_policy_table), saved_policy_table), user_household_associations=count( UserHouseholdAssociation, "user_household_associations" ), diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..2b9466f5a 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -10,18 +10,29 @@ from fastapi import Depends from fastapi.security import APIKeyHeader, HTTPAuthorizationCredentials, HTTPBearer +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.fastapi_routes.v2.user_auth import authenticate_account_request from policyengine_api.libs.user_auth import UNAUTHENTICATED_USER_HEADER from policyengine_api.readiness import is_ready +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from uuid import UUID from policyengine_api.json_types import JSONObject from policyengine_api.services.v2.associations.types import ( LegacyReferenceChecker, ) + from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + HouseholdCalculationRead, + ) from policyengine_api.services.v2.households.types import ( HouseholdPage, HouseholdRead, @@ -34,10 +45,13 @@ ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -45,6 +59,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.policies.types import ( @@ -190,11 +205,12 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: - """Return parameters in name order, optionally filtered by ``search``. + """Return parameters in name order, optionally filtered. - ``search`` is a case-insensitive substring of the name, label or description. + The filter's ``search`` is a case-insensitive substring of the name, + label or description; its ``names`` keep only those exact names. """ ... @@ -231,9 +247,10 @@ def list_parameter_values( offset: int = 0, limit: int = 100, ) -> MetadataPageResult[MetadataCanonicalParameterValue]: - """Return canonical values, optionally for one parameter. + """Return canonical values, optionally for one or several parameters. - With the filter's ``current``, only values in effect on the UTC day of its + The filter's ``parameter_id`` or ``parameter_names`` select parameters. + With its ``current``, only values in effect on the UTC day of its ``now`` (default: the present) are returned. """ ... @@ -311,6 +328,54 @@ def get_economy_options( """ ... + def get_model_catalog( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + These are the model and model version, entities, basic inputs, regions, + logical input datasets and economy choices, which are empty for a + catalog without regions. Raise ``MetadataCatalogUnavailableError`` when + a catalog with regions lacks what ``get_economy_options`` needs. + """ + ... + + def get_catalog_variables( + self, + country_id: str, + policyengine_version: str | None = None, + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also carries the current-law values of each parameter a variable's + adds or subtracts names. + """ + ... + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values; the result also carries the + stored labels of their ancestor nodes and the names the version lacks. + """ + ... + + +class V2CountryResourceService(Protocol): + """Route-independent countries list read for one request.""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and its capability, ordered by ID.""" + ... + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" @@ -383,6 +448,29 @@ def list_households( ... +class V2HouseholdCalculationResourceService(Protocol): + """Route-independent household calculation operations for one request.""" + + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationRead: + """Send a stored household's calculation to the simulation service. + + Raise ``HouseholdCalculationCountryError`` for a country without + household calculations, ``HouseholdNotFoundError`` or + ``PolicyNotFoundError`` for a record not stored in that country, and + the simulation service errors when it cannot calculate. + """ + ... + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation as it stands on the simulation service. + + Raise ``HouseholdCalculationNotFoundError`` when it has none. + """ + ... + + class V2UserPolicyResourceService(Protocol): """Route-independent user-policy operations for one request.""" @@ -727,9 +815,21 @@ def _running_policyengine_version() -> str: return importlib_metadata.version("policyengine") +@lru_cache(maxsize=1) +def _process_selected_catalogs() -> SelectedCatalogStore: + """Return the catalog versions this process's metadata reads have selected.""" + return SelectedCatalogStore() + + +@lru_cache(maxsize=1) +def _process_metadata_response_cache() -> MetadataResponseCache: + """Return the whole catalog-version responses this process keeps.""" + return MetadataResponseCache() + + def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.data.v2.database import ( - get_v2_session_factory, + get_v2_read_session_factory, ) from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, @@ -737,11 +837,24 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: from policyengine_api.services.v2.metadata.services import V2MetadataService return V2MetadataService( - MetadataDatabaseSession(get_v2_session_factory()()), + MetadataDatabaseSession(get_v2_read_session_factory()()), running_policyengine_version=_running_policyengine_version(), + selected_catalogs=_process_selected_catalogs(), ) +def _default_v2_country_service_factory() -> V2CountryResourceService: + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -771,6 +884,25 @@ def _default_v2_household_service_factory() -> V2HouseholdResourceService: ) +def _default_v2_household_calculation_service_factory() -> ( + V2HouseholdCalculationResourceService +): + from policyengine_api.services.v2.household_calculations.services import ( # noqa: PLC0415 # loaded on the first calculation request only + V2HouseholdCalculationService, + ) + from policyengine_api.services.v2.household_calculations.simulation_client import ( # noqa: PLC0415 # imports the generated simulation client on first use only + process_simulation_household_client, + ) + + return V2HouseholdCalculationService( + countries=_default_v2_country_service_factory(), + households=_default_v2_household_service_factory(), + policies=_default_v2_policy_service_factory(), + catalog_reader_factory=_default_v2_metadata_reader_factory, + simulation_factory=process_simulation_household_client, + ) + + _user_bearer_scheme = HTTPBearer( auto_error=False, scheme_name="UserAccessToken", @@ -917,8 +1049,12 @@ class NativeRouteDependencies: metadata_reader_factory: Callable[[], MetadataReader] specification_provider: Callable[[], JSONObject] v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None + v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None + v2_household_calculation_service_factory: ( + Callable[[], V2HouseholdCalculationResourceService] | None + ) = None v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( None ) @@ -932,6 +1068,9 @@ class NativeRouteDependencies: None ) v2_account_service_factory: Callable[[], V2AccountResourceService] | None = None + # Whole catalog-version responses (/v2/model-catalog and its variables) are + # kept here once built; None reads them on every request. + v2_metadata_response_cache: MetadataResponseCache | None = None # User-owned routes attach this at router level. The default validates the # Auth0 bearer token, or else accepts the unauthenticated-user header, and # fails closed (401/403/503); tests replace it with a callable returning a @@ -947,8 +1086,13 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_metadata_response_cache=_process_metadata_response_cache(), + v2_country_service_factory=_default_v2_country_service_factory, v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, + v2_household_calculation_service_factory=( + _default_v2_household_calculation_service_factory + ), v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), v2_user_household_service_factory=( _default_v2_user_household_service_factory diff --git a/policyengine_api/fastapi_routes/v2/countries/__init__.py b/policyengine_api/fastapi_routes/v2/countries/__init__.py new file mode 100644 index 000000000..3d7f8eaf0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for the v2 countries list.""" diff --git a/policyengine_api/fastapi_routes/v2/countries/response_models.py b/policyengine_api/fastapi_routes/v2/countries/response_models.py new file mode 100644 index 000000000..3ea3c7ee8 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/response_models.py @@ -0,0 +1,71 @@ +"""Strict HTTP response models for the native v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import BaseModel, ConfigDict + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse + +if TYPE_CHECKING: + from policyengine_api.services.v2.countries.types import CountryRead + + +class StrictCountryAPIModel(BaseModel): + """Strict response base with dataclass conversion support.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + +class CountryItem(StrictCountryAPIModel): + """One served country and the calculations it supports. + + ``id`` is the value other v2 resources take as ``country_id``. + ``household_enabled`` means household calculations and saved households + are available; ``society_wide_enabled`` means society-wide reports are, + and is only set for a country that is also household enabled. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool + + @classmethod + def from_read(cls, country: CountryRead) -> CountryItem: + """Build the response item from the service read record.""" + return cls.model_validate(country) + + +class CountryListResult(StrictCountryAPIModel): + """Every served country, ordered by ``id``.""" + + items: list[CountryItem] + + +class CountryListResponse(StrictCountryAPIModel): + """Success envelope: ``status`` "ok", a null ``message`` and the countries.""" + + status: Literal["ok"] = "ok" + message: None = None + result: CountryListResult + + +COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 405: { + "model": V2ErrorResponse, + "description": "The countries list supports GET only.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request has query parameters; the list takes none.", + }, + 500: { + "model": V2ErrorResponse, + "description": "The countries query failed internally.", + }, + 503: { + "model": V2ErrorResponse, + "description": "The v2 database is not configured or cannot be reached.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/countries/routes.py b/policyengine_api/fastapi_routes/v2/countries/routes.py new file mode 100644 index 000000000..279aaf54f --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/routes.py @@ -0,0 +1,84 @@ +"""Native FastAPI route for the v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import APIRouter, Depends +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_country_service_factory, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.countries.response_models import ( + COUNTRY_ERROR_RESPONSES, + CountryItem, + CountryListResponse, + CountryListResult, +) +from policyengine_api.fastapi_routes.v2.errors import V2ErrorMapping, run_v2_operation +from policyengine_api.query_parameters import CountryListQuery + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2CountryResourceService, + ) + + +CountryListQueryDependency = Annotated[ + CountryListQuery, Depends(query_dependency(CountryListQuery)) +] + +COUNTRY_ERROR_MAPPINGS = ( + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 countries are unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2CountryResourceService]: + if dependencies.v2_country_service_factory is not None: + return dependencies.v2_country_service_factory + return _default_v2_country_service_factory + + +def build_v2_country_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the countries route without opening a database connection.""" + router = APIRouter(prefix="/v2", responses=COUNTRY_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.get( + "/countries", + response_model=CountryListResponse, + summary="List the served countries and the calculations each supports", + ) + def list_countries( + _query: CountryListQueryDependency, + ) -> CountryListResponse | JSONResponse: + def read() -> CountryListResponse: + countries = service_factory().list_countries() + return CountryListResponse( + result=CountryListResult( + items=[CountryItem.from_read(country) for country in countries] + ) + ) + + return run_v2_operation( + read, + COUNTRY_ERROR_MAPPINGS, + fallback_message="V2 countries query failed", + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py new file mode 100644 index 000000000..59e3798f7 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Native FastAPI routes calculating stored v2 households.""" diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py new file mode 100644 index 000000000..eb78c9236 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/request_models.py @@ -0,0 +1,27 @@ +"""Strict HTTP request models for v2 household calculations.""" + +from __future__ import annotations + +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, +) + + +class HouseholdCalculationRequest(HouseholdCalculationInput): + """A stored household to calculate under a stored policy, or current law. + + ``household_id`` and ``policy_id`` name records of ``country_id`` stored + through ``/v2/households`` and ``/v2/policies``; without ``policy_id`` the + household is calculated under current law, for the household's + ``default_year``. Each group in ``axes`` varies its inputs together across + ``count`` points; several groups form a grid of at most 10,000 points. + ``add_missing_variables`` adds every variable the household lacks before + calculating. An identical calculation that succeeded in the last 28 days + is returned again instead of being repeated. + """ + + def to_service_input(self) -> HouseholdCalculationInput: + """Convert the request to the calculation service's input.""" + return HouseholdCalculationInput.model_validate( + self.model_dump(mode="python", exclude_unset=True) + ) diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py new file mode 100644 index 000000000..d1705fab7 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/response_models.py @@ -0,0 +1,117 @@ +"""Strict HTTP response models for v2 household calculations.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Any, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, JsonValue + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse + +if TYPE_CHECKING: + from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationRead, + ) + + +class StrictHouseholdCalculationAPIModel(BaseModel): + """Base for calculation responses; rejects unknown fields and NaN or infinity.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, from_attributes=True) + + +class HouseholdCalculationOutput(StrictHouseholdCalculationAPIModel): + """A succeeded calculation's household. + + ``household`` is the stored household in the situation shape (people and + groups keyed by their record IDs) with each calculated value filled in by + period. ``warnings`` lists inputs the country model ignored. In the US, + ``spm_config`` is the SPM selection the calculation applied and + ``spm_provenance`` its receipt; both are null elsewhere. + ``policyengine_version`` is the PolicyEngine.py version that calculated it. + """ + + household: dict[str, JsonValue] + warnings: list[str] + spm_config: dict[str, JsonValue] | None + spm_provenance: dict[str, JsonValue] | None + policyengine_version: str | None + + +class HouseholdCalculationError(StrictHouseholdCalculationAPIModel): + """Why a calculation failed; ``detail`` holds the calculation's own error.""" + + code: str + message: str + detail: dict[str, JsonValue] | None + + +class HouseholdCalculationItem(StrictHouseholdCalculationAPIModel): + """A household calculation and its outcome. + + ``status`` moves from ``pending`` through ``dispatching`` and ``running`` + to ``succeeded``, with ``result``, or ``failed``, with ``error``. Poll + ``GET /v2/household-calculations/{calculation_id}`` until it is one of + those two. ``catalog_version_id`` is the catalog version calculated. + """ + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + status: Literal["pending", "dispatching", "running", "succeeded", "failed"] + created_at: datetime + completed_at: datetime | None + result: HouseholdCalculationOutput | None + error: HouseholdCalculationError | None + + @classmethod + def from_read( + cls, calculation: HouseholdCalculationRead + ) -> HouseholdCalculationItem: + """Build the response item from the service's calculation.""" + return cls.model_validate(calculation) + + +class HouseholdCalculationResponse(StrictHouseholdCalculationAPIModel): + """Success envelope: ``status`` "ok", a null ``message`` and the calculation.""" + + status: Literal["ok"] = "ok" + message: None = None + result: HouseholdCalculationItem + + +HOUSEHOLD_CALCULATION_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 404: { + "model": V2ErrorResponse, + "description": "The household, policy or calculation does not exist.", + }, + 422: { + "model": V2ErrorResponse, + "description": ( + "The request does not match the schema, the country does not offer " + "household calculations, the household has no default year, or the " + "simulation service refuses the caller's data, with its reason in " + "the message: a policy changing a parameter the catalog version " + "lacks, a stored policy value or household it cannot calculate " + "(such as a group no person belongs to), or an invalid option." + ), + }, + 500: { + "model": V2ErrorResponse, + "description": "The household calculation request failed unexpectedly.", + }, + 503: { + "model": V2ErrorResponse, + "description": ( + "The simulation service, the country's catalog version or v2 " + "storage is unavailable; the simulation service's answer cannot " + "be read; or it refuses a reference the Public API has already " + "checked, or has no v2 worker for the catalog version." + ), + }, +} diff --git a/policyengine_api/fastapi_routes/v2/household_calculations/routes.py b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py new file mode 100644 index 000000000..019ba1c41 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/household_calculations/routes.py @@ -0,0 +1,153 @@ +"""Native FastAPI routes submitting stored households for calculation, and polling them. + +The calculation itself runs on the simulation service, which stores every +calculation as a simulation run; a calculation not saved to an account has no +report. The routes need no account. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, TypeVar +from uuid import UUID + +from fastapi import APIRouter +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + UnsupportedPreviewCountryError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_household_calculation_service_factory, +) +from policyengine_api.fastapi_routes.v2.errors import V2ErrorMapping, run_v2_operation +from policyengine_api.fastapi_routes.v2.household_calculations.request_models import ( + HouseholdCalculationRequest, +) +from policyengine_api.fastapi_routes.v2.household_calculations.response_models import ( + HOUSEHOLD_CALCULATION_ERROR_RESPONSES, + HouseholdCalculationItem, + HouseholdCalculationResponse, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2HouseholdCalculationResourceService, + ) + +OperationT = TypeVar("OperationT") + +HOUSEHOLD_CALCULATION_ERROR_MAPPINGS = ( + V2ErrorMapping( + ( + HouseholdCalculationCountryError, + HouseholdCalculationYearError, + SimulationServiceInvalidRequestError, + ), + 422, + ), + V2ErrorMapping( + ( + HouseholdNotFoundError, + PolicyNotFoundError, + HouseholdCalculationNotFoundError, + ), + 404, + ), + V2ErrorMapping( + (SimulationServiceUnavailableError, SimulationServiceRefusedError), 503 + ), + V2ErrorMapping( + ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, + UnsupportedPreviewCountryError, + ), + 503, + "The country's catalog version is unavailable", + ), + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 household calculation storage is unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2HouseholdCalculationResourceService]: + if dependencies.v2_household_calculation_service_factory is not None: + return dependencies.v2_household_calculation_service_factory + return _default_v2_household_calculation_service_factory + + +def _calculation_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + return run_v2_operation( + operation, + HOUSEHOLD_CALCULATION_ERROR_MAPPINGS, + fallback_message="V2 household calculation failed", + ) + + +def build_v2_household_calculation_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the household calculation routes without opening any connection.""" + router = APIRouter(prefix="/v2", responses=HOUSEHOLD_CALCULATION_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.post( + "/household-calculations", + response_model=HouseholdCalculationResponse, + status_code=202, + summary="Calculate a stored household under a stored policy", + ) + def submit_household_calculation( + body: HouseholdCalculationRequest, + ) -> HouseholdCalculationResponse | JSONResponse: + def submit() -> HouseholdCalculationResponse: + calculation = service_factory().submit_calculation(body.to_service_input()) + return HouseholdCalculationResponse( + result=HouseholdCalculationItem.from_read(calculation) + ) + + return _calculation_operation(submit) + + @router.get( + "/household-calculations/{calculation_id}", + response_model=HouseholdCalculationResponse, + summary="Read a household calculation and its result", + ) + def get_household_calculation( + calculation_id: UUID, + ) -> HouseholdCalculationResponse | JSONResponse: + def read() -> HouseholdCalculationResponse: + calculation = service_factory().get_calculation(calculation_id) + return HouseholdCalculationResponse( + result=HouseholdCalculationItem.from_read(calculation) + ) + + return _calculation_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py new file mode 100644 index 000000000..968c802c4 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/catalog_routes.py @@ -0,0 +1,124 @@ +"""API v2 routes that serve a catalog version in parts loaded on demand.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fastapi import APIRouter +from starlette.responses import JSONResponse, Response + +from policyengine_api.fastapi_routes.v2.metadata.common import ( + METADATA_RESPONSES, + CacheableMetadataRoute, + read_resource, + read_whole_version_resource, +) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogParameterQueryDependency, + CatalogQueryDependency, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataCatalogParametersResponse, + MetadataCatalogVariablesResponse, + MetadataModelCatalogResponse, + MetadataResourceSuccessResponse, +) +from policyengine_api.services.v2.metadata.types import ( + MetadataCatalogVariablesResult, + MetadataModelCatalogResult, +) + +if TYPE_CHECKING: + from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies + + +def build_v2_model_catalog_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the routes that serve one catalog version in parts. + + Each request opens and closes its own metadata reader, so building the + router touches no database. + """ + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) + + @router.get( + "/model-catalog", + response_model=MetadataModelCatalogResponse, + summary="Get the small parts of one PolicyEngine.py catalog version", + description=( + "Returns the selected catalog version's model, model version, " + "entities, basic inputs, regions, logical input datasets and " + "economy choices, which are empty lists for a catalog without " + "regions. The result names the selected PolicyEngine.py version, " + "so a client can request the version's other parts by that exact " + "version; a response for a named version may be reused for a day." + ), + ) + def get_model_catalog( + query: CatalogQueryDependency, + ) -> MetadataResourceSuccessResponse[MetadataModelCatalogResult] | Response: + return read_whole_version_resource( + dependencies, + "model-catalog", + query, + MetadataModelCatalogResponse, + lambda reader: reader.get_model_catalog( + query.country_id, query.policyengine_version + ), + ) + + @router.get( + "/model-catalog/variables", + response_model=MetadataCatalogVariablesResponse, + summary="Get every variable of one PolicyEngine.py catalog version", + description=( + "Returns all of the selected catalog version's variables in name " + "order, in one response. Where a variable's adds or subtracts list " + "comes from a parameter, part_parameter_values carries that " + "parameter's current-law values by date, so a client can pick the " + "list for any year." + ), + ) + def get_catalog_variables( + query: CatalogQueryDependency, + ) -> MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] | Response: + return read_whole_version_resource( + dependencies, + "model-catalog/variables", + query, + MetadataCatalogVariablesResponse, + lambda reader: reader.get_catalog_variables( + query.country_id, query.policyengine_version + ), + ) + + @router.get( + "/model-catalog/parameters", + response_model=MetadataCatalogParametersResponse, + summary="Get several parameters of one catalog version by name", + description=( + "Returns the named parameters (repeat name for each, at most 100) " + "in name order, each with its economy and household flags and its " + "current-law values by date, plus the stored labels of their " + "ancestor nodes for breadcrumbs and the requested names the " + "version has no parameter for. A client showing or editing a " + "reform loads all of its parameters in this one request." + ), + ) + def get_catalog_parameters( + query: CatalogParameterQueryDependency, + ) -> MetadataCatalogParametersResponse | JSONResponse: + return read_resource( + dependencies, + MetadataCatalogParametersResponse, + lambda reader: reader.get_catalog_parameters( + query.country_id, query.policyengine_version, names=query.name + ), + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/metadata/common.py b/policyengine_api/fastapi_routes/v2/metadata/common.py index fd4eaab5d..5d36bf50d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/common.py +++ b/policyengine_api/fastapi_routes/v2/metadata/common.py @@ -3,9 +3,13 @@ from __future__ import annotations import contextlib -from typing import TYPE_CHECKING, Any, TypeVar +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING, Any, Protocol, TypeVar +from fastapi.routing import APIRoute from pydantic import BaseModel +from starlette.responses import Response +from starlette.status import HTTP_200_OK from policyengine_api.data.v2.catalog.catalog_selection import ( InvalidPolicyEngineVersionError, @@ -18,26 +22,45 @@ V2ErrorResponse, v2_error_response, ) +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + body_entity_tag, +) +from policyengine_api.fastapi_routes.v2.metadata.response_models import ( + MetadataResourceSuccessResponse, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, ) +from policyengine_api.vary_headers import unique_vary_values if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Coroutine + from fastapi import Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.dependencies import ( NativeRouteDependencies, V2MetadataResourceReader, ) + from policyengine_api.query_parameters import CatalogQuery -ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { +PATH_COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, - "description": "The resource request or PolicyEngine.py version is invalid.", + "description": ( + "The country in the path is not supported. Query parameters are " + "checked first: an invalid one, including the PolicyEngine.py " + "version, returns 422." + ), }, +} +"""The extra response of the one metadata route that takes the country in its path.""" + +ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 404: { "model": V2ErrorResponse, "description": "The requested catalog resource is absent.", @@ -48,7 +71,10 @@ }, 422: { "model": V2ErrorResponse, - "description": "The request parameters do not match the resource schema.", + "description": ( + "The query parameters do not match the resource schema, for example " + "an unknown parameter or a malformed PolicyEngine.py version." + ), }, 500: { "model": V2ErrorResponse, @@ -61,7 +87,186 @@ } +CACHE_HEADER_DESCRIPTIONS: dict[str, dict[str, Any]] = { + "ETag": { + "description": "Strong validator: a hash of the exact response body.", + "schema": {"type": "string"}, + }, + "Cache-Control": { + "description": ( + "public, with max-age 86400 when the request names " + "policyengine_version and 300 otherwise; current parameter values " + "expire at the next UTC midnight." + ), + "schema": {"type": "string"}, + }, + "Vary": { + "description": ( + "Origin, Accept-Encoding: caches keep responses apart by origin " + "and by compression; a 304 carries the same value." + ), + "schema": {"type": "string"}, + }, +} + +METADATA_RESPONSES: dict[int | str, dict[str, Any]] = { + **ERROR_RESPONSES, + 200: {"headers": CACHE_HEADER_DESCRIPTIONS}, + 304: { + "description": ( + "Not modified: If-None-Match names the current ETag. No body is sent." + ), + "headers": CACHE_HEADER_DESCRIPTIONS, + }, +} +"""Every metadata route's documented responses, with its caching headers.""" + + +class CatalogVersionResult(Protocol): + """A whole-version result, which names the catalog version it was read from.""" + + @property + def policyengine_version(self) -> str: + """The exact PolicyEngine.py version of the catalog read.""" + ... + + ResponseT = TypeVar("ResponseT", bound=BaseModel) +ReadT = TypeVar("ReadT") +CatalogResultT = TypeVar("CatalogResultT", bound=CatalogVersionResult) + +EXPLICIT_VERSION_MAX_AGE_SECONDS = 86_400 +"""How long a response for a named catalog version may be reused (a day). + +A published catalog version never changes, so such a response stays valid +until its code's response format changes; the ETag lets a client revalidate +it cheaply after that. The one exception is a version's SPM availability in +``/v2/model-catalog``, recorded once after publication: until a publication +run records it, a response says the version offers no SPM settings. A +process keeps the selected catalog rows and built responses it has read for +its whole life, so it keeps saying so until it restarts (normally at the next +deployment, which follows the next publication run), and a client may reuse +such a response for up to this long after that. +""" +RUNNING_VERSION_MAX_AGE_SECONDS = 300 +"""How long a response for the running catalog version may be reused. + +Without ``policyengine_version`` the running PolicyEngine.py version answers, +and it changes whenever the API deploys a new one. +""" + + +UTC_DAY_READ = "utc_day_read" +"""The ``request.state`` attribute holding the instant a day-dependent read used. + +A route whose result depends on the UTC day, such as current parameter values, +sets it with ``take_utc_instant_for_read`` and reads with that same instant; +its response may then be reused only until the next UTC midnight after it. +""" + + +def utc_now() -> datetime: + """Return the current time in UTC; tests replace it to fix the clock.""" + return datetime.now(UTC) + + +def take_utc_instant_for_read(request: Request) -> datetime: + """Take the instant a day-dependent read uses, and record it on the request. + + The read and its response's cache lifetime then use the same UTC day. + """ + instant = utc_now() + setattr(request.state, UTC_DAY_READ, instant) + return instant + + +def _seconds_left_in_utc_day(read_instant: datetime, now: datetime) -> int: + """Return the seconds from ``now`` to the UTC midnight after ``read_instant``. + + Zero when that midnight has passed, so values read for one day are never + reused into the next. + """ + next_midnight = (read_instant.astimezone(UTC) + timedelta(days=1)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + return max(int((next_midnight - now).total_seconds()), 0) + + +class CacheableMetadataRoute(APIRoute): + """A metadata route whose successful responses carry caching headers. + + See ``with_cache_headers``. Failures are served unchanged, without them. + """ + + def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: + """Wrap FastAPI's handler so a 200 response gains its caching headers.""" + handle = super().get_route_handler() + + async def handle_with_cache_headers(request: Request) -> Response: + response = await handle(request) + if response.status_code != HTTP_200_OK: + return response + return with_cache_headers(request, response) + + return handle_with_cache_headers + + +def with_cache_headers(request: Request, response: Response) -> Response: + """Give a JSON response a strong ETag and a public cache lifetime. + + The ETag is a hash of the exact body, so it changes with any change to the + content, including the catalog format. A request naming its catalog with + ``policyengine_version`` may be reused for a day, one that takes the + running version for five minutes, and one whose route read for a UTC day + (``UTC_DAY_READ``) no later than the midnight after that day. Every such + response, and the 304 a request whose ``If-None-Match`` names the ETag + gets instead, varies by ``Origin`` and ``Accept-Encoding``. + """ + entity_tag = response.headers.get("etag") or body_entity_tag(response.body) + max_age = ( + EXPLICIT_VERSION_MAX_AGE_SECONDS + if "policyengine_version" in request.query_params + else RUNNING_VERSION_MAX_AGE_SECONDS + ) + read_instant = getattr(request.state, UTC_DAY_READ, None) + if read_instant is not None: + max_age = min(max_age, _seconds_left_in_utc_day(read_instant, utc_now())) + headers = {"Cache-Control": f"public, max-age={max_age}", "ETag": entity_tag} + if _names_entity_tag(request.headers.get("if-none-match"), entity_tag): + not_modified = Response(status_code=304, headers=headers) + not_modified.headers["Vary"] = _cacheable_vary(None) + return not_modified + response.headers.update(headers) + response.headers["Vary"] = _cacheable_vary(response.headers.get("vary")) + return response + + +CACHEABLE_VARY_VALUES = ("Origin", "Accept-Encoding") +"""What every cacheable metadata response, and its 304, varies by. + +The API reflects every origin in Access-Control-Allow-Origin, and a request +without Origin gets no such header, so a shared cache must keep responses +apart by Origin; gzip compresses the body for clients that accept it. A 304 +carries the Vary its 200 would. +""" + + +def _cacheable_vary(existing: str | None) -> str: + """Return ``existing`` with Origin and Accept-Encoding added if absent. + + The CORS and gzip middleware may append them again; the application's + outermost middleware (UniqueVaryMiddleware) then lists each value once. + """ + return unique_vary_values(", ".join([existing or "", *CACHEABLE_VARY_VALUES])) + + +def _names_entity_tag(if_none_match: str | None, entity_tag: str) -> bool: + if if_none_match is None: + return False + candidates = {candidate.strip() for candidate in if_none_match.split(",")} + return "*" in candidates or bool( + {entity_tag, f"W/{entity_tag}"}.intersection(candidates) + ) def read_resource( @@ -75,6 +280,60 @@ def read_resource( (400, 404 or 503, and 500 for anything unexpected), and the reader is closed whatever happens. """ + return run_metadata_read( + dependencies, lambda reader: response_type(result=operation(reader)) + ) + + +def read_whole_version_resource( + dependencies: NativeRouteDependencies, + route: str, + query: CatalogQuery, + response_type: type[MetadataResourceSuccessResponse[CatalogResultT]], + operation: Callable[[V2MetadataResourceReader], CatalogResultT], +) -> MetadataResourceSuccessResponse[CatalogResultT] | Response: + """Serve a whole catalog-version response, built once per process when kept. + + With the dependencies' response cache, the serialized body and its ETag + for ``route``, the query's country and the version it selects are read + and built only on the first successful request; failures are served as + ``read_resource`` serves them and never kept. Without a cache this is + ``read_resource``. + """ + cache = dependencies.v2_metadata_response_cache + if cache is None: + return read_resource(dependencies, response_type, operation) + + def build() -> BuiltResponse | Response: + result = read_resource(dependencies, response_type, operation) + if isinstance(result, MetadataResourceSuccessResponse): + return BuiltResponse( + body=result.model_dump_json().encode(), + policyengine_version=result.result.policyengine_version, + ) + return result + + kept = cache.get_or_build( + route, query.country_id, query.policyengine_version, build + ) + if isinstance(kept, KeptResponse): + return Response( + content=kept.body, + media_type="application/json", + headers={"ETag": kept.entity_tag}, + ) + return kept + + +def run_metadata_read( + dependencies: NativeRouteDependencies, + operation: Callable[[V2MetadataResourceReader], ReadT], +) -> ReadT | JSONResponse: + """Run one metadata read and return its result unchanged. + + Failures become the typed v2 error responses described by read_resource, + and the reader is closed whatever happens. + """ reader = None try: factory = dependencies.v2_metadata_reader_factory @@ -85,7 +344,7 @@ def read_resource( factory = _default_v2_metadata_reader_factory reader = factory() - return response_type(result=operation(reader)) + return operation(reader) except (InvalidMetadataPageError, InvalidPolicyEngineVersionError) as error: return v2_error_response(400, str(error)) except UnsupportedPreviewCountryError as error: diff --git a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py index 17ff62e27..448826c5d 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/geography_routes.py @@ -2,16 +2,22 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + RegionCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataDatasetDetailResponse, MetadataDatasetPageResponse, @@ -19,14 +25,10 @@ MetadataRegionDetailResponse, MetadataRegionPageResponse, ) -from policyengine_api.services.v2.metadata.types import MetadataRegionType if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_geography_router( dependencies: NativeRouteDependencies, @@ -36,7 +38,11 @@ def build_v2_metadata_geography_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/datasets", @@ -44,19 +50,16 @@ def build_v2_metadata_geography_router( summary="List logical inputs from a selected PolicyEngine.py catalog", ) def list_datasets( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataDatasetPageResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetPageResponse, lambda reader: reader.list_datasets( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -67,16 +70,15 @@ def list_datasets( ) def get_dataset( dataset_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataDatasetDetailResponse | JSONResponse: return read_resource( dependencies, MetadataDatasetDetailResponse, lambda reader: reader.get_dataset( - country_id, + query.country_id, dataset_id, - policyengine_version, + query.policyengine_version, ), ) @@ -86,21 +88,19 @@ def get_dataset( summary="List regions from a selected PolicyEngine.py catalog", ) def list_regions( - country_id: str, - policyengine_version: str | None = None, - region_type: MetadataRegionType | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: RegionCollectionQueryDependency, ) -> MetadataRegionPageResponse | JSONResponse: return read_resource( dependencies, MetadataRegionPageResponse, lambda reader: reader.list_regions( - country_id, - policyengine_version, - region_type=region_type.value if region_type is not None else None, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + region_type=( + None if query.region_type is None else query.region_type.value + ), + offset=query.offset, + limit=query.limit, ), ) @@ -111,16 +111,15 @@ def list_regions( ) def get_region_by_code( region_code: str, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region_by_code( - country_id, + query.country_id, region_code, - policyengine_version, + query.policyengine_version, ), ) @@ -131,16 +130,15 @@ def get_region_by_code( ) def get_region( region_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataRegionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataRegionDetailResponse, lambda reader: reader.get_region( - country_id, + query.country_id, region_id, - policyengine_version, + query.policyengine_version, ), ) @@ -150,15 +148,14 @@ def get_region( summary="Get compact economy-selection options from a selected catalog", ) def get_economy_options( - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataEconomyOptionsResponse | JSONResponse: return read_resource( dependencies, MetadataEconomyOptionsResponse, lambda reader: reader.get_economy_options( - country_id, - policyengine_version, + query.country_id, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py index 927b52536..ae3127725 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/model_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/model_routes.py @@ -2,16 +2,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Query +from fastapi import APIRouter from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + PATH_COUNTRY_ERROR_RESPONSES, + CacheableMetadataRoute, read_resource, ) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogPageQueryDependency, + CatalogQueryDependency, + CatalogVersionQueryDependency, + VariableCollectionQueryDependency, +) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataModelDetailResponse, MetadataModelPageResponse, @@ -25,9 +33,6 @@ if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - def build_v2_metadata_model_router( dependencies: NativeRouteDependencies, @@ -37,7 +42,11 @@ def build_v2_metadata_model_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/tax-benefit-models", @@ -45,37 +54,35 @@ def build_v2_metadata_model_router( summary="List models for one PolicyEngine.py catalog", ) def list_models( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelPageResponse, lambda reader: reader.list_models( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @router.get( "/tax-benefit-models/by-country/{country_id}", response_model=MetadataModelSelectionResponse, + responses=PATH_COUNTRY_ERROR_RESPONSES, summary="Get a country model and selected PolicyEngine.py version", ) def get_model_by_country( country_id: str, - policyengine_version: str | None = None, + query: CatalogVersionQueryDependency, ) -> MetadataModelSelectionResponse | JSONResponse: return read_resource( dependencies, MetadataModelSelectionResponse, lambda reader: reader.get_model_by_country( country_id, - policyengine_version, + query.policyengine_version, ), ) @@ -86,16 +93,15 @@ def get_model_by_country( ) def get_model( model_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelDetailResponse, lambda reader: reader.get_model( - country_id, + query.country_id, model_id, - policyengine_version, + query.policyengine_version, ), ) @@ -105,19 +111,16 @@ def get_model( summary="List selected PolicyEngine.py model versions", ) def list_model_versions( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: CatalogPageQueryDependency, ) -> MetadataModelVersionPageResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionPageResponse, lambda reader: reader.list_model_versions( - country_id, - policyengine_version, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, ), ) @@ -128,16 +131,15 @@ def list_model_versions( ) def get_model_version( version_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataModelVersionDetailResponse | JSONResponse: return read_resource( dependencies, MetadataModelVersionDetailResponse, lambda reader: reader.get_model_version( - country_id, + query.country_id, version_id, - policyengine_version, + query.policyengine_version, ), ) @@ -147,21 +149,17 @@ def get_model_version( summary="List variables from a selected PolicyEngine.py catalog", ) def list_variables( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: VariableCollectionQueryDependency, ) -> MetadataVariablePageResponse | JSONResponse: return read_resource( dependencies, MetadataVariablePageResponse, lambda reader: reader.list_variables( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + search=query.search, ), ) @@ -172,16 +170,15 @@ def list_variables( ) def get_variable( variable_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataVariableDetailResponse | JSONResponse: return read_resource( dependencies, MetadataVariableDetailResponse, lambda reader: reader.get_variable( - country_id, + query.country_id, variable_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py index 0f4714e5c..51383ccc0 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py +++ b/policyengine_api/fastapi_routes/v2/metadata/parameter_routes.py @@ -2,16 +2,23 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated +from typing import TYPE_CHECKING from uuid import UUID -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Request from starlette.responses import JSONResponse from policyengine_api.fastapi_routes.v2.metadata.common import ( - ERROR_RESPONSES, + METADATA_RESPONSES, + CacheableMetadataRoute, read_resource, + take_utc_instant_for_read, +) +from policyengine_api.fastapi_routes.v2.metadata.query_dependencies import ( + CatalogQueryDependency, + ParameterChildrenQueryDependency, + ParameterCollectionQueryDependency, + ParameterValueCollectionQueryDependency, ) from policyengine_api.fastapi_routes.v2.metadata.response_models import ( MetadataParameterChildPageResponse, @@ -20,26 +27,14 @@ MetadataParameterValueDetailResponse, MetadataParameterValuePageResponse, ) -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueFilter, +) if TYPE_CHECKING: from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -Offset = Annotated[int, Query(ge=0)] -Limit = Annotated[int, Query(ge=1, le=500)] - - -@dataclass(frozen=True) -class ParameterValueListingQuery: - """The query parameters of the canonical parameter-value listing route.""" - - country_id: str - policyengine_version: str | None = None - parameter_id: UUID | None = None - current: bool = False - offset: Offset = 0 - limit: Limit = 100 - def build_v2_metadata_parameter_router( dependencies: NativeRouteDependencies, @@ -49,7 +44,11 @@ def build_v2_metadata_parameter_router( Each request opens and closes its own metadata reader, so building the router touches no database. """ - router = APIRouter(prefix="/v2", responses=ERROR_RESPONSES) + router = APIRouter( + prefix="/v2", + responses=METADATA_RESPONSES, + route_class=CacheableMetadataRoute, + ) @router.get( "/parameters", @@ -57,21 +56,20 @@ def build_v2_metadata_parameter_router( summary="List parameters from a selected PolicyEngine.py catalog", ) def list_parameters( - country_id: str, - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, - search: str | None = None, + query: ParameterCollectionQueryDependency, ) -> MetadataParameterPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterPageResponse, lambda reader: reader.list_parameters( - country_id, - policyengine_version, - offset=offset, - limit=limit, - search=search, + query.country_id, + query.policyengine_version, + offset=query.offset, + limit=query.limit, + parameter_filter=ParameterFilter( + search=query.search, + names=None if query.name is None else tuple(query.name), + ), ), ) @@ -81,21 +79,17 @@ def list_parameters( summary="List direct children of one parameter path", ) def list_parameter_children( - country_id: str, - parent_path: str = "", - policyengine_version: str | None = None, - offset: Offset = 0, - limit: Limit = 100, + query: ParameterChildrenQueryDependency, ) -> MetadataParameterChildPageResponse | JSONResponse: return read_resource( dependencies, MetadataParameterChildPageResponse, lambda reader: reader.list_parameter_children( - country_id, - policyengine_version, - parent_path=parent_path, - offset=offset, - limit=limit, + query.country_id, + query.policyengine_version, + parent_path=query.parent_path, + offset=query.offset, + limit=query.limit, ), ) @@ -106,16 +100,15 @@ def list_parameter_children( ) def get_parameter( parameter_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterDetailResponse, lambda reader: reader.get_parameter( - country_id, + query.country_id, parameter_id, - policyengine_version, + query.policyengine_version, ), ) @@ -125,8 +118,12 @@ def get_parameter( summary="List canonical values from a selected PolicyEngine.py catalog", ) def list_parameter_values( - query: Annotated[ParameterValueListingQuery, Depends()], + request: Request, + query: ParameterValueCollectionQueryDependency, ) -> MetadataParameterValuePageResponse | JSONResponse: + # Current values are those in effect on the request's UTC day; the + # read and the response's lifetime use the same instant. + now = take_utc_instant_for_read(request) if query.current else None return read_resource( dependencies, MetadataParameterValuePageResponse, @@ -136,6 +133,12 @@ def list_parameter_values( value_filter=ParameterValueFilter( parameter_id=query.parameter_id, current=query.current, + now=now, + parameter_names=( + None + if query.parameter_name is None + else tuple(query.parameter_name) + ), ), offset=query.offset, limit=query.limit, @@ -149,16 +152,15 @@ def list_parameter_values( ) def get_parameter_value( value_id: UUID, - country_id: str, - policyengine_version: str | None = None, + query: CatalogQueryDependency, ) -> MetadataParameterValueDetailResponse | JSONResponse: return read_resource( dependencies, MetadataParameterValueDetailResponse, lambda reader: reader.get_parameter_value( - country_id, + query.country_id, value_id, - policyengine_version, + query.policyengine_version, ), ) diff --git a/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py new file mode 100644 index 000000000..b52d18d1e --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/query_dependencies.py @@ -0,0 +1,53 @@ +"""Typed FastAPI dependencies for the shared v2 metadata query contracts. + +Route modules import these aliases into their own globals, because FastAPI +resolves an endpoint's string annotations against its module only. +""" + +from __future__ import annotations + +from typing import Annotated + +from fastapi import Depends + +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.query_parameters import ( + CatalogPageQuery, + CatalogParameterQuery, + CatalogQuery, + CatalogVersionQuery, + ParameterChildrenQuery, + ParameterCollectionQuery, + ParameterValueCollectionQuery, + RegionCollectionQuery, + VariableCollectionQuery, +) + +CatalogQueryDependency = Annotated[ + CatalogQuery, Depends(query_dependency(CatalogQuery)) +] +CatalogVersionQueryDependency = Annotated[ + CatalogVersionQuery, Depends(query_dependency(CatalogVersionQuery)) +] +CatalogPageQueryDependency = Annotated[ + CatalogPageQuery, Depends(query_dependency(CatalogPageQuery)) +] +VariableCollectionQueryDependency = Annotated[ + VariableCollectionQuery, Depends(query_dependency(VariableCollectionQuery)) +] +RegionCollectionQueryDependency = Annotated[ + RegionCollectionQuery, Depends(query_dependency(RegionCollectionQuery)) +] +ParameterChildrenQueryDependency = Annotated[ + ParameterChildrenQuery, Depends(query_dependency(ParameterChildrenQuery)) +] +ParameterCollectionQueryDependency = Annotated[ + ParameterCollectionQuery, Depends(query_dependency(ParameterCollectionQuery)) +] +ParameterValueCollectionQueryDependency = Annotated[ + ParameterValueCollectionQuery, + Depends(query_dependency(ParameterValueCollectionQuery)), +] +CatalogParameterQueryDependency = Annotated[ + CatalogParameterQuery, Depends(query_dependency(CatalogParameterQuery)) +] diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_cache.py b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py new file mode 100644 index 000000000..1a1ad0b2d --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/metadata/response_cache.py @@ -0,0 +1,192 @@ +"""A bounded, process-wide store of whole catalog-version responses.""" + +from __future__ import annotations + +import hashlib +import threading +from collections import OrderedDict +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from starlette.responses import Response + +if TYPE_CHECKING: + from collections.abc import Callable + +DEFAULT_RESPONSE_CAPACITY = 8 + +KeptResponseKey = tuple[str, str, str] +"""A route name, a country ID and the exact PolicyEngine.py version served.""" + + +def body_entity_tag(body: bytes) -> str: + """Return the strong ETag of a response body: a hash of its exact bytes.""" + return f'"{hashlib.sha256(body).hexdigest()[:32]}"' + + +@dataclass(frozen=True) +class BuiltResponse: + """A serialized success body and the catalog version it was read from.""" + + body: bytes + policyengine_version: str + + +@dataclass(frozen=True) +class KeptResponse: + """A kept success body with its ETag, computed once when it was kept.""" + + body: bytes + entity_tag: str + + +@dataclass(frozen=True) +class FailedRead: + """A failed whole-version read, given to the requests that waited on it.""" + + status_code: int + body: bytes + + def response(self) -> Response: + """Return a new error response for one waiting request. + + Each request gets its own response object, because middleware adds + request-specific headers to the response it sends. + """ + return Response( + content=self.body, + status_code=self.status_code, + media_type="application/json", + ) + + +class _InFlightRead: + """One read in progress for a key, and how many requests wait on it.""" + + def __init__(self) -> None: + self.done = threading.Event() + self.outcome: KeptResponse | FailedRead | None = None + self.waiters = 0 + + +class MetadataResponseCache: + """Keep whole-version responses, built once each per process. + + A published catalog version never changes, so a body built for a route, + country and version stays valid for the life of the process. Responses + are kept under the exact version they were read from: a request without a + version (the running version, fixed for the process) and one naming that + version share one response. The most recently used ``capacity`` responses + are kept: the US variables body of the 6.10.0 catalog is 4,190,824 bytes + uncompressed, so the default of eight holds about 12 MB for both + countries' two whole-version routes at two versions, and at most about + 34 MB if every entry were that large. Requests that arrive while a read + for the same key is in progress wait for it, and all receive its outcome: + the kept response, or a copy of its error response. A failed read is not + kept, so the first request after it reads again. + """ + + def __init__(self, *, capacity: int = DEFAULT_RESPONSE_CAPACITY) -> None: + """Keep at most ``capacity`` responses (at least one).""" + if capacity < 1: + msg = "capacity must be positive" + raise ValueError(msg) + self._capacity = capacity + self._responses: OrderedDict[KeptResponseKey, KeptResponse] = OrderedDict() + self._running_versions: dict[tuple[str, str], str] = {} + self._lock = threading.Lock() + self._builds: dict[tuple[str, str, str | None], _InFlightRead] = {} + + def _stored( + self, route: str, country_id: str, requested_version: str | None + ) -> KeptResponse | None: + with self._lock: + version = ( + requested_version + if requested_version is not None + else self._running_versions.get((route, country_id)) + ) + if version is None: + return None + key = (route, country_id, version) + kept = self._responses.get(key) + if kept is not None: + self._responses.move_to_end(key) + return kept + + def get_or_build( + self, + route: str, + country_id: str, + requested_version: str | None, + build: Callable[[], BuiltResponse | Response], + ) -> KeptResponse | Response: + """Return the kept response, calling ``build`` only when none is kept. + + ``requested_version`` None means the running version. Requests that + arrive while a read for the same key is in progress wait for it and + receive its outcome: the kept response, or a copy of its error + response. A failed read is not kept, so the next request after it + reads again. + """ + kept = self._stored(route, country_id, requested_version) + if kept is not None: + return kept + key = (route, country_id, requested_version) + with self._lock: + in_flight = self._builds.get(key) + if in_flight is None: + in_flight = _InFlightRead() + self._builds[key] = in_flight + reads_itself = True + else: + in_flight.waiters += 1 + reads_itself = False + if not reads_itself: + in_flight.done.wait() + outcome = in_flight.outcome + if outcome is None: + # The read raised instead of returning a response: try again. + return self.get_or_build(route, country_id, requested_version, build) + return outcome.response() if isinstance(outcome, FailedRead) else outcome + try: + kept = self._stored(route, country_id, requested_version) + if kept is None: + built = build() + if not isinstance(built, BuiltResponse): + in_flight.outcome = FailedRead( + status_code=built.status_code, body=bytes(built.body) + ) + return built + kept = self._keep(route, country_id, requested_version, built) + in_flight.outcome = kept + return kept + finally: + with self._lock: + # Only this read's own entry is removed; a later read for the + # same key keeps its entry. + if self._builds.get(key) is in_flight: + del self._builds[key] + in_flight.done.set() + + def _keep( + self, + route: str, + country_id: str, + requested_version: str | None, + built: BuiltResponse, + ) -> KeptResponse: + key = (route, country_id, built.policyengine_version) + with self._lock: + if requested_version is None: + self._running_versions[(route, country_id)] = built.policyengine_version + kept = self._responses.get(key) + if kept is None: + kept = KeptResponse( + body=built.body, entity_tag=body_entity_tag(built.body) + ) + self._responses[key] = kept + self._responses.move_to_end(key) + while len(self._responses) > self._capacity: + self._responses.popitem(last=False) + return kept diff --git a/policyengine_api/fastapi_routes/v2/metadata/response_models.py b/policyengine_api/fastapi_routes/v2/metadata/response_models.py index d53660834..697ab6f76 100644 --- a/policyengine_api/fastapi_routes/v2/metadata/response_models.py +++ b/policyengine_api/fastapi_routes/v2/metadata/response_models.py @@ -6,10 +6,13 @@ from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -139,3 +142,21 @@ class MetadataEconomyOptionsResponse( MetadataResourceSuccessResponse[MetadataEconomyOptionsResult] ): """Success response holding the options for configuring an economy comparison.""" + + +class MetadataModelCatalogResponse( + MetadataResourceSuccessResponse[MetadataModelCatalogResult] +): + """Success response holding the small parts of one catalog version.""" + + +class MetadataCatalogVariablesResponse( + MetadataResourceSuccessResponse[MetadataCatalogVariablesResult] +): + """Success response holding every variable of one catalog version.""" + + +class MetadataCatalogParametersResponse( + MetadataResourceSuccessResponse[MetadataCatalogParametersResult] +): + """Success response holding parameters fetched by name, with their values.""" diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..1aa2032b3 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -7,11 +7,20 @@ from fastapi import APIRouter, Request from starlette.responses import JSONResponse +from policyengine_api.fastapi_routes.v2.countries.routes import ( + build_v2_country_router, +) from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( + build_v2_household_calculation_router, +) from policyengine_api.fastapi_routes.v2.households.routes import ( build_v2_household_router, ) from policyengine_api.fastapi_routes.v2.me.routes import build_v2_me_router +from policyengine_api.fastapi_routes.v2.metadata.catalog_routes import ( + build_v2_model_catalog_router, +) from policyengine_api.fastapi_routes.v2.metadata.geography_routes import ( build_v2_metadata_geography_router, ) @@ -52,10 +61,13 @@ def build_v2_router( router.include_router(build_v2_user_household_router(dependencies)) router.include_router(build_v2_user_policy_router(dependencies)) router.include_router(build_v2_household_router(dependencies)) + router.include_router(build_v2_household_calculation_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) + router.include_router(build_v2_country_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) + router.include_router(build_v2_model_catalog_router(dependencies)) @router.get( "/v2/openapi.json", diff --git a/policyengine_api/fastapi_routes/v2/user_households/routes.py b/policyengine_api/fastapi_routes/v2/user_households/routes.py index c642a79d8..4e86c9b86 100644 --- a/policyengine_api/fastapi_routes/v2/user_households/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_households/routes.py @@ -179,8 +179,13 @@ def read() -> UserHouseholdPageResponse: response_model=UserHouseholdDetailResponse, summary="Update or repoint one of the current user's saved households", description=( - "Changing legacy_household_id repoints the row and re-resolves " - "household_id from the Stage 11 copy mapping, or clears it." + "The row keeps its id, owner and creation time. Changing " + "legacy_household_id repoints the row and re-resolves household_id " + "from the Stage 11 copy mapping, or clears it. Changing household_id " + "alone points the row at that v2 household, which must exist in the " + "same country, and clears legacy_household_id. Supplied together, both " + "are stored, and household_id must be the copy of legacy_household_id " + "when one exists." ), ) def patch_user_household_route( diff --git a/policyengine_api/fastapi_routes/v2/user_policies/routes.py b/policyengine_api/fastapi_routes/v2/user_policies/routes.py index bb5a06acc..6ff192fee 100644 --- a/policyengine_api/fastapi_routes/v2/user_policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/user_policies/routes.py @@ -168,9 +168,13 @@ def read() -> UserPolicyPageResponse: response_model=UserPolicyDetailResponse, summary="Update or repoint one of the current user's saved policies", description=( - "Changing legacy_policy_id repoints the row and re-resolves policy_id " - "from the Stage 10 mirror mapping, or clears it. Rows maintained by " - "the v1 saved-policy mirror return 409." + "The row keeps its id, owner and creation time. Changing " + "legacy_policy_id repoints the row and re-resolves policy_id from the " + "Stage 10 mirror mapping, or clears it. Changing policy_id alone points " + "the row at that v2 policy, which must exist in the same country, and " + "clears legacy_policy_id. Supplied together, both are stored, and " + "policy_id must be the mirror copy of legacy_policy_id when one exists. " + "Rows maintained by the v1 saved-policy mirror return 409." ), ) def patch_user_policy_route( diff --git a/policyengine_api/migration_logging.py b/policyengine_api/migration_logging.py index 18f5ac7a9..e4b2e22ac 100644 --- a/policyengine_api/migration_logging.py +++ b/policyengine_api/migration_logging.py @@ -13,10 +13,13 @@ from policyengine_api.gcp_logging import logger from policyengine_api.migration_flags import ( ExplicitDatabaseSources, + MigrationContext, RouteImplementation, get_migration_log_context, + get_route_impl, infer_route_group, ) +from policyengine_api.migration_registry import ROUTE_GROUP_CONFIG_BY_NAME from policyengine_api.observability.identifiers import ( OBSERVABILITY_ID_HEADER, normalize_observability_id, @@ -61,6 +64,14 @@ V2_USER_RESOURCE_SEGMENTS = frozenset( {"user-policies", "user-households", "user-simulations", "user-reports", "me"} ) +# /v2/household-calculations reads the stored household, policy and catalog +# from Supabase when submitting, stores nothing itself, and always computes on +# the Cloud Run Simulation Entrypoint's v2 workers, whatever the household +# route group's database, entrypoint and compute flags select; it reads none +# of them, so none of them is checked for its log record. +V2_HOUSEHOLD_CALCULATION_SEGMENT = "household-calculations" +V2_HOUSEHOLD_CALCULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" +V2_HOUSEHOLD_CALCULATION_COMPUTE = "v2_primary" V2_RESOURCE_METHODS = frozenset({"GET", "POST", "PATCH", "DELETE"}) # A v2 resource path has at least the "v2" segment and the resource segment. V2_RESOURCE_PATH_MINIMUM_SEGMENTS = 2 @@ -240,6 +251,27 @@ def log_migration_request(served_request: ServedRequest) -> None: ) +def _household_calculation_context( + route_group: str, *, method: str, route_impl: RouteImplementation | None +) -> dict[str, str | None]: + """Return a household calculation's log record from what it actually uses.""" + try: + selected_route_impl = route_impl or get_route_impl(route_group) + except ValueError as error: + return {"route_group": route_group, "migration_flag_error": str(error)} + route_config = ROUTE_GROUP_CONFIG_BY_NAME[route_group] + return MigrationContext( + route_group=route_group, + route_impl=selected_route_impl, + db_entity=route_config.db_entity, + db_write=None, + db_read="supabase" if method == "POST" else None, + sim_flow=route_config.sim_flow, + sim_entrypoint=V2_HOUSEHOLD_CALCULATION_ENTRYPOINT, + sim_compute=V2_HOUSEHOLD_CALCULATION_COMPUTE, + ).to_log_dict() + + def _migration_context( *, method: str, @@ -247,6 +279,10 @@ def _migration_context( route_impl: RouteImplementation | None, ) -> dict[str, str | None]: route_group = infer_route_group(path) + if _v2_resource_segment(path) == V2_HOUSEHOLD_CALCULATION_SEGMENT: + return _household_calculation_context( + route_group, method=method, route_impl=route_impl + ) is_v2_metadata_read = _is_v2_metadata_resource_read(method, path) is_v2_supabase_resource = _is_v2_supabase_resource(method, path) uses_explicit_v2_source = is_v2_metadata_read or is_v2_supabase_resource diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..f46c5b8aa 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -37,8 +37,10 @@ class RouteGroupConfig: name="metadata", path_segments=( "metadata", + "countries", "datasets", "economy-options", + "model-catalog", "parameter-values", "parameters", "regions", @@ -58,6 +60,7 @@ class RouteGroupConfig: path_segments=( "household", "households", + "household-calculations", "user-household", "user-households", "calculate", diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index e06e9cdcb..adeca59e4 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -30,7 +30,9 @@ validate_policyengine_version, ) from policyengine_api.services.economic_impact_request import LocalAreaBreakdown +from policyengine_api.services.v2.metadata.types import MetadataRegionType from policyengine_api.spm import SPMSelection +from policyengine_api.supported_countries import SupportedCountryId from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, BUDGET_WINDOW_MAX_YEARS, @@ -42,7 +44,6 @@ DEFAULT_QUERY_LIMIT = 100 MAXIMUM_QUERY_LIMIT = 500 MAXIMUM_USER_ID_LENGTH = 255 -SUPPORTED_COUNTRY_IDS = frozenset({"us", "uk"}) def normalize_country_id(value: object) -> object: @@ -59,7 +60,7 @@ def validate_legacy_user_id(value: str) -> str: CountryId = Annotated[ - Literal["us", "uk"], + SupportedCountryId, BeforeValidator(normalize_country_id), Field(description="Supported PolicyEngine country ID"), ] @@ -232,6 +233,92 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CatalogVersionQuery(StrictQueryParameters): + """An optional exact catalog version, for routes with the country in the path.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class CatalogPageQuery(CatalogQuery, PaginationQuery): + """Query contract for one page of a catalog's resources.""" + + +class VariableCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's variables.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + + +class RegionCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's regions.""" + + region_type: MetadataRegionType | None = Field( + default=None, description="Keep only regions of this type" + ) + + +class ParameterChildrenQuery(CatalogPageQuery): + """Query contract for a page of one parameter path's direct children.""" + + parent_path: str = Field( + default="", + description="Dotted path whose direct children are listed; empty for the root", + ) + + +MAXIMUM_PARAMETER_NAMES = 100 +ParameterName = Annotated[ + str, + Field(min_length=1, max_length=512, description="Exact parameter path"), +] + + +class CatalogParameterQuery(CatalogQuery): + """Query contract for fetching several parameters of a catalog by name.""" + + name: list[ParameterName] = Field( + min_length=1, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Exact parameter paths; repeat the key for each one", + ) + + +class ParameterCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's parameters.""" + + search: str | None = Field( + default=None, + description="Case-insensitive substring of the name, label or description", + ) + name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths; repeat the key", + ) + + +class ParameterValueCollectionQuery(CatalogPageQuery): + """Query contract for a page of a catalog's current-law parameter values.""" + + parameter_id: ResourceId | None = None + parameter_name: list[ParameterName] | None = Field( + default=None, + max_length=MAXIMUM_PARAMETER_NAMES, + description="Keep only these exact parameter paths' values; repeat the key", + ) + current: bool = Field( + default=False, + description="Keep only the values in effect today (UTC)", + ) + + +class CountryListQuery(StrictQueryParameters): + """Query contract for the countries list, which takes no parameters.""" + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/countries/__init__.py b/policyengine_api/services/v2/countries/__init__.py new file mode 100644 index 000000000..f0f7b972b --- /dev/null +++ b/policyengine_api/services/v2/countries/__init__.py @@ -0,0 +1 @@ +"""Application services for the v2 countries list.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/__init__.py b/policyengine_api/services/v2/countries/database_connectors/__init__.py new file mode 100644 index 000000000..01c1fa018 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for the v2 countries service.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/reads.py b/policyengine_api/services/v2/countries/database_connectors/reads.py new file mode 100644 index 000000000..a13a8f3e4 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/reads.py @@ -0,0 +1,17 @@ +"""Database selections used by the v2 countries list and the startup check.""" + +from __future__ import annotations + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Country + + +def read_countries(session: Session) -> list[Country]: + """Return every countries row, ordered by country ID.""" + return list(session.exec(select(Country).order_by(col(Country.id))).all()) + + +def read_country_ids(session: Session) -> list[str]: + """Return the ID of every countries row, ordered.""" + return list(session.exec(select(Country.id).order_by(col(Country.id))).all()) diff --git a/policyengine_api/services/v2/countries/database_session.py b/policyengine_api/services/v2/countries/database_session.py new file mode 100644 index 000000000..455d4aa39 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for the v2 countries service.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class CountryDatabaseSession: + """Open the read sessions the countries service uses.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session diff --git a/policyengine_api/services/v2/countries/services.py b/policyengine_api/services/v2/countries/services.py new file mode 100644 index 000000000..96e0ad601 --- /dev/null +++ b/policyengine_api/services/v2/countries/services.py @@ -0,0 +1,49 @@ +"""Router-facing and startup service for the v2 countries table.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.database_connectors.reads import ( + read_countries, + read_country_ids, +) +from policyengine_api.services.v2.countries.transformations import country_read +from policyengine_api.services.v2.countries.validators import ( + validate_countries_match_code_list, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.types import CountryRead + + +class V2CountryService: + """List the served countries, and check them against the list in code.""" + + def __init__(self, database_session: CountryDatabaseSession) -> None: + """Read through ``database_session``, which opens one session per call.""" + self._database_session = database_session + + def list_countries(self) -> list[CountryRead]: + """Return every countries row, ordered by country ID.""" + with self._database_session.read() as session: + return [country_read(country) for country in read_countries(session)] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these. + + The application runs this at startup with the country list in code. + """ + with self._database_session.read() as session: + stored_country_ids = read_country_ids(session) + validate_countries_match_code_list( + stored_country_ids=stored_country_ids, + listed_country_ids=listed_country_ids, + ) diff --git a/policyengine_api/services/v2/countries/transformations.py b/policyengine_api/services/v2/countries/transformations.py new file mode 100644 index 000000000..fd1d65fe2 --- /dev/null +++ b/policyengine_api/services/v2/countries/transformations.py @@ -0,0 +1,19 @@ +"""Conversions between stored countries rows and service read records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.types import CountryRead + +if TYPE_CHECKING: + from policyengine_api.data.v2.models import Country + + +def country_read(country: Country) -> CountryRead: + """Convert one stored countries row to the service's read record.""" + return CountryRead( + id=country.id, + household_enabled=country.household_enabled, + society_wide_enabled=country.society_wide_enabled, + ) diff --git a/policyengine_api/services/v2/countries/types.py b/policyengine_api/services/v2/countries/types.py new file mode 100644 index 000000000..b4a1e74fc --- /dev/null +++ b/policyengine_api/services/v2/countries/types.py @@ -0,0 +1,19 @@ +"""Framework-independent data exchanged by the v2 countries layers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CountryRead: + """One served country and the calculations it supports. + + ``household_enabled`` allows household calculations and saved households; + ``society_wide_enabled`` allows society-wide reports, and implies + ``household_enabled``. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool diff --git a/policyengine_api/services/v2/countries/validators.py b/policyengine_api/services/v2/countries/validators.py new file mode 100644 index 000000000..fbb30ca5b --- /dev/null +++ b/policyengine_api/services/v2/countries/validators.py @@ -0,0 +1,48 @@ +"""Checks of the stored countries rows against the country list in code.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class CountryListMismatchError(RuntimeError): + """The countries table and the country list in code name different countries. + + ``listed_without_row`` holds the listed countries that have no row, and + ``rows_not_listed`` the rows for countries the list does not name, each + sorted by ID. + """ + + def __init__( + self, + *, + listed_without_row: tuple[str, ...], + rows_not_listed: tuple[str, ...], + ) -> None: + """Record both differences and describe them in the message.""" + self.listed_without_row = listed_without_row + self.rows_not_listed = rows_not_listed + super().__init__( + "the countries table does not match the country list in code: " + f"listed without a row: {', '.join(listed_without_row) or 'none'}; " + f"rows not listed: {', '.join(rows_not_listed) or 'none'}" + ) + + +def validate_countries_match_code_list( + *, + stored_country_ids: Iterable[str], + listed_country_ids: Iterable[str], +) -> None: + """Raise ``CountryListMismatchError`` unless both name the same countries.""" + stored = set(stored_country_ids) + listed = set(listed_country_ids) + if stored == listed: + return + raise CountryListMismatchError( + listed_without_row=tuple(sorted(listed - stored)), + rows_not_listed=tuple(sorted(stored - listed)), + ) diff --git a/policyengine_api/services/v2/household_calculations/__init__.py b/policyengine_api/services/v2/household_calculations/__init__.py new file mode 100644 index 000000000..0501ab630 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/__init__.py @@ -0,0 +1 @@ +"""Household calculations on v2, run by the Simulation Entrypoint.""" diff --git a/policyengine_api/services/v2/household_calculations/services.py b/policyengine_api/services/v2/household_calculations/services.py new file mode 100644 index 000000000..7a20d30d6 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/services.py @@ -0,0 +1,135 @@ +"""Calculate stored households through the simulation service, and read them. + +Submitting a calculation checks the country's ``countries`` row allows +household calculations (a household-only country does), checks the stored +household and policy exist in that country through the same services as +``/v2/households`` and ``/v2/policies`` (so one of another country, or none, +is not found), selects the country's running catalog version, and sends the +simulation service their IDs and that version's PolicyEngine.py version. +The simulation service reads the household, the policy's parameter values +and the catalog version from the Public API's tables itself, and calculates +the household's default year. It records every calculation, so one not +saved to an account is still stored, as a simulation run with no report. + +Reading a calculation asks the simulation service for it as it stands. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationRead, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationNotFoundError, + require_default_year, + require_household_country, +) + +if TYPE_CHECKING: + from collections.abc import Callable + from uuid import UUID + + from policyengine_api.services.v2.household_calculations.types import ( + CatalogSource, + CountrySource, + HouseholdCalculationInput, + HouseholdSource, + PolicySource, + SimulationHouseholdCalculations, + ) + + +class V2HouseholdCalculationService: + """Submit household calculations to the simulation service and read them.""" + + def __init__( + self, + *, + countries: CountrySource, + households: HouseholdSource, + policies: PolicySource, + catalog_reader_factory: Callable[[], CatalogSource], + simulation_factory: Callable[[], SimulationHouseholdCalculations], + ) -> None: + """Read through these services for one request. + + ``catalog_reader_factory`` opens a catalog reader, with its own + database session, only once a submission needs the catalog version. + ``simulation_factory`` returns the simulation service's client only + once a request is sent to it, so a request the Public API refuses + gets its own answer even when the simulation service is not + configured. + """ + self._countries = countries + self._households = households + self._policies = policies + self._catalog_reader_factory = catalog_reader_factory + self._simulation_factory = simulation_factory + + def submit_calculation( + self, calculation: HouseholdCalculationInput + ) -> HouseholdCalculationRead: + """Send the calculation to the simulation service and return it. + + Raise HouseholdCalculationCountryError, HouseholdNotFoundError, + PolicyNotFoundError or HouseholdCalculationYearError before anything + is sent, and the simulation errors when the service cannot calculate. + """ + country_id = calculation.country_id + require_household_country(self._countries.list_countries(), country_id) + household = self._households.get_household( + country_id=country_id, household_id=calculation.household_id + ) + if calculation.policy_id is not None: + self._policies.get_policy( + country_id=country_id, policy_id=calculation.policy_id + ) + # The simulation service calculates the household's default year; + # one without it is refused here, with 422. + require_default_year(household.default_year) + catalog_reader = self._catalog_reader_factory() + try: + catalog = catalog_reader.get_model_by_country(country_id) + finally: + catalog_reader.close() + submission: dict[str, Any] = { + "country": country_id, + "policyengine_version": catalog.policyengine_version, + "household_id": str(household.id), + "policy_id": ( + None if calculation.policy_id is None else str(calculation.policy_id) + ), + "options": { + # The household function varies the run's year, the + # household's default year, along an axis without a period. + "axes": ( + None + if calculation.axes is None + else [ + [ + axis.model_dump(mode="json", exclude_unset=True) + for axis in group + ] + for group in calculation.axes + ] + ), + "add_missing_variables": calculation.add_missing_variables, + }, + # A calculation is reused for 28 days after it succeeded; callers + # without an account must not start new Modal calls at will. + "force": False, + "trigger": "initial", + } + return HouseholdCalculationRead.from_simulation( + self._simulation_factory().submit(submission) + ) + + def get_calculation(self, calculation_id: UUID) -> HouseholdCalculationRead: + """Return the calculation, or raise HouseholdCalculationNotFoundError.""" + calculation = self._simulation_factory().read(calculation_id) + if calculation is None: + msg = f"Household calculation {calculation_id} not found." + raise HouseholdCalculationNotFoundError(msg) + return HouseholdCalculationRead.from_simulation(calculation) diff --git a/policyengine_api/services/v2/household_calculations/simulation_client.py b/policyengine_api/services/v2/household_calculations/simulation_client.py new file mode 100644 index 000000000..6ff1689a7 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/simulation_client.py @@ -0,0 +1,244 @@ +"""The simulation service's household calculation routes, checked by its models. + +Calls go to the Cloud Run Simulation Entrypoint (``SIMULATION_ENTRYPOINT_URL``), +the only simulation service with these routes, over HTTP with the gateway +credentials the Public API already uses (``GATEWAY_AUTH_*``) and the request's +correlation headers. + +The submission is checked against the generated client's submission model and +sent as built, and each answer is checked against the generated model for its +status and returned as decoded, both in their original key order, because +the generated models write their keys in an order of their own and the +household function lists warnings and calculated values in the household's +key order. The household's key order is the stored household's as the v2 +database returns it, which orders each record's keys itself, so it can +differ from the order a caller sent. + +A 400 is a refusal, whose reason the service gives (a household, policy or +catalog version it cannot find, or a household without a default year, +which the Public API checks first, or no household function for the +version); a 422 is the caller's data the service refuses, with its reason +as text (a parameter the catalog version lacks, or a stored policy value or +household the household functions cannot take) or one entry per request +field it refuses; anything else, +including a 503, no answer, an answer of another shape, a gateway token that +cannot be fetched or missing configuration, means the service is unavailable. + +One client serves the process, so its connections and its gateway token are +reused across requests. +""" + +from __future__ import annotations + +from functools import lru_cache +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, TypeVar + +import httpx +from policyengine_api_simulation_client.models.household_calculation_response import ( + HouseholdCalculationResponse, +) +from policyengine_api_simulation_client.models.household_calculation_submission import ( + HouseholdCalculationSubmission, +) +from policyengine_api_simulation_client.models.household_refusal_response import ( + HouseholdRefusalResponse, +) +from policyengine_api_simulation_client.models.household_request_field_error import ( + HouseholdRequestFieldError, +) +from policyengine_api_simulation_client.models.household_route_error import ( + HouseholdRouteError, +) +from policyengine_observability import ( # type: ignore[import-untyped] # the package has no type information + instrument_httpx, +) + +from policyengine_api.libs.gateway_auth import ( + GatewayAuthError, + GatewayAuthTokenProvider, + GatewayBearerAuth, + _require_all_or_none_gateway_auth_env, + gateway_auth_required, +) +from policyengine_api.libs.simulation_entrypoint import ( + _attach_current_request_id, + resolve_simulation_entrypoint_url, +) +from policyengine_api.observability import get_runtime +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from uuid import UUID + +CALCULATIONS_PATH = "/v2/household-calculations" + +REQUEST_TIMEOUT_SECONDS = 30.0 +CLOUD_RUN_SIMULATION_ENTRYPOINT = "cloud_run_simulation_entrypoint" + + +def _send(send: Callable[[], httpx.Response]) -> httpx.Response: + """Send one request; raise SimulationServiceUnavailableError if none answers.""" + try: + return send() + except (httpx.HTTPError, GatewayAuthError) as error: + msg = "The simulation service could not be reached." + raise SimulationServiceUnavailableError(msg) from error + + +ModelT = TypeVar("ModelT") + + +def _decoded( + response: httpx.Response, read_model: Callable[[Mapping[str, Any]], ModelT] +) -> tuple[object, ModelT]: + """Return the decoded body and the generated model read from it. + + Raise SimulationServiceUnavailableError when the body is not JSON or not + of the model's shape. + """ + try: + body = response.json() + return body, read_model(body) + except (ValueError, KeyError, TypeError, AttributeError) as error: + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) from error + + +def _calculation(response: httpx.Response) -> dict[str, Any]: + body, _calculation_model = _decoded( + response, HouseholdCalculationResponse.from_dict + ) + if not isinstance(body, dict): + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) + return body + + +def _invalid_request_reason(refusal: HouseholdRefusalResponse) -> str: + """Return the service's reason, or each refused field and why. + + A field's location is given without the request's ``body`` prefix. Raise + SimulationServiceUnavailableError for a refusal of another shape: the + generated model keeps a ``detail`` it cannot read as text or fields. + """ + detail: object = refusal.detail + if isinstance(detail, str): + reason = detail + else: + fields = ( + [item for item in detail if isinstance(item, HouseholdRequestFieldError)] + if isinstance(detail, list) + else [] + ) + if not isinstance(detail, list) or len(fields) != len(detail): + msg = "The simulation service's answer could not be read." + raise SimulationServiceUnavailableError(msg) + reasons = [] + for item in fields: + location = [str(part) for part in item.loc] + if location[:1] == ["body"]: + location = location[1:] + reasons.append(f"{'.'.join(location)}: {item.msg}") + reason = "; ".join(reasons) or "the request is invalid" + return f"The simulation service cannot calculate this household: {reason}" + + +def _unavailable(status_code: int) -> SimulationServiceUnavailableError: + return SimulationServiceUnavailableError( + f"The simulation service answered {status_code}." + ) + + +def _gateway_auth() -> GatewayBearerAuth | None: + """Return the gateway credentials, or None when none are set and none required.""" + _require_all_or_none_gateway_auth_env() + token_provider = GatewayAuthTokenProvider() + if token_provider.configured: + return GatewayBearerAuth(token_provider) + if gateway_auth_required(): + msg = "Gateway auth is required but GATEWAY_AUTH_* is not set." + raise GatewayAuthError(msg) + return None + + +class SimulationHouseholdClient: + """Submit and read household calculations on the simulation service.""" + + def __init__( + self, base_url: str, *, httpx_args: dict[str, Any] | None = None + ) -> None: + """Call the service at ``base_url`` with these httpx client arguments.""" + http_client = httpx.Client( + base_url=base_url, + timeout=httpx.Timeout(REQUEST_TIMEOUT_SECONDS), + event_hooks={"request": [_attach_current_request_id]}, + **(httpx_args or {}), + ) + instrument_httpx(http_client, get_runtime()) + self._base_url = base_url.rstrip("/") + self._http_client = http_client + + @property + def base_url(self) -> str: + """Return the simulation service's base URL, without a trailing slash.""" + return self._base_url + + @classmethod + def from_environment(cls) -> SimulationHouseholdClient: + """Call the configured Cloud Run Simulation Entrypoint with gateway auth. + + Raise SimulationServiceUnavailableError when the URL is missing or + invalid, or gateway auth is partly configured or required but unset. + """ + try: + base_url = resolve_simulation_entrypoint_url( + CLOUD_RUN_SIMULATION_ENTRYPOINT + ) + auth = _gateway_auth() + except (ValueError, GatewayAuthError) as error: + msg = "The simulation service is not configured." + raise SimulationServiceUnavailableError(msg) from error + return cls(base_url, httpx_args={} if auth is None else {"auth": auth}) + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Submit a calculation and return the service's calculation answer.""" + HouseholdCalculationSubmission.from_dict(submission) + response = _send( + lambda: self._http_client.post(CALCULATIONS_PATH, json=submission) + ) + if response.status_code == HTTPStatus.ACCEPTED: + return _calculation(response) + if response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY: + _body, invalid = _decoded(response, HouseholdRefusalResponse.from_dict) + raise SimulationServiceInvalidRequestError(_invalid_request_reason(invalid)) + if response.status_code == HTTPStatus.BAD_REQUEST: + _body, refusal = _decoded(response, HouseholdRouteError.from_dict) + raise SimulationServiceRefusedError(refusal.detail) + raise _unavailable(response.status_code) + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the service's calculation answer, or None when it has none.""" + response = _send( + lambda: self._http_client.get(f"{CALCULATIONS_PATH}/{calculation_id}") + ) + if response.status_code == HTTPStatus.OK: + return _calculation(response) + if response.status_code == HTTPStatus.NOT_FOUND: + _decoded(response, HouseholdRouteError.from_dict) + return None + raise _unavailable(response.status_code) + + +@lru_cache(maxsize=1) +def process_simulation_household_client() -> SimulationHouseholdClient: + """Return the process's client, built from the environment on first use. + + A configuration error is raised again on the next call, not kept. + """ + return SimulationHouseholdClient.from_environment() diff --git a/policyengine_api/services/v2/household_calculations/types.py b/policyengine_api/services/v2/household_calculations/types.py new file mode 100644 index 000000000..d0768b4b7 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/types.py @@ -0,0 +1,234 @@ +"""Framework-independent data exchanged by the household calculation layers.""" + +from __future__ import annotations + +import math +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Any, Literal, Protocol, Self +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator + +from policyengine_api.query_parameters import CountryId + +if TYPE_CHECKING: + from collections.abc import Mapping + + from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.households.types import HouseholdRead + from policyengine_api.services.v2.metadata.types import ( + MetadataModelSelectionResult, + ) + from policyengine_api.services.v2.policies.types import PolicyRead + +# The simulation service's limits on axes: the product of the groups' point +# counts is at most MAXIMUM_AXIS_POINTS. +MAXIMUM_AXIS_POINTS = 10_000 +MAXIMUM_AXIS_GROUPS = 8 +MAXIMUM_PARALLEL_AXES = 32 +CalculationStatus = Literal["pending", "dispatching", "running", "succeeded", "failed"] + + +class StrictCalculationModel(BaseModel): + """Base for household calculation data: no unknown fields, finite numbers.""" + + model_config = ConfigDict(extra="forbid", allow_inf_nan=False, frozen=True) + + +class HouseholdCalculationAxis(StrictCalculationModel): + """One input varied across points. + + The variable ``name`` takes ``count`` evenly spaced values from ``min`` to + ``max`` in ``period`` (the household's ``default_year`` when omitted), for + the entity member at position ``index`` (the first when omitted). + """ + + name: Annotated[str, Field(min_length=1, max_length=255)] + count: Annotated[int, Field(ge=1, le=MAXIMUM_AXIS_POINTS)] + min: int | float + max: int | float + period: Annotated[str, Field(min_length=1, max_length=64)] | int | None = None + index: Annotated[int, Field(ge=0)] | None = None + + +AxisGroup = Annotated[ + list[HouseholdCalculationAxis], + Field(min_length=1, max_length=MAXIMUM_PARALLEL_AXES), +] + + +class HouseholdCalculationInput(StrictCalculationModel): + """A stored household to calculate under a stored policy, or current law. + + The calculation's year is the household's default year, the year its + values name. ``axes`` vary inputs + across points: the axes of one group vary together, so they share one + point count, and the groups form a grid of at most 10,000 points. + ``add_missing_variables`` adds every yearly, monthly and + eternal variable the household lacks, as ``/calculate-full`` does. + """ + + country_id: CountryId + household_id: UUID + policy_id: UUID | None = None + axes: Annotated[list[AxisGroup], Field(max_length=MAXIMUM_AXIS_GROUPS)] | None = ( + None + ) + add_missing_variables: bool = False + + @field_validator("axes") + @classmethod + def limit_axis_points( + cls, axes: list[list[HouseholdCalculationAxis]] | None + ) -> list[list[HouseholdCalculationAxis]] | None: + """Refuse groups of uneven point counts and grids of too many points.""" + if axes is None: + return None + for group in axes: + if len({axis.count for axis in group}) != 1: + msg = "the axes of one group must share one point count" + raise ValueError(msg) + if math.prod(group[0].count for group in axes) > MAXIMUM_AXIS_POINTS: + msg = f"the axis groups may vary at most {MAXIMUM_AXIS_POINTS} points" + raise ValueError(msg) + return axes + + +class HouseholdCalculationResult(StrictCalculationModel): + """A succeeded calculation's household and its notes. + + ``household`` is the stored household in the situation shape, its people + and groups named by their record IDs, with each requested value filled + in. ``spm_config`` is the SPM selection the calculation applied and + ``spm_provenance`` its receipt, both null for a country without SPM. + """ + + household: dict[str, JsonValue] + warnings: list[str] + spm_config: dict[str, JsonValue] | None + spm_provenance: dict[str, JsonValue] | None + policyengine_version: str | None + + +class HouseholdCalculationFailure(StrictCalculationModel): + """Why a calculation failed, with the simulation service's error detail.""" + + code: str + message: str + detail: dict[str, JsonValue] | None + + +class HouseholdCalculationRead(StrictCalculationModel): + """A household calculation as it stands: what it calculates and its outcome.""" + + calculation_id: UUID + country_id: str + household_id: UUID + policy_id: UUID | None + year: int + catalog_version_id: UUID + status: CalculationStatus + created_at: datetime + completed_at: datetime | None + result: HouseholdCalculationResult | None + error: HouseholdCalculationFailure | None + + @classmethod + def from_simulation(cls, calculation: Mapping[str, Any]) -> Self: + """Build the read from the simulation service's calculation answer.""" + result = calculation.get("result") + error = calculation.get("error") + bundle = result.get("bundle") if isinstance(result, dict) else None + return cls( + calculation_id=calculation["calculation_id"], + country_id=calculation["country"], + household_id=calculation["household_id"], + policy_id=calculation["policy_id"], + year=calculation["year"], + catalog_version_id=calculation["catalog_version_id"], + status=calculation["status"], + created_at=calculation["created_at"], + completed_at=calculation["completed_at"], + result=( + None + if result is None + else HouseholdCalculationResult( + household=result["household"], + warnings=result["warnings"], + spm_config=result["spm_config"], + spm_provenance=result["spm_provenance"], + policyengine_version=( + bundle.get("policyengine_version") + if isinstance(bundle, dict) + else None + ), + ) + ), + error=None if error is None else HouseholdCalculationFailure(**error), + ) + + +class SimulationServiceUnavailableError(RuntimeError): + """The simulation service could not be reached, or could not answer.""" + + +class SimulationServiceRefusedError(RuntimeError): + """The simulation service refused the calculation; the message says why.""" + + +class SimulationServiceInvalidRequestError(ValueError): + """The simulation service refuses the caller's data. + + Such as a policy changing a parameter the catalog version lacks, a stored + policy value or household the household functions cannot take, or an + invalid option. The message gives the service's reason, or each refused + field and why. + """ + + +class CountrySource(Protocol): + """The countries table (``V2CountryService``).""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and its capability.""" + ... + + +class HouseholdSource(Protocol): + """Stored households (``V2HouseholdService``).""" + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return one household of this country, or raise HouseholdNotFoundError.""" + ... + + +class PolicySource(Protocol): + """Stored policies (``V2PolicyService``).""" + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + """Return one policy of this country, or raise PolicyNotFoundError.""" + ... + + +class CatalogSource(Protocol): + """The country's catalog versions (a v2 metadata reader).""" + + def get_model_by_country(self, country_id: str) -> MetadataModelSelectionResult: + """Return the country's model and the running catalog version.""" + ... + + def close(self) -> None: + """Close the reader's database session.""" + ... + + +class SimulationHouseholdCalculations(Protocol): + """The simulation service's household calculation routes.""" + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Submit a calculation and return it as the service answers.""" + ... + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the calculation as the service answers, or None if unknown.""" + ... diff --git a/policyengine_api/services/v2/household_calculations/validators.py b/policyengine_api/services/v2/household_calculations/validators.py new file mode 100644 index 000000000..9eb7dcd93 --- /dev/null +++ b/policyengine_api/services/v2/household_calculations/validators.py @@ -0,0 +1,47 @@ +"""Checks of a household calculation against what is already loaded.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + from policyengine_api.services.v2.countries.types import CountryRead + + +class HouseholdCalculationCountryError(ValueError): + """The country does not offer household calculations.""" + + +class HouseholdCalculationYearError(ValueError): + """The household has no default year to calculate.""" + + +class HouseholdCalculationNotFoundError(LookupError): + """The simulation service has no such household calculation.""" + + +def require_household_country( + countries: Iterable[CountryRead], country_id: str +) -> None: + """Raise HouseholdCalculationCountryError unless the country calculates households. + + A household-only country (household calculations without society-wide + reports) passes. + """ + for country in countries: + if country.id == country_id and country.household_enabled: + return + msg = f"Household calculations are not available for country {country_id}." + raise HouseholdCalculationCountryError(msg) + + +def require_default_year(default_year: int | None) -> None: + """Raise HouseholdCalculationYearError unless the household has a default year. + + The simulation service calculates that year, the year its values name. + """ + if default_year is None: + msg = "This household has no default year to calculate." + raise HouseholdCalculationYearError(msg) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py index cd5dbc276..b6d79b6fa 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_datasets.py @@ -48,3 +48,17 @@ def read_dataset( ), ) return rows[0] if rows else None + + +def read_input_datasets(session: Session, *, model_version_id: UUID) -> list[Dataset]: + """Return every logical input dataset of a model version in name order.""" + return read_rows( + session, + select(Dataset) + .where( + col(Dataset.tax_benefit_model_version_id) == model_version_id, + col(Dataset.is_output_dataset).is_(False), + col(Dataset.storage_path).is_(None), + ) + .order_by(col(Dataset.name)), + ) diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py new file mode 100644 index 000000000..24968399a --- /dev/null +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_nodes.py @@ -0,0 +1,36 @@ +"""Database selections for parameter-node labels.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import ParameterNode +from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import ParameterNodeLabelRow + +if TYPE_CHECKING: + from collections.abc import Collection + from uuid import UUID + + +def read_parameter_node_labels( + session: Session, *, model_version_id: UUID, node_names: Collection[str] +) -> list[ParameterNodeLabelRow]: + """Return the name and stored label of each named node of a model version. + + Names the version has no node for are absent. Rows are in name order. + """ + if not node_names: + return [] + rows = read_rows( + session, + select(col(ParameterNode.name), col(ParameterNode.label)) + .where( + col(ParameterNode.tax_benefit_model_version_id) == model_version_id, + col(ParameterNode.name).in_(sorted(node_names)), + ) + .order_by(col(ParameterNode.name)), + ) + return [ParameterNodeLabelRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py index cd945768e..49f232a8e 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameter_tree.py @@ -114,11 +114,7 @@ def _parameter_children_statement( ), sa.case((is_node, "node"), else_="parameter").label("type"), sa.case((is_node, direct_child_count), else_=None).label("child_count"), - parameter_id.label("parameter_id"), - col(Parameter.label).label("parameter_label"), - col(Parameter.description).label("parameter_description"), - col(Parameter.data_type).label("parameter_data_type"), - col(Parameter.unit).label("parameter_unit"), + Parameter, ) .select_from( paths.outerjoin( @@ -152,7 +148,7 @@ def read_parameter_children( """Return a page of the direct children of parent_path in a model version's tree. Each row has the child's path, label, type (node or parameter), child count - and, for parameters, the parameter's summary columns. The session's dialect + and the stored Parameter (None for a node). The session's dialect selects SQLite or PostgreSQL string functions; limit + 1 rows are fetched so the caller can tell whether another page exists. """ diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py index 02062798a..eadd87be9 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_parameters.py @@ -9,11 +9,16 @@ from policyengine_api.data.v2.models import Parameter, ParameterValue from policyengine_api.services.v2.metadata.database_connectors.reads import read_rows +from policyengine_api.services.v2.metadata.types import CurrentLawValueRow if TYPE_CHECKING: + from collections.abc import Collection from uuid import UUID - from policyengine_api.services.v2.metadata.types import ParameterValueSelection + from policyengine_api.services.v2.metadata.types import ( + ParameterFilter, + ParameterValueSelection, + ) def _escape_like(value: str) -> str: @@ -26,17 +31,23 @@ def read_parameters( model_version_id: UUID, offset: int, limit: int, - search: str | None, + parameter_filter: ParameterFilter, ) -> list[Parameter]: """Return a page of a model version's parameters in name order. - search matches name, label or description as a literal, case-insensitive - substring. Fetch limit + 1 rows so the caller can tell whether another page - exists. + The filter's search matches name, label or description as a literal, + case-insensitive substring, and its names keep only parameters with those + exact names. Fetch limit + 1 rows so the caller can tell whether another + page exists. """ statement = select(Parameter).where( col(Parameter.tax_benefit_model_version_id) == model_version_id ) + if parameter_filter.names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(parameter_filter.names)) + ) + search = parameter_filter.search if search: pattern = f"%{_escape_like(search)}%" statement = statement.where( @@ -52,6 +63,26 @@ def read_parameters( ) +def read_parameters_by_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[Parameter]: + """Return the named parameters of a model version in name order. + + Names the version has no parameter for are absent. + """ + if not parameter_names: + return [] + return read_rows( + session, + select(Parameter) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + ) + .order_by(col(Parameter.name)), + ) + + def read_parameter( session: Session, *, model_version_id: UUID, parameter_id: UUID ) -> Parameter | None: @@ -77,8 +108,9 @@ def read_parameter_values( """Return a page of a model version's current-law parameter values. Values tied to a policy or dynamic are excluded. The selection's - parameter_id keeps one parameter's values and its selected_day keeps those - in effect on that day. Rows are ordered by parameter name, then newest + parameter_id keeps one parameter's values, its parameter_names the values + of the parameters with those names, and its selected_day those in effect + on that day. Rows are ordered by parameter name, then newest first, and limit + 1 are fetched so the caller can tell whether another page exists. """ @@ -95,6 +127,10 @@ def read_parameter_values( ) if parameter_id is not None: statement = statement.where(col(ParameterValue.parameter_id) == parameter_id) + if selection.parameter_names is not None: + statement = statement.where( + col(Parameter.name).in_(sorted(selection.parameter_names)) + ) if selected_day is not None: statement = statement.where( col(ParameterValue.start_date) <= selected_day, @@ -131,3 +167,33 @@ def read_parameter_value( ), ) return rows[0] if rows else None + + +def read_current_law_values_by_parameter_name( + session: Session, *, model_version_id: UUID, parameter_names: Collection[str] +) -> list[CurrentLawValueRow]: + """Return the current-law values of the named parameters of a model version. + + Values tied to a policy or dynamic are excluded. Rows are ordered by + parameter name, then start date. + """ + if not parameter_names: + return [] + rows = read_rows( + session, + select( + col(Parameter.name), + col(ParameterValue.value_json), + col(ParameterValue.start_date), + col(ParameterValue.end_date), + ) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where( + col(Parameter.tax_benefit_model_version_id) == model_version_id, + col(Parameter.name).in_(sorted(parameter_names)), + col(ParameterValue.policy_id).is_(None), + col(ParameterValue.dynamic_id).is_(None), + ) + .order_by(col(Parameter.name), col(ParameterValue.start_date)), + ) + return [CurrentLawValueRow(*row) for row in rows] diff --git a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py index 4d84be396..1378f6528 100644 --- a/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py +++ b/policyengine_api/services/v2/metadata/database_connectors/reads_variables.py @@ -62,3 +62,13 @@ def read_variable( ), ) return rows[0] if rows else None + + +def read_all_variables(session: Session, *, model_version_id: UUID) -> list[Variable]: + """Return every variable of a model version in name order.""" + return read_rows( + session, + select(Variable) + .where(col(Variable.tax_benefit_model_version_id) == model_version_id) + .order_by(col(Variable.name)), + ) diff --git a/policyengine_api/services/v2/metadata/selected_catalogs.py b/policyengine_api/services/v2/metadata/selected_catalogs.py new file mode 100644 index 000000000..832705432 --- /dev/null +++ b/policyengine_api/services/v2/metadata/selected_catalogs.py @@ -0,0 +1,41 @@ +"""A process-wide store of the catalog versions metadata reads have selected.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.catalog_selection import SelectedCatalog + +CatalogSelectionKey = tuple[str, str | None] +"""A country ID and the requested PolicyEngine.py version. + +A version of None means the running version, which is fixed for the process. +""" + + +class SelectedCatalogStore: + """Keep each catalog version a request has selected, for the process's life. + + A published catalog version never changes, so once a request has found it, + later requests for the same country and version need not query for it + again. Only successful selections are kept: an unsupported country, an + invalid version or an absent catalog is looked up, and fails, each time. + The kept records are detached, fully loaded rows that are only read. + """ + + def __init__(self) -> None: + """Start with no selections kept.""" + self._selections: dict[CatalogSelectionKey, SelectedCatalog] = {} + self._lock = threading.Lock() + + def get(self, key: CatalogSelectionKey) -> SelectedCatalog | None: + """Return the selection kept for ``key``, or None.""" + with self._lock: + return self._selections.get(key) + + def keep(self, key: CatalogSelectionKey, selected: SelectedCatalog) -> None: + """Keep ``selected`` as the selection for ``key``.""" + with self._lock: + self._selections[key] = selected diff --git a/policyengine_api/services/v2/metadata/services.py b/policyengine_api/services/v2/metadata/services.py index 938eb660e..4496e3b30 100644 --- a/policyengine_api/services/v2/metadata/services.py +++ b/policyengine_api/services/v2/metadata/services.py @@ -14,15 +14,21 @@ from policyengine_api.services.v2.metadata.database_connectors.reads_datasets import ( read_dataset, read_datasets, + read_input_datasets, +) +from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_nodes import ( # noqa: E501 # a dotted module path cannot be wrapped + read_parameter_node_labels, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameter_tree import ( # noqa: E501 # a dotted module path cannot be wrapped read_parameter_children, ) from policyengine_api.services.v2.metadata.database_connectors.reads_parameters import ( + read_current_law_values_by_parameter_name, read_parameter, read_parameter_value, read_parameter_values, read_parameters, + read_parameters_by_name, ) from policyengine_api.services.v2.metadata.database_connectors.reads_regions import ( read_input_dataset, @@ -30,13 +36,19 @@ read_regions, ) from policyengine_api.services.v2.metadata.database_connectors.reads_variables import ( + read_all_variables, read_variable, read_variables, ) from policyengine_api.services.v2.metadata.transformations import ( + ancestor_paths, + dated_values_by_parameter_name, + metadata_catalog_parameters, + metadata_catalog_variables, metadata_dataset, metadata_economy_options, metadata_model, + metadata_model_catalog, metadata_model_selection, metadata_model_version, metadata_parameter, @@ -44,15 +56,20 @@ metadata_parameter_value, metadata_region, metadata_variable, + national_region, page_result, parameter_value_selection, + part_parameter_paths, ) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDetailResult, MetadataEconomyOptionsResult, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -60,6 +77,7 @@ MetadataParameterSummary, MetadataRegion, MetadataVariable, + ParameterFilter, ParameterValueFilter, ) from policyengine_api.services.v2.metadata.validators import ( @@ -69,11 +87,16 @@ ) if TYPE_CHECKING: + from collections.abc import Callable, Sequence from uuid import UUID + from policyengine_api.data.v2.models import Dataset, Region from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) class V2MetadataService: @@ -90,12 +113,18 @@ def __init__( database_session: MetadataDatabaseSession, *, running_policyengine_version: str, + selected_catalogs: SelectedCatalogStore | None = None, ) -> None: - """Bind to a request's session and the running PolicyEngine.py version.""" + """Bind to a request's session and the running PolicyEngine.py version. + + With ``selected_catalogs``, a catalog version found once is taken from + that store instead of being queried again. + """ self._database_session = database_session self._running_policyengine_version = validate_policyengine_version( running_policyengine_version ) + self._selected_catalogs = selected_catalogs def close(self) -> None: """Close the request's database session.""" @@ -104,12 +133,20 @@ def close(self) -> None: def _select_catalog( self, country_id: str, policyengine_version: str | None ) -> SelectedCatalog: - return read_metadata_catalog( + key = (country_id, policyengine_version) + if self._selected_catalogs is not None: + kept = self._selected_catalogs.get(key) + if kept is not None: + return kept + selected = read_metadata_catalog( self._database_session.session, country_id=country_id, running_policyengine_version=self._running_policyengine_version, policyengine_version=policyengine_version, ) + if self._selected_catalogs is not None: + self._selected_catalogs.keep(key, selected) + return selected def _select_paginated_catalog( self, @@ -255,12 +292,13 @@ def list_parameters( *, offset: int = 0, limit: int = 100, - search: str | None = None, + parameter_filter: ParameterFilter | None = None, ) -> MetadataPageResult[MetadataParameterSummary]: """Return a page of the catalog's parameters in name order. - search keeps parameters whose name, label or description contains it, - ignoring case. + The filter's search keeps parameters whose name, label or description + contains it, ignoring case, and its names keep only parameters with + those exact names. No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -270,7 +308,9 @@ def list_parameters( model_version_id=selected.model_version.id, offset=offset, limit=limit, - search=search, + parameter_filter=( + parameter_filter if parameter_filter is not None else ParameterFilter() + ), ) return page_result( selected, @@ -343,8 +383,9 @@ def list_parameter_values( """Return a page of current-law parameter values, newest first per parameter. Values belonging to a reform or dynamic are never listed. The filter's - parameter_id keeps one parameter's values, and current=True keeps those in - effect on the UTC day of now (today by default). No filter lists them all. + parameter_id keeps one parameter's values, parameter_names the values of + the parameters with those exact names, and current=True those in effect + on the UTC day of now (today by default). No filter lists them all. """ selected = self._select_paginated_catalog( country_id, policyengine_version, offset=offset, limit=limit @@ -516,28 +557,147 @@ def get_economy_options( region, its default input dataset, the current-law id or the years. """ selected = self._select_catalog(country_id, policyengine_version) - regions = read_regions( - self._database_session.session, - model_version_id=selected.model_version.id, + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + dataset = _validated_national_dataset( + selected, + country_id=country_id, + regions=regions, + find_dataset=lambda dataset_id: read_input_dataset( + session, + model_version_id=selected.model_version.id, + dataset_id=dataset_id, + ), ) - national_region = next( - (region for region in regions if region.code == country_id), None + return metadata_economy_options( + selected, regions=regions, national_dataset=dataset + ) + + def get_model_catalog( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataModelCatalogResult: + """Return the small parts of the selected catalog version. + + They are the model and model version, entities, basic inputs, every + region and logical input dataset, and the economy choices. A catalog + without regions offers no economy calculation, so its choices are + empty; one with regions raises MetadataCatalogUnavailableError when it + lacks the national region, its default dataset or the model-version + options, as get_economy_options does. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + regions = read_regions(session, model_version_id=selected.model_version.id) + datasets = read_input_datasets( + session, model_version_id=selected.model_version.id ) national_dataset = ( - read_input_dataset( - self._database_session.session, - model_version_id=selected.model_version.id, - dataset_id=national_region.default_dataset_id, + _validated_national_dataset( + selected, + country_id=country_id, + regions=regions, + find_dataset=lambda dataset_id: next( + (dataset for dataset in datasets if dataset.id == dataset_id), + None, + ), ) - if national_region is not None + if regions else None ) - dataset = validate_economy_options( + return metadata_model_catalog( selected, - country_id=country_id, regions=regions, + datasets=datasets, national_dataset=national_dataset, ) - return metadata_economy_options( - selected, regions=regions, national_dataset=dataset + + def get_catalog_variables( + self, country_id: str, policyengine_version: str | None = None + ) -> MetadataCatalogVariablesResult: + """Return every variable of the selected catalog version, in name order. + + It also returns the current-law values of each parameter a variable's + adds or subtracts names, so a client can pick the list for any year. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + variables = read_all_variables( + session, model_version_id=selected.model_version.id + ) + part_paths = part_parameter_paths(variables) + part_values = read_current_law_values_by_parameter_name( + session, + model_version_id=selected.model_version.id, + parameter_names=part_paths, + ) + return metadata_catalog_variables( + selected, + variables=variables, + part_paths=part_paths, + part_values=part_values, + ) + + def get_catalog_parameters( + self, + country_id: str, + policyengine_version: str | None = None, + *, + names: Sequence[str], + ) -> MetadataCatalogParametersResult: + """Return the named parameters of the selected version, in name order. + + Each carries its current-law values, and the result carries the stored + labels of their ancestor nodes and the requested names the version + lacks. Repeated names count once. + """ + selected = self._select_catalog(country_id, policyengine_version) + session = self._database_session.session + model_version_id = selected.model_version.id + requested_names = set(names) + parameters = read_parameters_by_name( + session, model_version_id=model_version_id, parameter_names=requested_names + ) + found_names = {parameter.name for parameter in parameters} + value_rows = read_current_law_values_by_parameter_name( + session, model_version_id=model_version_id, parameter_names=found_names + ) + node_rows = read_parameter_node_labels( + session, + model_version_id=model_version_id, + node_names={ + ancestor for name in found_names for ancestor in ancestor_paths(name) + }, + ) + return metadata_catalog_parameters( + selected, + parameters=parameters, + values_by_name=dated_values_by_parameter_name( + value_rows, parameter_names=found_names + ), + node_rows=node_rows, + missing_names=requested_names - found_names, ) + + +def _validated_national_dataset( + selected: SelectedCatalog, + *, + country_id: str, + regions: list[Region], + find_dataset: Callable[[UUID], Dataset | None], +) -> Dataset: + """Return the national region's default dataset, validated for economy choices. + + find_dataset loads a dataset of the selected version by id. Raise + MetadataCatalogUnavailableError when the national region, its dataset or + the model-version options are missing. + """ + region = national_region(regions, country_id) + return validate_economy_options( + selected, + country_id=country_id, + regions=regions, + national_dataset=( + None if region is None else find_dataset(region.default_dataset_id) + ), + ) diff --git a/policyengine_api/services/v2/metadata/transformations.py b/policyengine_api/services/v2/metadata/transformations.py index 68ebe21fe..67d860a56 100644 --- a/policyengine_api/services/v2/metadata/transformations.py +++ b/policyengine_api/services/v2/metadata/transformations.py @@ -7,11 +7,20 @@ from policyengine_api.dataset_display import get_dataset_display_label from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, + MetadataEnumValue, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -20,8 +29,10 @@ MetadataRegion, MetadataRegionOption, MetadataRegionType, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, + ParameterNodeLabelRow, ParameterValueSelection, ) @@ -103,9 +114,79 @@ def metadata_variable(variable: Variable) -> MetadataVariable: default_value=variable.default_value, adds=variable.adds, subtracts=variable.subtracts, + is_input_variable=variable.is_input_variable, + hidden_input=variable.hidden_input, + module_name=variable.module_name, + adds_parameter_path=variable.adds_parameter_path, + subtracts_parameter_path=variable.subtracts_parameter_path, + enum_values=( + None + if variable.enum_values is None + else [ + MetadataEnumValue.model_validate(enum_value) + for enum_value in variable.enum_values + ] + ), + unit=variable.unit, + definition_period=variable.definition_period, ) +def part_parameter_paths(variables: list[Variable]) -> set[str]: + """Return every parameter path the variables' adds or subtracts names.""" + return { + path + for variable in variables + for path in (variable.adds_parameter_path, variable.subtracts_parameter_path) + if path is not None + } + + +def metadata_catalog_variables( + selected: SelectedCatalog, + *, + variables: list[Variable], + part_paths: set[str], + part_values: list[CurrentLawValueRow], +) -> MetadataCatalogVariablesResult: + """Build every variable of the selected version and its part parameters' values. + + part_paths holds part_parameter_paths of the variables, and part_values the + current-law values of those parameters in name and start-date order. Each + path gets an entry, empty when no value carries its name. + """ + values_by_path = dated_values_by_parameter_name( + part_values, parameter_names=part_paths + ) + return MetadataCatalogVariablesResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + variables=[metadata_variable(variable) for variable in variables], + part_parameter_values=values_by_path, + ) + + +def dated_values_by_parameter_name( + value_rows: list[CurrentLawValueRow], *, parameter_names: set[str] +) -> dict[str, list[MetadataDatedValue]]: + """Group value rows by parameter name, with an empty list for each other name. + + value_rows are in parameter-name and start-date order. + """ + values_by_name: dict[str, list[MetadataDatedValue]] = { + name: [] for name in sorted(parameter_names) + } + for row in value_rows: + values_by_name.setdefault(row.name, []).append( + MetadataDatedValue( + start_date=as_utc(row.start_date), + end_date=None if row.end_date is None else as_utc(row.end_date), + value=row.value_json, + ) + ) + return values_by_name + + def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: """Return the API representation of a stored parameter.""" return MetadataParameterSummary( @@ -115,6 +196,51 @@ def metadata_parameter(parameter: Parameter) -> MetadataParameterSummary: description=parameter.description, data_type=parameter.data_type, unit=parameter.unit, + economy=parameter.economy, + household=parameter.household, + ) + + +def ancestor_paths(parameter_name: str) -> list[str]: + """Return the paths of every node above parameter_name, from the root down. + + A dot separates a child, and so does the ``[`` of a bracket index: + ``gov.tax.rates[0].rate`` has the ancestors ``gov``, ``gov.tax``, + ``gov.tax.rates`` and ``gov.tax.rates[0]``. + """ + return [ + parameter_name[:position] + for position, character in enumerate(parameter_name) + if character in ".[" and position > 0 + ] + + +def metadata_catalog_parameters( + selected: SelectedCatalog, + *, + parameters: list[Parameter], + values_by_name: dict[str, list[MetadataDatedValue]], + node_rows: list[ParameterNodeLabelRow], + missing_names: set[str], +) -> MetadataCatalogParametersResult: + """Build the requested parameters with their values and ancestor labels. + + values_by_name maps each parameter's name to its current-law values (see + dated_values_by_parameter_name), node_rows are the stored labels of their + ancestor nodes, and missing_names the requested names the version lacks. + """ + return MetadataCatalogParametersResult( + policyengine_version=selected.policyengine_version, + catalog_format=selected.model_version.catalog_format, + parameters=[ + MetadataCatalogParameter( + **metadata_parameter(parameter).model_dump(), + values=values_by_name[parameter.name], + ) + for parameter in parameters + ], + node_labels={row.name: row.label for row in node_rows}, + missing_names=sorted(missing_names), ) @@ -129,6 +255,18 @@ def metadata_parameter_value(value: ParameterValue) -> MetadataCanonicalParamete ) +def as_utc(moment: datetime) -> datetime: + """Return moment in UTC, treating a naive time as UTC. + + The database may return times in its session's time zone (or, in SQLite, + without one); serving them in UTC keeps a response's bytes, and so its + ETag, the same whatever that setting is. + """ + if moment.tzinfo is None: + return moment.replace(tzinfo=UTC) + return moment.astimezone(UTC) + + def utc_day_start(selected_time: datetime) -> datetime: """Return midnight UTC on the day of selected_time, treating a naive time as UTC.""" if selected_time.tzinfo is None: @@ -153,6 +291,7 @@ def parameter_value_selection( return ParameterValueSelection( parameter_id=value_filter.parameter_id, selected_day=selected_day, + parameter_names=value_filter.parameter_names, ) @@ -187,31 +326,26 @@ def metadata_region(region: Region) -> MetadataRegion: def metadata_parameter_children(rows: list[Any]) -> list[MetadataParameterChild]: """Build parameter-tree children from read_parameter_children rows. - Parameter leaves carry the parameter's summary, and a child with no label is - labelled with the last segment of its path. + Parameter leaves carry the same summary as the parameter routes, and a child + with no label is labelled with the last segment of its path. """ - items = [] - for row in rows: - parameter = None - if row.type == "parameter": - parameter = MetadataParameterSummary( - id=row.parameter_id, - name=row.path, - label=row.parameter_label, - description=row.parameter_description, - data_type=row.parameter_data_type, - unit=row.parameter_unit, - ) - items.append( - MetadataParameterChild( - path=row.path, - label=row.label or row.path.rsplit(".", 1)[-1], - type=row.type, - child_count=row.child_count, - parameter=parameter, - ) + return [ + MetadataParameterChild( + path=row.path, + label=row.label or row.path.rsplit(".", 1)[-1], + type=row.type, + child_count=row.child_count, + parameter=( + metadata_parameter(row.Parameter) if row.type == "parameter" else None + ), ) - return items + for row in rows + ] + + +def national_region(regions: list[Region], country_id: str) -> Region | None: + """Return the country's national region (its code is the country ID), or None.""" + return next((region for region in regions if region.code == country_id), None) def metadata_economy_options( @@ -221,9 +355,22 @@ def metadata_economy_options( The national dataset is the only dataset choice, and it is the default. """ + options = _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) return MetadataEconomyOptionsResult( policyengine_version=selected.policyengine_version, current_law_id=selected.model_version.current_law_id, + region=options.region, + time_period=options.time_period, + datasets=options.datasets, + ) + + +def _economy_option_lists( + selected: SelectedCatalog, *, regions: list[Region], national_dataset: Dataset +) -> MetadataEconomyOptions: + return MetadataEconomyOptions( region=[ MetadataRegionOption( name=region.code, @@ -243,3 +390,43 @@ def metadata_economy_options( ) ], ) + + +def metadata_model_catalog( + selected: SelectedCatalog, + *, + regions: list[Region], + datasets: list[Dataset], + national_dataset: Dataset | None, +) -> MetadataModelCatalogResult: + """Build the small parts of the selected catalog version. + + Without a national dataset (a catalog without regions) the economy choices + are empty. Entities and basic inputs a version published before catalog + format 2 lacks are empty lists, and a version whose SPM availability is + not recorded yet offers none. + """ + model_version = selected.model_version + return MetadataModelCatalogResult( + policyengine_version=selected.policyengine_version, + catalog_format=model_version.catalog_format, + model=metadata_model(selected), + model_version=metadata_model_version(selected), + entities=[ + MetadataEntity.model_validate(entity) + for entity in model_version.entities or [] + ], + basic_inputs=list(model_version.basic_inputs or []), + regions=[metadata_region(region) for region in regions], + datasets=[metadata_dataset(dataset) for dataset in datasets], + economy_options=( + _economy_option_lists( + selected, regions=regions, national_dataset=national_dataset + ) + if national_dataset is not None + else MetadataEconomyOptions(region=[], time_period=[], datasets=[]) + ), + spm=MetadataSPMAvailability.model_validate( + model_version.spm_availability or {"available": False} + ), + ) diff --git a/policyengine_api/services/v2/metadata/types.py b/policyengine_api/services/v2/metadata/types.py index 7f41e1575..eb15bca55 100644 --- a/policyengine_api/services/v2/metadata/types.py +++ b/policyengine_api/services/v2/metadata/types.py @@ -5,34 +5,65 @@ from dataclasses import dataclass from datetime import datetime from enum import StrEnum -from typing import Generic, Literal, TypeVar +from typing import Any, Generic, Literal, NamedTuple, TypeVar from uuid import UUID from pydantic import BaseModel, ConfigDict, JsonValue +@dataclass(frozen=True) +class ParameterFilter: + """Which parameters a listing keeps. + + ``search`` keeps parameters whose name, label or description contains it, + ignoring case, and ``names`` the parameters with those exact names. + """ + + search: str | None = None + names: tuple[str, ...] | None = None + + @dataclass(frozen=True) class ParameterValueFilter: """Which current-law parameter values a listing keeps. - ``parameter_id`` keeps one parameter's values, and ``current`` keeps the - values in effect on the UTC day of ``now`` (today when ``now`` is None). + ``parameter_id`` keeps one parameter's values, ``parameter_names`` the + values of the parameters with those exact names, and ``current`` the values + in effect on the UTC day of ``now`` (today when ``now`` is None). """ parameter_id: UUID | None = None current: bool = False now: datetime | None = None + parameter_names: tuple[str, ...] | None = None @dataclass(frozen=True) class ParameterValueSelection: - """The parameter and effective day a parameter-value read is narrowed to. + """The parameters and effective day a parameter-value read is narrowed to. - None for either leaves the read unnarrowed by it. + None for any of them leaves the read unnarrowed by it. """ parameter_id: UUID | None selected_day: datetime | None + parameter_names: tuple[str, ...] | None = None + + +class CurrentLawValueRow(NamedTuple): + """One current-law value of a parameter, read with the parameter's name.""" + + name: str + value_json: Any + start_date: datetime + end_date: datetime | None + + +class ParameterNodeLabelRow(NamedTuple): + """A parameter node's name and its stored label (None when it has none).""" + + name: str + label: str | None class StrictResponseModel(BaseModel): @@ -71,11 +102,29 @@ class MetadataModelVersion(StrictResponseModel): description: str | None +class MetadataEnumValue(StrictResponseModel): + """One value of an enum variable: its stored name and its display label. + + The label keeps the type the country package declares it with. + """ + + value: str + label: str | int | float | bool | None + + class MetadataVariable(StrictResponseModel): """A variable the tax-benefit model computes or accepts as input. adds and subtracts, when present, list the variables summed into or - subtracted from this one. + subtracted from this one. When the model takes that list from a + parameter, adds_parameter_path or subtracts_parameter_path names the + parameter, whose dated values give the list for each year (adds and + subtracts hold its value on 2025-01-01). unit is the country package's + unit (for example ``currency-USD``) and definition_period the period a + value covers (``year``, ``month`` or ``eternity``). The catalog format 2 + fields (is_input_variable, hidden_input, module_name, the two parameter + paths, enum_values, unit and definition_period) are null in versions + published before that format. """ id: UUID @@ -88,12 +137,22 @@ class MetadataVariable(StrictResponseModel): default_value: JsonValue adds: list[str] | None subtracts: list[str] | None + is_input_variable: bool | None + hidden_input: bool | None + module_name: str | None + adds_parameter_path: str | None + subtracts_parameter_path: str | None + enum_values: list[MetadataEnumValue] | None + unit: str | None + definition_period: str | None class MetadataParameterSummary(StrictResponseModel): """A policy parameter, described without its values. - Its current-law values are listed by GET /v2/parameter-values. + Its current-law values are listed by GET /v2/parameter-values. economy and + household say whether a reform of an economy or of a household may change + it; they are null in versions published before catalog format 2. """ id: UUID @@ -102,6 +161,8 @@ class MetadataParameterSummary(StrictResponseModel): description: str | None data_type: str | None unit: str | None + economy: bool | None + household: bool | None class MetadataCanonicalParameterValue(StrictResponseModel): @@ -136,8 +197,10 @@ class MetadataDataset(StrictResponseModel): class MetadataRegion(StrictResponseModel): """A region the model can simulate, such as a country, state or constituency. - When requires_filter is true, the region is simulated by filtering its - default dataset on filter_field and filter_value. + When requires_filter is true, filter_strategy says how the region selects + households of its default dataset. With "row_filter", it keeps the + households whose filter_field column equals filter_value (or is in it, + when filter_value is a JSON list). """ id: UUID @@ -237,3 +300,114 @@ class MetadataEconomyOptionsResult(StrictResponseModel): region: list[MetadataRegionOption] time_period: list[MetadataTimePeriodOption] datasets: list[MetadataDatasetOption] + + +class MetadataEntity(StrictResponseModel): + """A kind of member a household holds, such as a person or a tax unit. + + plural is the key the household's members of this kind are listed under, + and is_person marks the entity every household input about a person uses. + """ + + key: str + plural: str + is_person: bool + + +class MetadataEconomyOptions(StrictResponseModel): + """The region, year and dataset choices for an economy calculation. + + Every list is empty for a catalog without regions, such as a + household-only country's, which offers no economy calculation. + """ + + region: list[MetadataRegionOption] + time_period: list[MetadataTimePeriodOption] + datasets: list[MetadataDatasetOption] + + +class MetadataSPMAvailability(StrictResponseModel): + """Whether a catalog version offers SPM settings for a household. + + When ``available``, ``settings_schema`` is the JSON schema of a + household's ``spm`` selection and ``defaults`` the certified selection + that applies to every field a household leaves out. A version without + them (any country but the US, or a bundle without certified SPM support) + is not available, and both are null. + """ + + available: bool + settings_schema: dict[str, JsonValue] | None = None + defaults: dict[str, JsonValue] | None = None + + +class MetadataModelCatalogResult(StrictResponseModel): + """The small parts of one catalog version, read when a client starts. + + policyengine_version names the selected catalog version, so a client can + request its other parts for that exact version. catalog_format is null, + and entities and basic_inputs are empty, for a version published before + catalog format 2. ``spm`` says whether the version offers SPM settings. + Variables, parameters and their values are served by their own routes. + """ + + policyengine_version: str + catalog_format: int | None + model: MetadataModel + model_version: MetadataModelVersionDetail + entities: list[MetadataEntity] + basic_inputs: list[str] + regions: list[MetadataRegion] + datasets: list[MetadataDataset] + economy_options: MetadataEconomyOptions + spm: MetadataSPMAvailability + + +class MetadataDatedValue(StrictResponseModel): + """A current-law parameter value from start_date until end_date. + + A null end_date means the value applies until a later value starts, or + indefinitely. + """ + + start_date: datetime + end_date: datetime | None + value: JsonValue + + +class MetadataCatalogVariablesResult(StrictResponseModel): + """Every variable of one catalog version, in name order. + + part_parameter_values has an entry for each parameter path a variable's + adds_parameter_path or subtracts_parameter_path names: that parameter's + current-law values in start-date order, or an empty list when the + catalog has no such parameter. + """ + + policyengine_version: str + catalog_format: int | None + variables: list[MetadataVariable] + part_parameter_values: dict[str, list[MetadataDatedValue]] + + +class MetadataCatalogParameter(MetadataParameterSummary): + """A parameter with its current-law values in start-date order.""" + + values: list[MetadataDatedValue] + + +class MetadataCatalogParametersResult(StrictResponseModel): + """Requested parameters of one catalog version, in name order. + + node_labels maps each ancestor of a requested parameter that the version + stores as a parameter node to the node's stored label (null when it has + none), for breadcrumbs; a bracket index such as ``rates[0]`` counts as a + child of its schedule ``rates``. missing_names lists requested names the + version has no parameter for. + """ + + policyengine_version: str + catalog_format: int | None + parameters: list[MetadataCatalogParameter] + node_labels: dict[str, str | None] + missing_names: list[str] diff --git a/policyengine_api/services/v2/user_households/database_connectors/updates.py b/policyengine_api/services/v2/user_households/database_connectors/updates.py index ed6cfb989..e0de95d8e 100644 --- a/policyengine_api/services/v2/user_households/database_connectors/updates.py +++ b/policyengine_api/services/v2/user_households/database_connectors/updates.py @@ -5,14 +5,16 @@ from typing import TYPE_CHECKING from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_households.types import ( + USER_HOUSEHOLD_REFERENCE_FIELDS, +) if TYPE_CHECKING: - from uuid import UUID - from sqlmodel import Session from policyengine_api.data.v2.models import UserHouseholdAssociation from policyengine_api.services.v2.user_households.types import ( + UserHouseholdReferences, UserHouseholdUpdateInput, ) @@ -22,17 +24,20 @@ def update_user_household( association: UserHouseholdAssociation, association_input: UserHouseholdUpdateInput, *, - repointed_household_id: UUID | None = None, - repoint: bool = False, + references: UserHouseholdReferences | None = None, ) -> UserHouseholdAssociation: - """Apply the supplied fields, bump ``updated_at`` and return the row. + """Apply the supplied presentation fields, bump ``updated_at`` and return the row. - With ``repoint``, also set ``household_id`` to ``repointed_household_id``. + With ``references``, also store both of its household references; the + service resolves them, so the request's own reference fields are never copied. """ - for field_name, value in association_input.model_dump(exclude_unset=True).items(): + for field_name, value in association_input.model_dump( + exclude_unset=True, exclude=set(USER_HOUSEHOLD_REFERENCE_FIELDS) + ).items(): setattr(association, field_name, value) - if repoint: - association.household_id = repointed_household_id + if references is not None: + association.household_id = references.household_id + association.legacy_household_id = references.legacy_household_id association.updated_at = utc_now() session.add(association) session.flush() diff --git a/policyengine_api/services/v2/user_households/services.py b/policyengine_api/services/v2/user_households/services.py index a2630e897..e2c791b76 100644 --- a/policyengine_api/services/v2/user_households/services.py +++ b/policyengine_api/services/v2/user_households/services.py @@ -42,6 +42,7 @@ from policyengine_api.services.v2.user_households.types import ( UserHouseholdFilter, UserHouseholdListing, + UserHouseholdReferences, ) from policyengine_api.services.v2.user_households.validators import ( require_user_household, @@ -267,12 +268,17 @@ def patch_user_household( ) -> UserHouseholdRead: """Update the supplied fields of the principal's association in this country. - A new ``legacy_household_id`` must exist in v1 and repoints ``household_id`` - to the v2 copy of that household, or to None when there is no copy. Raise - UserHouseholdNotFoundError when the association is not the principal's. + The association keeps its id, owner, country and creation time. A new + ``legacy_household_id`` must exist in v1 (LegacyReferenceNotFoundError) and + repoints ``household_id`` to the v2 copy of that household, or to None when + there is no copy. A new ``household_id`` must exist in this country + (AssociationHouseholdNotFoundError or AssociationCountryConflictError). + Without ``legacy_household_id`` it clears the v1 reference; with it, both + are stored, and it must be the v2 copy of that v1 household when one exists + (AssociationReferenceConflictError). Raise UserHouseholdNotFoundError when + the association is not the principal's. """ - repoint = "legacy_household_id" in association_input.model_fields_set - if repoint: + if "legacy_household_id" in association_input.model_fields_set: if association_input.legacy_household_id is None: msg = "legacy_household_id must not be null" raise TypeError(msg) @@ -290,13 +296,17 @@ def patch_user_household( country_id=country_id, association_id=association_id, ) - repointed_household_id = ( - read_copied_household_ids( - session, - country_id=country_id, - legacy_household_ids=(association_input.legacy_household_id,), - ).get(association_input.legacy_household_id) - if repoint and association_input.legacy_household_id is not None + references = ( + UserHouseholdReferences( + household_id=resolve_user_household_reference( + session, + country_id=country_id, + household_id=association_input.household_id, + legacy_household_id=association_input.legacy_household_id, + ), + legacy_household_id=association_input.legacy_household_id, + ) + if association_input.repoints else None ) return user_household_read( @@ -304,8 +314,7 @@ def patch_user_household( session, association, association_input, - repointed_household_id=repointed_household_id, - repoint=repoint, + references=references, ) ) diff --git a/policyengine_api/services/v2/user_households/types.py b/policyengine_api/services/v2/user_households/types.py index 599c1e35d..40309e868 100644 --- a/policyengine_api/services/v2/user_households/types.py +++ b/policyengine_api/services/v2/user_households/types.py @@ -61,33 +61,56 @@ def require_household_reference(self) -> UserHouseholdCreationInput: return self +USER_HOUSEHOLD_REFERENCE_FIELDS = frozenset({"household_id", "legacy_household_id"}) + + class UserHouseholdUpdateInput(StrictUserHouseholdInput): """A partial update of a saved household; only supplied fields are changed. - At least one field must be supplied. ``legacy_household_id`` may point the - association at another v1 household but cannot be cleared. + At least one field must be supplied. ``household_id`` points the association + at another v2 household; supplied without ``legacy_household_id``, it also + clears the v1 reference, which no longer describes the saved household. + ``legacy_household_id`` alone points the association at another v1 household + and takes ``household_id`` from that household's v2 copy, or clears it when + there is none. Supplied together, both are stored, and ``household_id`` must + be that v2 copy when one exists. Neither reference can be set to null. """ + household_id: ResourceId | None = None legacy_household_id: LegacyRecordId | None = None name: AssociationName | None = None description: AssociationDescription | None = None @model_validator(mode="after") def require_valid_supplied_field(self) -> UserHouseholdUpdateInput: - """Require a supplied field, and reject an explicit null legacy_household_id.""" - mutable_fields = {"legacy_household_id", "name", "description"} + """Require a supplied field, and reject an explicit null household reference.""" + mutable_fields = {*USER_HOUSEHOLD_REFERENCE_FIELDS, "name", "description"} supplied = self.model_fields_set.intersection(mutable_fields) if not supplied: msg = ( - "At least one of legacy_household_id, name, or description must be " - "supplied" + "At least one of household_id, legacy_household_id, name, or " + "description must be supplied" ) raise ValueError(msg) - if "legacy_household_id" in supplied and self.legacy_household_id is None: - msg = "legacy_household_id must not be null" - raise ValueError(msg) + for reference_field in sorted(USER_HOUSEHOLD_REFERENCE_FIELDS): + if reference_field in supplied and getattr(self, reference_field) is None: + msg = f"{reference_field} must not be null" + raise ValueError(msg) return self + @property + def repoints(self) -> bool: + """Whether the update changes which household the association references.""" + return bool(self.model_fields_set & USER_HOUSEHOLD_REFERENCE_FIELDS) + + +@dataclass(frozen=True) +class UserHouseholdReferences: + """The v2 and v1 household references a repointed association stores.""" + + household_id: UUID | None + legacy_household_id: int | None + @dataclass(frozen=True) class UserHouseholdRead: diff --git a/policyengine_api/services/v2/user_policies/database_connectors/updates.py b/policyengine_api/services/v2/user_policies/database_connectors/updates.py index 328c3d32d..2208e5636 100644 --- a/policyengine_api/services/v2/user_policies/database_connectors/updates.py +++ b/policyengine_api/services/v2/user_policies/database_connectors/updates.py @@ -5,15 +5,17 @@ from typing import TYPE_CHECKING from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.services.v2.user_policies.types import ( + USER_POLICY_REFERENCE_FIELDS, +) if TYPE_CHECKING: - from uuid import UUID - from sqlmodel import Session from policyengine_api.data.v2.models import LegacyUserPolicyMapping, UserPolicy from policyengine_api.services.v2.user_policies.types import ( LegacyUserPolicyRevision, + UserPolicyReferences, UserPolicyUpdateInput, ) @@ -23,17 +25,20 @@ def update_user_policy( association: UserPolicy, association_input: UserPolicyUpdateInput, *, - repointed_policy_id: UUID | None = None, - repoint: bool = False, + references: UserPolicyReferences | None = None, ) -> UserPolicy: - """Apply the supplied fields, bump ``updated_at`` and return the row. + """Apply the supplied presentation fields, bump ``updated_at`` and return the row. - With ``repoint``, also set ``policy_id`` to ``repointed_policy_id``. + With ``references``, also store both of its policy references; the service + resolves them, so the request's own reference fields are never copied. """ - for field_name, value in association_input.model_dump(exclude_unset=True).items(): + for field_name, value in association_input.model_dump( + exclude_unset=True, exclude=set(USER_POLICY_REFERENCE_FIELDS) + ).items(): setattr(association, field_name, value) - if repoint: - association.policy_id = repointed_policy_id + if references is not None: + association.policy_id = references.policy_id + association.legacy_policy_id = references.legacy_policy_id association.updated_at = utc_now() session.add(association) session.flush() diff --git a/policyengine_api/services/v2/user_policies/services.py b/policyengine_api/services/v2/user_policies/services.py index 4ccbad927..8bf3dbe4a 100644 --- a/policyengine_api/services/v2/user_policies/services.py +++ b/policyengine_api/services/v2/user_policies/services.py @@ -61,6 +61,7 @@ UserPolicyListing, UserPolicyPage, UserPolicyRead, + UserPolicyReferences, UserPolicyUpdateInput, ) from policyengine_api.services.v2.user_policies.validators import ( @@ -500,13 +501,18 @@ def patch_user_policy( ) -> UserPolicyRead: """Update the supplied fields of the principal's association in this country. - A new ``legacy_policy_id`` must exist in v1 and repoints ``policy_id`` to the - v2 copy of that policy, or to None when there is no copy. Raise - UserPolicyNotFoundError when the association is not the principal's, and - MirrorOwnedAssociationError when the v1 saved-policy mirror maintains it. + The association keeps its id, owner, country and creation time. A new + ``legacy_policy_id`` must exist in v1 (LegacyReferenceNotFoundError) and + repoints ``policy_id`` to the v2 copy of that policy, or to None when there + is no copy. A new ``policy_id`` must exist in this country + (AssociationPolicyNotFoundError or AssociationCountryConflictError). + Without ``legacy_policy_id`` it clears the v1 reference; with it, both are + stored, and it must be the v2 copy of that v1 policy when one exists + (AssociationReferenceConflictError). Raise UserPolicyNotFoundError when the + association is not the principal's, and MirrorOwnedAssociationError when + the v1 saved-policy mirror maintains it. """ - repoint = "legacy_policy_id" in association_input.model_fields_set - if repoint: + if "legacy_policy_id" in association_input.model_fields_set: if association_input.legacy_policy_id is None: msg = "legacy_policy_id must not be null" raise TypeError(msg) @@ -530,13 +536,17 @@ def patch_user_policy( ), association_id=association.id, ) - repointed_policy_id = ( - read_mirrored_policy_ids( - session, - country_id=country_id, - legacy_policy_ids=(association_input.legacy_policy_id,), - ).get(association_input.legacy_policy_id) - if repoint and association_input.legacy_policy_id is not None + references = ( + UserPolicyReferences( + policy_id=resolve_user_policy_reference( + session, + country_id=country_id, + policy_id=association_input.policy_id, + legacy_policy_id=association_input.legacy_policy_id, + ), + legacy_policy_id=association_input.legacy_policy_id, + ) + if association_input.repoints else None ) return user_policy_read( @@ -544,8 +554,7 @@ def patch_user_policy( session, association, association_input, - repointed_policy_id=repointed_policy_id, - repoint=repoint, + references=references, ) ) diff --git a/policyengine_api/services/v2/user_policies/types.py b/policyengine_api/services/v2/user_policies/types.py index c24b2d49c..296fc821c 100644 --- a/policyengine_api/services/v2/user_policies/types.py +++ b/policyengine_api/services/v2/user_policies/types.py @@ -82,13 +82,22 @@ def require_policy_reference(self) -> UserPolicyCreationInput: return self +USER_POLICY_REFERENCE_FIELDS = frozenset({"policy_id", "legacy_policy_id"}) + + class UserPolicyUpdateInput(StrictAssociationInput): """A partial update of a saved policy; only supplied fields are changed. - At least one field must be supplied. ``legacy_policy_id`` may point the - association at another v1 policy but cannot be cleared. + At least one field must be supplied. ``policy_id`` points the association at + another v2 policy; supplied without ``legacy_policy_id``, it also clears the + v1 reference, which no longer describes the saved policy. ``legacy_policy_id`` + alone points the association at another v1 policy and takes ``policy_id`` from + that policy's v2 copy, or clears it when there is none. Supplied together, + both are stored, and ``policy_id`` must be that v2 copy when one exists. + Neither reference can be set to null. """ + policy_id: ResourceId | None = None legacy_policy_id: LegacyRecordId | None = None name: AssociationName | None = None description: AssociationDescription | None = None @@ -97,21 +106,41 @@ class UserPolicyUpdateInput(StrictAssociationInput): @model_validator(mode="after") def require_supplied_field(self) -> UserPolicyUpdateInput: - """Require a supplied field, and reject an explicit null legacy_policy_id.""" + """Require a supplied field, and reject an explicit null policy reference.""" supplied = self.model_fields_set.intersection( - {"legacy_policy_id", "name", "description", "baseline", "provenance"} + { + *USER_POLICY_REFERENCE_FIELDS, + "name", + "description", + "baseline", + "provenance", + } ) if not supplied: msg = ( - "At least one of legacy_policy_id, name, description, baseline, " - "or provenance must be supplied" + "At least one of policy_id, legacy_policy_id, name, description, " + "baseline, or provenance must be supplied" ) raise ValueError(msg) - if "legacy_policy_id" in supplied and self.legacy_policy_id is None: - msg = "legacy_policy_id must not be null" - raise ValueError(msg) + for reference_field in sorted(USER_POLICY_REFERENCE_FIELDS): + if reference_field in supplied and getattr(self, reference_field) is None: + msg = f"{reference_field} must not be null" + raise ValueError(msg) return self + @property + def repoints(self) -> bool: + """Whether the update changes which policy the association references.""" + return bool(self.model_fields_set & USER_POLICY_REFERENCE_FIELDS) + + +@dataclass(frozen=True) +class UserPolicyReferences: + """The v2 and v1 policy references a repointed association stores.""" + + policy_id: UUID | None + legacy_policy_id: int | None + class LegacyUserPolicySnapshot(StrictAssociationInput): """Detached complete committed v1 saved-policy row.""" diff --git a/policyengine_api/startup_country_check.py b/policyengine_api/startup_country_check.py new file mode 100644 index 000000000..69f233988 --- /dev/null +++ b/policyengine_api/startup_country_check.py @@ -0,0 +1,106 @@ +"""Refuse to start when the v2 countries table and the country list differ. + +Which countries the Public API serves is decided in code +(``policyengine_api.supported_countries``). The v2 ``countries`` table records +what each supports, and must hold one row for each listed country and no +other. ``policyengine_api.asgi`` runs this check while the application loads, +so a disagreement stops the worker before it serves a request. + +The check runs only where a v2 runtime database is selected: on Cloud Run +(``K_SERVICE`` is set), or when a v2 runtime database URL or its secret +resource is configured. A local or test startup without v2 configuration +opens no connection. + +Startup stops when the database answers and its answer disagrees with the +list: different countries, a missing table, or no privilege to read it. +When the check cannot get an answer, it logs why and startup continues: the +v1 routes do not read the v2 database, and their copies into it already +continue without it. That covers a database that cannot be reached or +refuses the credentials, and v2 configuration that cannot be used, including +a secret that cannot be resolved; the readiness check already reports the +last as not ready, and recovers once the secret resolves. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Protocol + +from sqlalchemy.exc import InterfaceError, OperationalError + +from policyengine_api.data.v2.settings import ( + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, + V2ConfigurationError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + +logger = logging.getLogger(__name__) + +CLOUD_RUN_SERVICE = "K_SERVICE" +V2_RUNTIME_SELECTION_VARIABLES = ( + CLOUD_RUN_SERVICE, + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, +) + + +class CountryListVerifier(Protocol): + """Compare the stored countries rows with a list of country IDs.""" + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these.""" + ... + + +def _default_country_list_verifier() -> CountryListVerifier: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + +def v2_runtime_database_selected(environ: Mapping[str, str]) -> bool: + """Whether this process runs on Cloud Run or has a v2 runtime database.""" + return any(environ.get(name, "").strip() for name in V2_RUNTIME_SELECTION_VARIABLES) + + +def check_countries_table_at_startup( + environ: Mapping[str, str] | None = None, + verifier_factory: Callable[[], CountryListVerifier] | None = None, +) -> None: + """Raise unless the countries table names exactly the countries in code. + + Raises ``CountryListMismatchError`` on a disagreement and SQLAlchemy's + ``ProgrammingError`` when the table or the privilege to read it is + missing. Unusable v2 configuration is logged with its message, which + never contains a secret, and a database connection failure by its error + type only, as its detail can name the host; both skip the check. + """ + values = os.environ if environ is None else environ + if not v2_runtime_database_selected(values): + return + factory = verifier_factory or _default_country_list_verifier + try: + factory().verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + except V2ConfigurationError as error: + logger.warning( + "Skipped the countries table check at startup: the v2 runtime " + "configuration cannot be used (%s)", + error, + ) + except (OperationalError, InterfaceError) as error: + logger.warning( + "Skipped the countries table check at startup: the v2 database " + "could not be queried (%s)", + type(error).__name__, + ) diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py new file mode 100644 index 000000000..0c5ad01a1 --- /dev/null +++ b/policyengine_api/supported_countries.py @@ -0,0 +1,33 @@ +"""The countries the Public API serves, listed once in code. + +``SupportedCountryId`` is the one country list. Every other list derives from +it: the v2 routes' ``CountryId`` type, the countries catalog extraction +publishes, and the countries the v1 routes copy into v2 (dual-write). Adding +a country starts here. + +The v2 ``countries`` table records what each country supports, and must hold +one row for each listed country and no other: a v2 Alembic revision inserts +the row when a country is added. The application refuses to start when they +differ (``policyengine_api.startup_country_check``). Because a deploy runs +that revision before it starts the new release, instances of the previous +release that start afterwards, or a rollback to it, refuse to start until +the revision is downgraded. + +A new country also needs the US-and-UK checks widened on the tables that +pair v1 records with their v2 copies (``legacy_policy_mappings``, +``legacy_user_policy_mappings``, ``legacy_household_mappings``) and on the +v1 ``household_mirror_events`` table, which the v1 household insert writes +in its own transaction. + +The module imports nothing from the package, so modules that the query +models import, such as the catalog package, can import it too. +""" + +from __future__ import annotations + +from typing import Final, Literal, get_args + +SupportedCountryId = Literal["us", "uk"] +SUPPORTED_COUNTRY_IDS: Final[tuple[SupportedCountryId, ...]] = get_args( + SupportedCountryId +) diff --git a/policyengine_api/vary_headers.py b/policyengine_api/vary_headers.py new file mode 100644 index 000000000..98d834086 --- /dev/null +++ b/policyengine_api/vary_headers.py @@ -0,0 +1,56 @@ +"""ASGI middleware that lists each Vary value of a response once.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from starlette.datastructures import MutableHeaders + +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send + + +def unique_vary_values(vary: str) -> str: + """Return ``vary`` with each value listed once, in first-seen order. + + Values are compared without regard to case, as HTTP field names are. + """ + values: list[str] = [] + seen: set[str] = set() + for value in vary.split(","): + name = value.strip() + if name and name.lower() not in seen: + seen.add(name.lower()) + values.append(name) + return ", ".join(values) + + +class UniqueVaryMiddleware: + """List each Vary value of every HTTP response once. + + Starlette's CORS and gzip middleware append Origin and Accept-Encoding to + Vary without checking whether a route already named them. This wraps the + outermost middleware, so the response that leaves the application names + each value once. + """ + + def __init__(self, app: ASGIApp) -> None: + """Wrap ``app``.""" + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + """Pass the request through, rewriting a duplicated Vary header.""" + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_unique_vary(message: Message) -> None: + if message["type"] == "http.response.start": + message.setdefault("headers", []) + headers = MutableHeaders(scope=message) + vary = headers.get("vary") + if vary is not None: + headers["vary"] = unique_vary_values(vary) + await send(message) + + await self.app(scope, receive, send_with_unique_vary) diff --git a/pyproject.toml b/pyproject.toml index ef1bee9d6..234fc0e6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,10 @@ dependencies = [ "google-cloud-logging>=3,<4", "google-cloud-secret-manager>=2,<3", "gunicorn", + # Catalog publication reads certified HDF5 artifacts from Hugging Face. + "h5py>=3.16,<4", "httpx>=0.27.0", + "huggingface-hub>=1.12,<2", "markupsafe>=3,<4", "microdf_python>=1.0.0", "openai", @@ -61,6 +64,7 @@ dependencies = [ "psycopg[binary]>=3.3,<4", "pyjwt[crypto]>=2.10,<3", "typing-extensions>=4.4,<5", + "policyengine-api-simulation-client>=0.20261027.288", ] [project.optional-dependencies] @@ -158,6 +162,8 @@ files = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] explicit_package_bases = true follow_imports = "silent" @@ -212,6 +218,8 @@ banned-module-level-imports = [ "policyengine", "policyengine_api.data.v2.database", "policyengine_api.libs.auth0_management", + "policyengine_api.services.v2.countries.database_session", + "policyengine_api.services.v2.countries.services", "policyengine_api.services.v2.households.database_session", "policyengine_api.services.v2.households.services", "policyengine_api.services.v2.legacy_refs.database_session", @@ -242,6 +250,7 @@ runtime-evaluated-base-classes = [ "policyengine_api.data.v2.models.base.TimestampedModel", "policyengine_api.data.v2.models.comparison_runs.Stage12RunTimestamps", "policyengine_api.data.v2.models.reports.GeographicImpactBase", + "policyengine_api.fastapi_routes.v2.household_calculations.response_models.StrictHouseholdCalculationAPIModel", "policyengine_api.fastapi_routes.v2.households.document_models.HouseholdEntityRecord", "policyengine_api.fastapi_routes.v2.households.document_models.StrictHouseholdDocumentModel", "policyengine_api.fastapi_routes.v2.households.response_models.HouseholdSuccessResponse", @@ -266,6 +275,8 @@ runtime-evaluated-base-classes = [ "policyengine_api.query_parameters.PaginationQuery", "policyengine_api.query_parameters.StrictQueryParameters", "policyengine_api.services.v2.associations.types.StrictAssociationModel", + "policyengine_api.services.v2.household_calculations.types.HouseholdCalculationInput", + "policyengine_api.services.v2.household_calculations.types.StrictCalculationModel", "policyengine_api.services.v2.households.types.HouseholdCreationInput", "policyengine_api.services.v2.households.types.StrictHouseholdInput", "policyengine_api.services.v2.metadata.types.MetadataModelVersion", diff --git a/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py b/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py index 5437a95c0..0f7567704 100644 --- a/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py +++ b/rules/fixtures/api.v2.runtime-privileges-declared/reads_good.py @@ -1,7 +1,7 @@ """Good fixture: a runtime read of only the tables the runtime role is granted. -The user-policy lookup reads ``user_policies``, which the runtime role holds -SELECT on, and nothing else. +The user-policy lookup reads ``user_policy_associations``, which the runtime +role holds SELECT on, and nothing else. """ from __future__ import annotations diff --git a/rules/registry.toml b/rules/registry.toml index 66b86bb8d..b322b3fc3 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -395,7 +395,7 @@ invocation = "make test" [[rule]] id = "api.startup.no-import-side-effects" category = "startup" -statement = "Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases." +statement = "Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases." rationale = "docs/engineering/skills/testing.md#focused-commands" scope = ["policyengine_api/**/*.py"] tier = "mechanical" @@ -671,7 +671,7 @@ id = "api.v2.runtime-privileges-tested-as-runtime-role" category = "migration" statement = "The cross-database integration tests run the v2 user services as policyengine_v2_runtime and the comparison-run services as policyengine_stage12_, holding only their declared privileges, so a statement those privileges don't allow fails CI." rationale = "docs/engineering/skills/alembic-migrations.md#table-privileges" -scope = ["tests/integration/conftest.py", "tests/integration/test_v2_user_identity_persistence.py", "tests/integration/test_v2_stage12_comparison_persistence.py"] +scope = ["tests/integration/conftest.py", "tests/integration/test_v2_user_identity_persistence.py", "tests/integration/test_v2_saved_policy_view.py", "tests/integration/test_v2_stage12_comparison_persistence.py", "tests/integration/test_v2_country_routes.py"] tier = "mechanical" status = "enforced" autofix = false @@ -918,9 +918,9 @@ invocation = "ruff check ." [[rule]] id = "api.quality.mypy-v2" category = "quality" -statement = "Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed." +statement = "Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed." rationale = "docs/engineering/skills/v2-code-organization.md" -scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py"] +scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", "policyengine_api/startup_country_check.py", "policyengine_api/supported_countries.py"] tier = "mechanical" status = "enforced" autofix = false @@ -934,6 +934,8 @@ paths = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" diff --git a/scripts/process_v1_household_mirror_event.py b/scripts/process_v1_household_mirror_event.py index f7af31072..49717019b 100755 --- a/scripts/process_v1_household_mirror_event.py +++ b/scripts/process_v1_household_mirror_event.py @@ -20,6 +20,7 @@ process_household_event_after_commit, ) from policyengine_api.services.household_service import HouseholdService +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -149,7 +150,7 @@ def _parser() -> argparse.ArgumentParser: required=True, choices=("staging", "production"), ) - parser.add_argument("--country-id", required=True, choices=("us", "uk")) + parser.add_argument("--country-id", required=True, choices=SUPPORTED_COUNTRY_IDS) parser.add_argument("--legacy-household-id", required=True, type=int) return parser diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..987c9dee8 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -355,6 +355,50 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="household_calculations_v2", + current_contract="typed_v2_resources", + future_owner_pr="Household calculations on v2", + requests=( + ContractRequest( + method="POST", + path="/v2/household-calculations", + expected_status=202, + stable_response_fields=( + "status", + "message", + "result.calculation_id", + "result.country_id", + "result.household_id", + "result.policy_id", + "result.year", + "result.catalog_version_id", + "result.status", + "result.created_at", + ), + route_group="household", + ), + ContractRequest( + method="GET", + path="/v2/household-calculations/{calculation_id}", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.calculation_id", + "result.status", + "result.completed_at", + "result.result.household", + "result.result.warnings", + "result.result.spm_config", + "result.result.spm_provenance", + "result.result.policyengine_version", + "result.error", + ), + route_group="household", + ), + ), + ), WorkflowContract( name="user_household_associations_v2", current_contract="typed_v2_resources", @@ -587,6 +631,24 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="country_selection_v2", + current_contract="typed_v2_resources", + future_owner_pr="Countries table and one country list in code", + requests=( + ContractRequest( + method="GET", + path="/v2/countries", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + ), + route_group="metadata", + ), + ), + ), WorkflowContract( name="metadata_resources_v2_preview", current_contract="typed_v2_resources", @@ -671,6 +733,56 @@ class WorkflowContract: ), route_group="metadata", ), + ContractRequest( + method="GET", + path="/v2/model-catalog?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.entities", + "result.basic_inputs", + "result.regions", + "result.datasets", + "result.economy_options", + "result.spm", + ), + route_group="metadata", + ), + ContractRequest( + method="GET", + path="/v2/model-catalog/variables?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.variables", + "result.part_parameter_values", + ), + route_group="metadata", + ), + ContractRequest( + method="GET", + path=( + "/v2/model-catalog/parameters?country_id=us" + "&name=gov.irs.income.bracket.rates.1" + ), + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.policyengine_version", + "result.catalog_format", + "result.parameters", + "result.node_labels", + "result.missing_names", + ), + route_group="metadata", + ), ), ), WorkflowContract( diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index ff861db14..a4580d9c7 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -14,12 +14,14 @@ def test_app_v2_workflow_contract_registry_is_complete(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", @@ -31,10 +33,12 @@ def test_app_v2_workflow_contract_registry_is_complete(): "typed_v2_resources" if workflow.name in { + "country_selection_v2", "metadata_resources_v2_preview", "policy_resources_v2", "user_policy_associations_v2", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", diff --git a/tests/fixtures/integration/live_household_calculations.py b/tests/fixtures/integration/live_household_calculations.py new file mode 100644 index 000000000..960891496 --- /dev/null +++ b/tests/fixtures/integration/live_household_calculations.py @@ -0,0 +1,314 @@ +"""Households and helpers for the live staging household calculation checks. + +They need only httpx and the standard library, because the parallel live test +job installs nothing else. +""" + +from __future__ import annotations + +import math +import os +import time +from dataclasses import dataclass, field +from functools import partial +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Callable + + import httpx + +CALCULATION_PATH = "/v2/household-calculations" +CALCULATION_TIMEOUT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_CALCULATION_TIMEOUT_SECONDS", "600") +) +POLL_INTERVAL_SECONDS = float( + os.environ.get("STAGING_API_TEST_POLL_INTERVAL_SECONDS", "5") +) +# A 502, 503 or 504 can be a short outage, such as the simulation service +# failing to read its database or version manifest for a moment; a +# calculation's submission and each poll are sent again while such answers +# last at most TRANSIENT_LIMIT_SECONDS in a row. Any other failure, and a +# longer run of such answers, fails the check with the answer's body. +TRANSIENT_STATUS_CODES = {502, 503, 504} +TRANSIENT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_TRANSIENT_LIMIT_SECONDS", "90") +) +FINISHED_STATUSES = {"succeeded", "failed"} +YEAR = "2026" +REFORM_START = f"{YEAR}-01-01" +OPEN_END = "2100-12-31" +COLLECTIONS = { + "us": { + "household": "households", + "family": "families", + "tax_unit": "tax_units", + "spm_unit": "spm_units", + "marital_unit": "marital_units", + }, + "uk": {"household": "households", "benunit": "benunits"}, +} + + +@dataclass(frozen=True) +class LiveHousehold: + """A household whose people all share one group of each kind. + + ``people`` and ``groups`` map variable names to values by period; a null + value asks for the variable to be calculated. ``axes`` vary inputs across + points, as in a ``/calculate`` situation. ``reform`` sets each named + parameter to a value from the start of ``YEAR`` with no end, through a + policy stored with ``/v2/policies``. ``add_missing_variables`` calculates + every variable the household lacks, as ``/calculate-full`` does. + ``expects_warnings`` says the household has a deprecated input, which the + calculation drops with a warning before the country model runs. + """ + + name: str + country_id: str + people: dict[str, dict[str, Any]] + groups: dict[str, dict[str, Any]] = field(default_factory=dict) + axes: list[list[dict[str, Any]]] | None = None + reform: dict[str, int | float] | None = None + add_missing_variables: bool = False + expects_warnings: bool = False + + def document(self) -> dict[str, Any]: + """Return the household as ``/v2/households`` stores it.""" + collections = COLLECTIONS[self.country_id] + document: dict[str, Any] = { + "people": [ + { + "id": person_id, + "values": values, + "memberships": { + collection: f"{collection}-1" for collection in collections + }, + } + for person_id, values in self.people.items() + ] + } + for collection in collections: + document[collection] = [ + {"id": f"{collection}-1", "values": self.groups.get(collection, {})} + ] + return document + + def situation(self) -> dict[str, Any]: + """Return the same household as a ``/calculate`` situation.""" + situation: dict[str, Any] = {"people": self.people} + for collection, plural in COLLECTIONS[self.country_id].items(): + situation[plural] = { + f"{collection}-1": { + "members": list(self.people), + **self.groups.get(collection, {}), + } + } + if self.axes is not None: + situation["axes"] = self.axes + return situation + + +@dataclass(frozen=True) +class FinishedCalculation: + """A calculation's final answer and how long each step took.""" + + item: dict[str, Any] + submit_seconds: float + total_seconds: float + + +def probe_amount(integration_probe_id: str) -> int: + """Return a small run-specific amount that makes each household new.""" + return int(integration_probe_id.rsplit("-", 1)[-1], 16) % 997 + + +def store_household(client: httpx.Client, household: LiveHousehold) -> str: + """Store the household through ``/v2/households`` and return its ID.""" + response = client.post( + "/v2/households", + params={"country_id": household.country_id}, + json={ + "country_id": household.country_id, + "default_year": int(YEAR), + "household_data": household.document(), + }, + ) + assert response.status_code in {200, 201}, response.text + return response.json()["result"]["item"]["id"] + + +def _one_catalog_item( + client: httpx.Client, resource: str, country_id: str, **filters: str +) -> dict[str, Any]: + response = client.get( + f"/v2/{resource}", params={"country_id": country_id, "limit": 1, **filters} + ) + assert response.status_code == 200, response.text[:500] + (item,) = response.json()["result"]["items"] + return item + + +def store_reform(client: httpx.Client, household: LiveHousehold) -> str | None: + """Store the household's reform through ``/v2/policies``; return its ID.""" + if household.reform is None: + return None + country_id = household.country_id + model = _one_catalog_item(client, "tax-benefit-models", country_id) + parameter_values = [ + { + "parameter_id": _one_catalog_item( + client, "parameters", country_id, name=parameter_name + )["id"], + "value": value, + "start_date": f"{REFORM_START}T00:00:00Z", + } + for parameter_name, value in household.reform.items() + ] + response = client.post( + "/v2/policies", + params={"country_id": country_id}, + json={ + "country_id": country_id, + "tax_benefit_model_id": model["id"], + "parameter_values": parameter_values, + }, + ) + assert response.status_code in {200, 201}, response.text[:500] + return response.json()["result"]["item"]["id"] + + +def _send_through_short_outages(send: Callable[[], httpx.Response]) -> httpx.Response: + """Send the request again while it gets 502, 503 or 504 for a short time.""" + outage_started: float | None = None + while True: + response = send() + if response.status_code not in TRANSIENT_STATUS_CODES: + return response + outage_started = outage_started or time.monotonic() + if time.monotonic() - outage_started >= TRANSIENT_LIMIT_SECONDS: + msg = ( + f"HTTP {response.status_code} for {TRANSIENT_LIMIT_SECONDS:.0f} s: " + f"{response.text[:500]}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) + + +def calculate( + client: httpx.Client, + household: LiveHousehold, + household_id: str, + policy_id: str | None = None, +) -> FinishedCalculation: + """Submit the stored household's calculation and poll until it finishes.""" + started = time.monotonic() + body: dict[str, Any] = { + "country_id": household.country_id, + "household_id": household_id, + "policy_id": policy_id, + "add_missing_variables": household.add_missing_variables, + } + if household.axes is not None: + body["axes"] = household.axes + deadline = started + CALCULATION_TIMEOUT_SECONDS + response = _send_through_short_outages( + lambda: client.post(CALCULATION_PATH, json=body) + ) + submit_seconds = time.monotonic() - started + assert response.status_code == 202, response.text + item = response.json()["result"] + while item["status"] not in FINISHED_STATUSES: + if time.monotonic() >= deadline: + msg = ( + f"Calculation {item['calculation_id']} did not finish within " + f"{CALCULATION_TIMEOUT_SECONDS:.0f} s; last status {item['status']}" + ) + raise AssertionError(msg) + time.sleep(POLL_INTERVAL_SECONDS) + polled = _send_through_short_outages( + partial(client.get, f"{CALCULATION_PATH}/{item['calculation_id']}") + ) + assert polled.status_code == 200, polled.text + item = polled.json()["result"] + return FinishedCalculation( + item=item, + submit_seconds=submit_seconds, + total_seconds=time.monotonic() - started, + ) + + +def calculate_with_v1(client: httpx.Client, household: LiveHousehold) -> dict: + """Return API v1's answer for the same household and reform. + + ``/calculate-full`` answers when the household adds missing variables, + ``/calculate`` otherwise. + """ + route = "calculate-full" if household.add_missing_variables else "calculate" + request: dict[str, Any] = {"household": household.situation()} + if household.reform is not None: + request["policy"] = { + parameter_name: {f"{REFORM_START}.{OPEN_END}": value} + for parameter_name, value in household.reform.items() + } + response = client.post(f"/{household.country_id}/{route}", json=request) + assert response.status_code == 200, response.text + payload = response.json() + assert payload["status"] == "ok", payload + return payload + + +def _same_value(v1_value: object, v2_value: object) -> bool: + if isinstance(v1_value, list) and isinstance(v2_value, list): + return len(v1_value) == len(v2_value) and all( + _same_value(v1_item, v2_item) + for v1_item, v2_item in zip(v1_value, v2_value, strict=True) + ) + # API v1 writes an undefined number as a bare NaN; v2 writes "NaN". + if isinstance(v1_value, float) and math.isnan(v1_value): + return v2_value == "NaN" + if isinstance(v1_value, bool) or isinstance(v2_value, bool): + return v1_value == v2_value + if isinstance(v1_value, int | float) and isinstance(v2_value, int | float): + return math.isclose(v1_value, v2_value, rel_tol=1e-9, abs_tol=1e-6) + return v1_value == v2_value + + +def household_differences( + v1_household: dict[str, Any], v2_household: dict[str, Any] +) -> list[str]: + """Return every value that differs between the two calculated households. + + Each entity, variable and period in either household is compared, so a + value only one of them has is a difference too. + """ + differences: list[str] = [] + for plural in sorted(set(v1_household) | set(v2_household)): + v1_entities = v1_household.get(plural) + v2_entities = v2_household.get(plural) + if not isinstance(v1_entities, dict) or not isinstance(v2_entities, dict): + if v1_entities != v2_entities: + differences.append(f"{plural}: {v1_entities!r} != {v2_entities!r}") + continue + for entity_id in sorted(set(v1_entities) | set(v2_entities)): + v1_variables = v1_entities.get(entity_id) or {} + v2_variables = v2_entities.get(entity_id) or {} + for variable in sorted(set(v1_variables) | set(v2_variables)): + v1_value = v1_variables.get(variable) + v2_value = v2_variables.get(variable) + if variable == "members" or not ( + isinstance(v1_value, dict) and isinstance(v2_value, dict) + ): + if not _same_value(v1_value, v2_value): + differences.append( + f"{plural}.{entity_id}.{variable}: " + f"{v1_value!r} != {v2_value!r}" + ) + continue + differences.extend( + f"{plural}.{entity_id}.{variable}.{period}: " + f"{v1_value.get(period)!r} != {v2_value.get(period)!r}" + for period in sorted(set(v1_value) | set(v2_value)) + if not _same_value(v1_value.get(period), v2_value.get(period)) + ) + return differences diff --git a/tests/fixtures/v2_catalog.py b/tests/fixtures/v2_catalog.py index 77f5cbd20..05bbf00e1 100644 --- a/tests/fixtures/v2_catalog.py +++ b/tests/fixtures/v2_catalog.py @@ -2,12 +2,12 @@ from __future__ import annotations -from dataclasses import dataclass from datetime import UTC, datetime from types import SimpleNamespace from typing import TYPE_CHECKING from policyengine_api.data.v2.catalog.extraction import extract_catalog +from policyengine_api.data.v2.catalog.region_references import RegionReference if TYPE_CHECKING: from policyengine_api.data.v2.catalog.records import NormalizedCatalog @@ -20,49 +20,164 @@ } -class RegionRegistry(list): - """Small iterable with the public registry's country identifier.""" +def region_references() -> dict[str, tuple[RegionReference, ...]]: + """Return small US and UK reference files. - def __init__(self, country_id: str, regions: list[SimpleNamespace]): - super().__init__(regions) - self.country_id = country_id + The US place has no filter rule, so publication leaves it out. + """ + return { + "us": ( + RegionReference( + code="us", + label="United States", + region_type="national", + parent_code=None, + state_code=None, + state_name=None, + filter_value=None, + v1_code="us", + ), + RegionReference( + code="state/ca", + label="California", + region_type="state", + parent_code="us", + state_code="CA", + state_name="California", + filter_value="6", + v1_code="state/ca", + ), + RegionReference( + code="congressional_district/CA-01", + label="California's 1st congressional district", + region_type="congressional_district", + parent_code="state/ca", + state_code="CA", + state_name="California", + filter_value="601", + v1_code="congressional_district/CA-01", + ), + RegionReference( + code="place/CA-44000", + label="Los Angeles", + region_type="place", + parent_code="state/ca", + state_code="CA", + state_name="California", + filter_value=None, + v1_code="place/CA-44000", + ), + ), + "uk": ( + RegionReference( + code="uk", + label="United Kingdom", + region_type="national", + parent_code=None, + state_code=None, + state_name=None, + filter_value=None, + v1_code="uk", + ), + RegionReference( + code="country/england", + label="England", + region_type="country", + parent_code="uk", + state_code=None, + state_name=None, + filter_value='["NORTH_EAST","LONDON"]', + v1_code="country/england", + ), + RegionReference( + code="constituency/E14001063", + label="Aldershot", + region_type="constituency", + parent_code="country/england", + state_code=None, + state_name=None, + filter_value="E14001063", + v1_code="constituency/Aldershot", + ), + RegionReference( + code="country/northern_ireland", + label="Northern Ireland", + region_type="country", + parent_code="uk", + state_code=None, + state_name=None, + filter_value="NORTHERN_IRELAND", + v1_code="country/ni", + ), + RegionReference( + code="local_authority/N09000011", + label="Ards and North Down", + region_type="local_authority", + parent_code="country/northern_ireland", + state_code=None, + state_name=None, + filter_value="N09000011", + v1_code="local_authority/Ards and North Down", + ), + ), + } -def _strategy(field: str, value: str | int) -> SimpleNamespace: +def _certified_artifact(path: str, *, revision: str = "fixture") -> SimpleNamespace: return SimpleNamespace( - strategy_type="row_filter", - variable_name=field, - variable_value=value, - additional_filters={}, + path=path, + revision=revision, + sha256="0" * 64, + repo_id=None, + repo_type=None, ) -@dataclass(frozen=True, kw_only=True) -class _Region: - """One public region; ``source`` gives the mutable object the registry holds.""" - - code: str - label: str - region_type: str - parent_code: str | None = None - dataset_path: str | None = None - strategy: SimpleNamespace | None = None - state_code: str | None = None - state_name: str | None = None - - def source(self) -> SimpleNamespace: - """Return the region as a registry entry; it needs a filter if scoped.""" - return SimpleNamespace( - code=self.code, - label=self.label, - region_type=self.region_type, - parent_code=self.parent_code, - dataset_path=self.dataset_path, - scoping_strategy=self.strategy, - requires_filter=self.strategy is not None, - state_code=self.state_code, - state_name=self.state_name, +def _release_contents(country_id: str) -> tuple[dict, dict, SimpleNamespace]: + """Return a country's certified datasets, region datasets and data package.""" + if country_id == "us": + datasets = { + "populace_us_2024": _certified_artifact("populace_us_2024.h5"), + "populace_us_2024_acs_local": _certified_artifact( + "populace_us_2024_acs_local.h5" + ), + } + region_datasets = { + "national": SimpleNamespace(path_template="populace_us_2024.h5"), + "state": SimpleNamespace(path_template="populace_us_2024_acs_local.h5"), + "congressional_district": SimpleNamespace( + path_template="populace_us_2024_acs_local.h5" + ), + } + data_package = SimpleNamespace( + repo_id="policyengine/populace-us", + repo_type="dataset", + version="0.1.0", + release_manifest_revision="fixture", ) + return datasets, region_datasets, data_package + datasets = { + "microcosm_uk_2024_25": _certified_artifact("microcosm_uk_2024_25.h5"), + "microcosm_uk_2024_25_dense": _certified_artifact( + "microcosm_uk_2024_25_dense.h5" + ), + } + # As in PolicyEngine.py 6.10.0, constituencies and local authorities are + # routed to the local-area dataset, which carries their codes. + region_datasets = { + "national": SimpleNamespace(path_template="microcosm_uk_2024_25.h5"), + "constituency": SimpleNamespace(path_template="microcosm_uk_2024_25_dense.h5"), + "local_authority": SimpleNamespace( + path_template="microcosm_uk_2024_25_dense.h5" + ), + } + data_package = SimpleNamespace( + repo_id="example-org/example-uk-data", + repo_type="model", + version="1.0.0", + release_manifest_revision="fixture", + ) + return datasets, region_datasets, data_package def bundle(*, policyengine_version: str = POLICYENGINE_VERSION) -> dict: @@ -88,7 +203,6 @@ def _model_source( country_id: str, country_package_version: str, policyengine_version: str, - regions: list[SimpleNamespace], ) -> SimpleNamespace: model_name = f"policyengine-{country_id}" default_dataset = { @@ -107,6 +221,14 @@ def _model_source( default_value=0.0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name="household.income.employment_income", + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter_node = SimpleNamespace( name="gov.example", @@ -119,6 +241,8 @@ def _model_source( description="An example rate", data_type=float, unit="/1", + economy=True, + household=False, parameter_values=[ SimpleNamespace( value=0.1, @@ -141,13 +265,21 @@ def _model_source( name=model_name, version=country_package_version, ) + datasets, region_datasets, data_package = _release_contents(country_id) version_holder.release_manifest = SimpleNamespace( country_id=country_id, policyengine_version=policyengine_version, default_dataset=default_dataset, default_dataset_uri=default_uri, + datasets=datasets, + region_datasets=region_datasets, + data_package=data_package, ) - version_holder.region_registry = RegionRegistry(country_id, regions) + version_holder.entities = [ + SimpleNamespace(key="person", plural="people", is_person=True), + SimpleNamespace(key="household", plural="households", is_person=False), + ] + version_holder.basic_inputs = ["employment_income"] version_holder.variables = [variable] version_holder.parameter_nodes = [parameter_node] version_holder.parameters = [parameter] @@ -162,60 +294,13 @@ def source_models( policyengine_version: str = POLICYENGINE_VERSION, ) -> dict[str, SimpleNamespace]: """Return US and UK public-model fixtures.""" - us_default_uri = "hf://policyengine/us/populace_us_2024.h5@fixture" - uk_default_uri = "hf://policyengine/uk/microcosm_uk_2024_25.h5@fixture" return { - "us": _model_source( - country_id="us", - country_package_version=DEPENDENCY_VERSIONS["policyengine-us"], - policyengine_version=policyengine_version, - regions=[ - _Region( - code="us", - label="United States", - region_type="national", - dataset_path=us_default_uri, - ).source(), - _Region( - code="state/ca", - label="California", - region_type="state", - parent_code="us", - dataset_path="hf://policyengine/us/populace_us_ca_2024.h5@fixture", - strategy=_strategy("state_fips", 6), - state_code="CA", - state_name="California", - ).source(), - _Region( - code="place/CA-44000", - label="Los Angeles", - region_type="place", - parent_code="state/ca", - state_code="CA", - state_name="California", - ).source(), - ], - ), - "uk": _model_source( - country_id="uk", - country_package_version=DEPENDENCY_VERSIONS["policyengine-uk"], + country_id: _model_source( + country_id=country_id, + country_package_version=DEPENDENCY_VERSIONS[f"policyengine-{country_id}"], policyengine_version=policyengine_version, - regions=[ - _Region( - code="uk", - label="United Kingdom", - region_type="national", - dataset_path=uk_default_uri, - ).source(), - _Region( - code="country/england", - label="England", - region_type="country", - parent_code="uk", - strategy=_strategy("country", "ENGLAND"), - ).source(), - ], - ), + ) + for country_id in ("us", "uk") } @@ -234,4 +319,80 @@ def normalized_catalog( policyengine_version=policyengine_version, models=source_models(policyengine_version=policyengine_version), installed_version=installed_version, + region_references=region_references(), + ) + + +def add_format_2_variable_records(model: SimpleNamespace) -> None: + """Add two variables that between them fill every format 2 variable field. + + ``filing_status`` is an enum whose labels are a string and an integer, with + no unit and a yearly period. ``countable_income`` names the parameters that + hold the lists it adds and subtracts, with a currency unit and a monthly + period. + """ + for name in ("gov.example.countable_income", "gov.example.deductions"): + parameter = SimpleNamespace( + name=name, + label=None, + description=None, + data_type=list, + unit=None, + economy=True, + household=True, + parameter_values=[ + SimpleNamespace( + value=["employment_income"], + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ], + ) + model.parameters.append(parameter) + model.parameters_by_name[name] = parameter + variables = ( + SimpleNamespace( + name="filing_status", + label="Filing status", + entity="person", + description=None, + data_type="Enum", + possible_values=["SINGLE", "JOINT"], + default_value="SINGLE", + adds=None, + subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name="household.demographic.filing_status", + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=[ + SimpleNamespace(value="SINGLE", label="Single"), + SimpleNamespace(value="JOINT", label=2), + ], + unit=None, + definition_period="year", + ), + SimpleNamespace( + name="countable_income", + label="Countable income", + entity="person", + description=None, + data_type=float, + possible_values=None, + default_value=0.0, + adds=["employment_income"], + subtracts=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.countable_income", + adds_parameter_path="gov.example.countable_income", + subtracts_parameter_path="gov.example.deductions", + enum_values=None, + unit="currency-USD", + definition_period="month", + ), ) + for variable in variables: + model.variables.append(variable) + model.variables_by_name[variable.name] = variable diff --git a/tests/fixtures/v2_countries.py b/tests/fixtures/v2_countries.py new file mode 100644 index 000000000..36d4b6604 --- /dev/null +++ b/tests/fixtures/v2_countries.py @@ -0,0 +1,28 @@ +"""Countries rows for SQLite tests that enforce v2 foreign keys. + +``V2_METADATA.create_all`` creates an empty countries table, while the v2 +Alembic revision that adds the table inserts the US and UK rows. Tests that +turn on SQLite foreign-key enforcement add the same rows before writing +records that name a country. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.models import Country + +if TYPE_CHECKING: + from sqlmodel import Session + +# The rows the countries revision inserts, ordered by country ID. +LISTED_COUNTRY_ROWS = ( + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, +) + + +def add_listed_countries(session: Session) -> None: + """Insert and commit the US and UK countries rows.""" + session.add_all(Country(**row) for row in LISTED_COUNTRY_ROWS) + session.commit() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b90efebd2..eb855ebb2 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -105,8 +105,8 @@ def managed_v2_roles(disposable_v2_database_url: str) -> ManagedV2Roles: engine.dispose() if not plan.can_apply: pytest.fail( - "the disposable v2 database lacks declared roles or tables: " - f"{plan.missing_roles} {plan.missing_tables}" + "the disposable v2 database lacks declared roles, tables or views: " + f"{plan.missing_roles} {plan.missing_tables} {plan.missing_views}" ) return ManagedV2Roles( deployment_environment=deployment_environment, diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 814971236..de14ee519 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -3,6 +3,7 @@ import itertools import os from collections.abc import Iterator +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 @@ -21,6 +22,7 @@ V2_ALEMBIC_HEAD_REVISION, V2MigrationTargetError, load_v2_alembic_settings, + validate_v2_head_schema, ) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL @@ -30,6 +32,8 @@ STAGE_12_PREVIOUS_REVISION = "724b1b11a33e" STAGE_12_COMPARISON_PREVIOUS_REVISION = "439303be14fe" STAGE_12A_PREVIOUS_REVISION = "91af4fc8d8c1" +SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" +COUNTRIES_PREVIOUS_REVISION = "f5c1d9b4685b" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -147,7 +151,7 @@ def _assert_stage_12a_association_shape(engine) -> None: "subject", ] for table_name, reference in ( - ("user_policies", "policy"), + ("user_policy_associations", "policy"), ("user_household_associations", "household"), ("user_simulation_associations", "simulation"), ("user_report_associations", "report"), @@ -171,7 +175,7 @@ def _assert_stage_12a_association_shape(engine) -> None: report_policy_key = next( foreign_key for foreign_key in inspect(engine).get_foreign_keys("user_report_associations") - if foreign_key["referred_table"] == "user_policies" + if foreign_key["referred_table"] == "user_policy_associations" ) assert report_policy_key["options"]["ondelete"] == "SET NULL" for table_name in ("user_simulation_associations", "user_report_associations"): @@ -187,7 +191,6 @@ def _assert_stage_12a_association_shape(engine) -> None: for constraint in inspect(engine).get_check_constraints("users") } assert { - "ck_users_primary_country", "ck_users_user_type", "ck_users_standard_primary_country", } <= user_checks @@ -352,6 +355,75 @@ def _assert_base(engine: Engine) -> None: ) +# Every column that names a record's country, with the check it replaced. +COUNTRY_REFERENCE_COLUMNS = { + ("users", "primary_country"): "ck_users_primary_country", + ("households", "country_id"): "ck_households_country", + ("policies", "country_id"): "ck_policies_country", + ("user_household_associations", "country_id"): ( + "ck_user_household_associations_country" + ), + ("user_policy_associations", "country_id"): ("ck_user_policy_associations_country"), + ("user_simulation_associations", "country_id"): ( + "ck_user_simulation_associations_country" + ), + ("user_report_associations", "country_id"): "ck_user_report_associations_country", +} +# The rows the countries revision inserts: the countries the replaced checks +# allowed, both household and society-wide capable. +LISTED_COUNTRIES = [("uk", True, True), ("us", True, True)] + + +def _country_foreign_keys(engine, table_name: str) -> list[dict]: + return [ + foreign_key + for foreign_key in inspect(engine).get_foreign_keys(table_name) + if foreign_key["referred_table"] == "countries" + ] + + +def _check_names(engine, table_name: str) -> set[str]: + return { + constraint["name"] + for constraint in inspect(engine).get_check_constraints(table_name) + } + + +def _listed_countries(engine) -> list[tuple]: + with engine.connect() as connection: + return [ + tuple(row) + for row in connection.execute( + text( + "SELECT id, household_enabled, society_wide_enabled " + "FROM countries ORDER BY id" + ) + ) + ] + + +def _assert_countries_shape(engine) -> None: + assert _listed_countries(engine) == LISTED_COUNTRIES + assert "ck_countries_society_wide_requires_household" in _check_names( + engine, "countries" + ) + for (table_name, column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + country_keys = _country_foreign_keys(engine, table_name) + assert len(country_keys) == 1, table_name + assert country_keys[0]["name"] == f"fk_{table_name}_{column_name}_countries" + assert country_keys[0]["constrained_columns"] == [column_name] + assert country_keys[0]["referred_columns"] == ["id"] + assert country_keys[0]["options"]["ondelete"] == "RESTRICT" + assert check_name not in _check_names(engine, table_name) + + +def _assert_pre_countries_shape(engine) -> None: + assert "countries" not in inspect(engine).get_table_names(schema="public") + for (table_name, _column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + assert _country_foreign_keys(engine, table_name) == [] + assert check_name in _check_names(engine, table_name) + + def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: database_url = _disposable_url() config = _config() @@ -364,6 +436,13 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: } _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) + + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + _assert_revision(engine, COUNTRIES_PREVIOUS_REVISION) + _assert_pre_countries_shape(engine) + _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) _assert_revision(engine, STAGE_12A_PREVIOUS_REVISION) @@ -396,10 +475,10 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -# Dormant at the parent revision: its models map these tables, and deleting a -# user cascades through them, but no parent-revision code inserts into them. -# They are the only tables Stage 12A may give a new required column. -PARENT_REVISION_READ_ONLY_TABLES = frozenset( +# Dormant at the Stage 12A parent revision: its models map these tables, and +# deleting a user cascades through them, but no parent-revision code inserts +# into them. They are the only tables Stage 12A may give a new required column. +STAGE_12A_PARENT_READ_ONLY_TABLES = frozenset( {"user_simulation_associations", "user_report_associations"} ) USER_ASSOCIATION_TABLES = ( @@ -409,8 +488,18 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_report_associations", ) COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) -# Nullable references that a parent-revision check constraint requires. -REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) +# Nullable references that a parent-revision check constraint requires: a +# standard user's country, and one of each association's two references. +REFERENCES_REQUIRED_BY_CHECKS = frozenset( + { + ("simulations", "household_id"), + ("users", "primary_country"), + ("user_policies", "policy_id"), + ("user_household_associations", "household_id"), + ("user_simulation_associations", "simulation_id"), + ("user_report_associations", "report_id"), + } +) # Required values that a parent-revision check constraint restricts. CHECKED_VALUES = {("datasets", "year"): 2026} @@ -547,26 +636,53 @@ def _rewrite_mapped_columns(connection, table: sa.Table, row: dict) -> None: assert result.rowcount == 1, table.name +@dataclass(frozen=True) +class PreviousRevisionSchemas: + """A previous revision's reflected schema and head's, after the upgrade. + + ``read_only_tables`` are the previous revision's tables that only new code + writes, so their test rows are inserted through head's table. + """ + + parent: sa.MetaData + head: sa.MetaData + read_only_tables: frozenset[str] + + def insert_table(self, table_name: str) -> sa.Table | None: + """Head's table for a read-only table, otherwise ``None``.""" + if table_name in self.read_only_tables: + return self.head.tables[table_name] + return None + + +def _parent_table(parent: sa.MetaData, table_name: str) -> sa.Table: + """Return a previous revision's table, by its name at that revision. + + From the saved-policy rename on, ``user_policies`` is a view, which the + previous revision's reflected tables leave out, and the table is + ``user_policy_associations``. + """ + if table_name == "user_policies" and table_name not in parent.tables: + return parent.tables["user_policy_associations"] + return parent.tables[table_name] + + def _assert_user_deletion_cascades( connection, - parent: sa.MetaData, - head: sa.MetaData, + schemas: PreviousRevisionSchemas, stored_rows: dict[str, dict], sequence: Iterator[int], ) -> None: - """Delete, through the parent's users table, a user who owns Stage 12A rows.""" + """Delete, through the parent's users table, a user who owns association rows.""" + parent, head = schemas.parent, schemas.head users = parent.tables["users"] owner = _insert_and_read_back( connection, users, _row_values(users, stored_rows, sequence) ) owner_rows = {**stored_rows, "users": owner} for table_name in USER_ASSOCIATION_TABLES: - table = parent.tables[table_name] - head_table = ( - head.tables[table_name] - if table_name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + table = _parent_table(parent, table_name) + head_table = schemas.insert_table(table_name) _insert_and_read_back( connection, table, @@ -596,27 +712,33 @@ def _assert_user_deletion_cascades( assert remaining == 0, table_name -def test_stage_12a_schema_serves_the_previous_revision() -> None: - """Run the previous Public API revision's table usage against head. +def _assert_head_serves_previous_revision( + previous_revision: str, + read_only_tables: frozenset[str], +) -> None: + """Run a previous Public API revision's table usage against head. The parent revision's schema is reflected and then used, after upgrading, to insert a row into every table it maps (supplying only what it requires), read the row back with every mapped column, rewrite those - columns, delete it, and delete a user who owns Stage 12A rows so the + columns, delete it, and delete a user who owns association rows so the cascades run. Head may not give any table the parent revision writes a - new required column without a server default. + new required column without a server default; ``read_only_tables`` are the + parent's tables that only new code writes. Since the saved-policy + rename, the parent's ``user_policies`` is head's temporary view of that + name, so the parent's statements on it run through the view. """ database_url = _disposable_url() config = _config() engine = create_engine(database_url) try: - command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) + command.downgrade(config, previous_revision) parent = sa.MetaData() parent.reflect(engine) command.upgrade(config, "head") head = sa.MetaData() - head.reflect(engine) + head.reflect(engine, views=True) parent_tables = [ table for table in parent.sorted_tables if table.name != "alembic_version" ] @@ -634,21 +756,18 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: for table in parent_tables if _new_required_columns(table, head.tables[table.name]) } - assert set(tables_with_new_required_columns) <= ( - PARENT_REVISION_READ_ONLY_TABLES - ), tables_with_new_required_columns + assert set(tables_with_new_required_columns) <= read_only_tables, ( + tables_with_new_required_columns + ) + schemas = PreviousRevisionSchemas(parent, head, read_only_tables) sequence = itertools.count() with engine.connect() as connection: transaction = connection.begin() try: stored_rows: dict[str, dict] = {} for table in parent_tables: - head_table = ( - head.tables[table.name] - if table.name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + head_table = schemas.insert_table(table.name) stored_rows[table.name] = _insert_and_read_back( connection, table, @@ -660,7 +779,7 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: _rewrite_mapped_columns(connection, table, stored_rows[table.name]) _assert_user_deletion_cascades( - connection, parent, head, stored_rows, sequence + connection, schemas, stored_rows, sequence ) for table in reversed(parent_tables): @@ -677,6 +796,21 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: engine.dispose() +def test_stage_12a_schema_serves_the_previous_revision() -> None: + _assert_head_serves_previous_revision( + STAGE_12A_PREVIOUS_REVISION, + STAGE_12A_PARENT_READ_ONLY_TABLES, + ) + + +def test_countries_schema_serves_the_previous_revision() -> None: + """The release before the countries table keeps working during a deploy.""" + _assert_head_serves_previous_revision( + COUNTRIES_PREVIOUS_REVISION, + frozenset(), + ) + + def test_upgrade_to_head_validates_the_resulting_schema_against_metadata() -> None: database_url = _disposable_url() config = _config() @@ -957,3 +1091,762 @@ def reflected(name: str) -> sa.Table: connection.execute(sa.delete(table).where(table.c.id == row_id)) command.upgrade(config, "head") engine.dispose() + + +def _constraint_names(engine, table_name: str) -> set[str]: + with engine.connect() as connection: + return set( + connection.execute( + text( + "SELECT conname FROM pg_constraint " + "WHERE conrelid = to_regclass(:table_name)" + ), + {"table_name": f"public.{table_name}"}, + ).scalars() + ) + + +def _index_names(engine, table_name: str) -> set[str]: + with engine.connect() as connection: + return set( + connection.execute( + text( + "SELECT indexname FROM pg_indexes " + "WHERE schemaname = 'public' AND tablename = :table_name" + ), + {"table_name": table_name}, + ).scalars() + ) + + +def _referring_foreign_keys(engine, table_name: str) -> set[tuple[str, str]]: + with engine.connect() as connection: + return { + (row.on_table, row.conname) + for row in connection.execute( + text( + "SELECT conrelid::regclass::text AS on_table, conname " + "FROM pg_constraint WHERE contype = 'f' " + "AND confrelid = to_regclass(:table_name)" + ), + {"table_name": f"public.{table_name}"}, + ) + } + + +def _closure_of_referred_tables( + metadata: sa.MetaData, table_names: set[str] +) -> set[str]: + closure: set[str] = set() + pending = list(table_names) + while pending: + table_name = pending.pop() + if table_name in closure: + continue + closure.add(table_name) + pending.extend( + foreign_key.referred_table.name + for foreign_key in metadata.tables[table_name].foreign_key_constraints + ) + return closure + + +def _saved_policy_with_references(connection, parent: sa.MetaData) -> dict: + """Store a user, a policy, a saved policy, its v1 mapping and a report.""" + sequence = itertools.count() + stored_rows: dict[str, dict] = {} + owner_tables = _closure_of_referred_tables(parent, {"users", "policies"}) + for table in parent.sorted_tables: + if table.name in owner_tables: + values = _row_values(table, stored_rows, sequence) + if table.name == "users": + # A standard user needs a country (ck_users_standard_primary_country). + values["primary_country"] = "us" + stored_rows[table.name] = _insert_and_read_back(connection, table, values) + saved_policy_id = uuid4() + connection.execute( + parent.tables["user_policies"] + .insert() + .values( + id=saved_policy_id, + user_id=stored_rows["users"]["id"], + policy_id=stored_rows["policies"]["id"], + country_id=stored_rows["policies"]["country_id"], + name="Saved reform", + baseline="current-law", + ) + ) + connection.execute( + parent.tables["legacy_user_policy_mappings"] + .insert() + .values( + id=uuid4(), + country_id=stored_rows["policies"]["country_id"], + legacy_user_policy_id=41, + user_policy_id=saved_policy_id, + fingerprint_version=1, + fingerprint_sha256="a" * 64, + ) + ) + report_id = uuid4() + connection.execute( + parent.tables["user_report_associations"] + .insert() + .values( + id=report_id, + user_id=stored_rows["users"]["id"], + country_id=stored_rows["policies"]["country_id"], + legacy_report_id=42, + user_policy_id=saved_policy_id, + baseline_user_policy_id=saved_policy_id, + ) + ) + return { + "stored_rows": stored_rows, + "saved_policy_id": saved_policy_id, + "report_id": report_id, + } + + +def _saved_policy_row(connection, table_name: str, saved_policy_id: UUID) -> dict: + return dict( + connection.execute( + sa.select(sa.text("*")) + .select_from(sa.table(table_name)) + .where(sa.column("id") == saved_policy_id) + ) + .mappings() + .one() + ) + + +def _delete_saved_policy_rows(engine, parent: sa.MetaData, saved: dict) -> None: + """Remove the stored rows at head, children first, so later tests start clean.""" + with engine.begin() as connection: + connection.execute( + text("DELETE FROM user_report_associations WHERE id = :id"), + {"id": saved["report_id"]}, + ) + # Deleting the saved policy cascades to its v1 mapping. + connection.execute( + text("DELETE FROM user_policy_associations WHERE id = :id"), + {"id": saved["saved_policy_id"]}, + ) + for table in reversed(parent.sorted_tables): + stored_row = saved["stored_rows"].get(table.name) + if stored_row is not None: + connection.execute( + table.delete().where(_primary_key_clause(table, stored_row)) + ) + + +# The names head gives the renamed table's constraints, indexes and the +# foreign keys that refer to it. Names longer than PostgreSQL's 63 characters +# end in a hash, as SQLAlchemy shortens the model's convention names. +SAVED_POLICY_SCHEMA_NAMES_AT_HEAD = { + "constraints": { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + # The countries table replaced the US-and-UK check with a foreign key. + "fk_user_policy_associations_country_id_countries", + "ck_user_policy_associations_policy_ref", + "ck_user_policy_associations_baseline", + "fk_user_policy_associations_policy_country", + "fk_user_policy_associations_user_id_users", + }, + "indexes": { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + "ix_user_policy_associations_user_id", + "ix_user_policy_associations_policy_id", + "ix_user_policies_country_user_created_id", + "ix_user_policies_country_policy", + "ix_user_policies_country_legacy_policy", + }, + "referring_foreign_keys": { + ( + "legacy_user_policy_mappings", + "fk_legacy_user_policy_mappings_association_country", + ), + ( + "user_report_associations", + "fk_user_report_associations_user_policy_id_user_policy__0820", + ), + ( + "user_report_associations", + "fk_user_report_associations_baseline_user_policy_id_use_0a3d", + ), + }, +} +RENAME_PRIVILEGE_PROBE_ROLE = "rename_privilege_probe" + + +def _schema_names(engine, table_name: str) -> dict[str, set]: + return { + "constraints": _constraint_names(engine, table_name), + "indexes": _index_names(engine, table_name), + "referring_foreign_keys": _referring_foreign_keys(engine, table_name), + } + + +def _saved_policy_references(connection, saved: dict) -> tuple: + """Return the saved policy a v1 mapping and a report's two keys point at.""" + mapped = connection.execute( + text( + "SELECT user_policy_id FROM legacy_user_policy_mappings " + "WHERE legacy_user_policy_id = 41" + ) + ).scalar_one_or_none() + report_references = connection.execute( + text( + "SELECT user_policy_id, baseline_user_policy_id " + "FROM user_report_associations WHERE id = :id" + ), + {"id": saved["report_id"]}, + ).one() + return (mapped, *report_references) + + +def _references_after_deleting_saved_policy( + engine, table_name: str, saved: dict +) -> tuple: + """Delete the saved policy in a rolled-back transaction; return the references. + + The v1 mapping cascades away and the report's keys are set to null. + """ + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.execute( + sa.delete(sa.table(table_name, sa.column("id"))).where( + sa.column("id") == saved["saved_policy_id"] + ) + ) + return _saved_policy_references(connection, saved) + finally: + transaction.rollback() + + +def _probe_privileges(connection, table_name: str) -> dict[str, bool]: + """Return which of three privileges the probe role holds, one at a time. + + ``has_table_privilege`` with a list of privileges is true when any one is + held, so each is checked separately. + """ + return { + privilege: connection.execute( + text("SELECT has_table_privilege(:role, :table, :privilege)"), + { + "role": RENAME_PRIVILEGE_PROBE_ROLE, + "table": f"public.{table_name}", + "privilege": privilege, + }, + ).scalar_one() + for privilege in ("SELECT", "UPDATE", "INSERT") + } + + +GRANTED_TO_PROBE = {"SELECT": True, "UPDATE": True, "INSERT": False} +CLEARED_REFERENCES = (None, None, None) + + +def _view_names(engine) -> set[str]: + return set(inspect(engine).get_view_names(schema="public")) + + +def _column_names(engine, relation_name: str) -> list[str]: + return [ + column["name"] + for column in inspect(engine).get_columns(relation_name, schema="public") + ] + + +def _assert_view_under_old_name_at_head(engine, config) -> None: + """Head keeps a view named ``user_policies`` with the table's columns. + + Neither the head schema validation nor ``alembic check`` reports it. + """ + assert _view_names(engine) == {"user_policies"} + assert _column_names(engine, "user_policies") == _column_names( + engine, "user_policy_associations" + ) + with engine.connect() as connection: + validate_v2_head_schema(connection, V2_METADATA) + command.check(config) + + +def _drop_rename_privilege_probe(engine) -> None: + with engine.begin() as connection: + connection.execute( + text( + "DO $$ BEGIN IF EXISTS (SELECT FROM pg_roles WHERE rolname = " + f"'{RENAME_PRIVILEGE_PROBE_ROLE}') THEN " + f"DROP OWNED BY {RENAME_PRIVILEGE_PROBE_ROLE}; " + f"DROP ROLE {RENAME_PRIVILEGE_PROBE_ROLE}; END IF; END $$" + ) + ) + + +def test_saved_policy_rename_keeps_rows_references_and_privileges() -> None: + """Upgrading renames the table in place, and downgrading renames it back. + + Saved policies, their v1 mappings and the reports that name them survive + both directions, the references and a table grant follow the table, and + the constraints and indexes named after the table take its name. Head has + a view named ``user_policies`` with the table's columns, which neither the + schema validation nor ``alembic check`` reports. The downgrade drops the + view and restores every name the parent revision had. + """ + config = _config() + engine = create_engine(_disposable_url()) + parent = sa.MetaData() + saved = None + try: + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + parent.reflect(engine) + names_at_parent = _schema_names(engine, "user_policies") + _drop_rename_privilege_probe(engine) + with engine.begin() as connection: + saved = _saved_policy_with_references(connection, parent) + connection.execute(text(f"CREATE ROLE {RENAME_PRIVILEGE_PROBE_ROLE}")) + connection.execute( + text( + "GRANT SELECT, UPDATE ON user_policies TO " + f"{RENAME_PRIVILEGE_PROBE_ROLE}" + ) + ) + before = _saved_policy_row( + connection, "user_policies", saved["saved_policy_id"] + ) + saved_policy_id = saved["saved_policy_id"] + + command.upgrade(config, "head") + + table_names = set(inspect(engine).get_table_names(schema="public")) + assert "user_policies" not in table_names + _assert_view_under_old_name_at_head(engine, config) + with engine.connect() as connection: + after = _saved_policy_row( + connection, "user_policy_associations", saved_policy_id + ) + references = _saved_policy_references(connection, saved) + privileges = _probe_privileges(connection, "user_policy_associations") + assert after == before + assert references == (saved_policy_id, saved_policy_id, saved_policy_id) + assert privileges == GRANTED_TO_PROBE + assert ( + _schema_names(engine, "user_policy_associations") + == SAVED_POLICY_SCHEMA_NAMES_AT_HEAD + ) + assert ( + _references_after_deleting_saved_policy( + engine, "user_policy_associations", saved + ) + == CLEARED_REFERENCES + ) + + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + + table_names = set(inspect(engine).get_table_names(schema="public")) + assert "user_policy_associations" not in table_names + assert "user_policies" in table_names + assert _view_names(engine) == set() + with engine.connect() as connection: + restored = _saved_policy_row(connection, "user_policies", saved_policy_id) + restored_references = _saved_policy_references(connection, saved) + restored_privileges = _probe_privileges(connection, "user_policies") + assert restored == before + assert restored_references == references + assert restored_privileges == GRANTED_TO_PROBE + assert _schema_names(engine, "user_policies") == names_at_parent + assert ( + _references_after_deleting_saved_policy(engine, "user_policies", saved) + == CLEARED_REFERENCES + ) + finally: + command.upgrade(config, "head") + if saved is not None: + _delete_saved_policy_rows(engine, parent, saved) + _drop_rename_privilege_probe(engine) + engine.dispose() + + +def test_previous_release_still_finds_user_policies() -> None: + """The release running during the deploy uses saved policies by the old name. + + A saved policy stored before the rename is found under ``user_policies`` + at head, and a row written, changed and deleted through the view is + written, changed and deleted in ``user_policy_associations``. The runtime + role's own use of the view is in ``test_v2_saved_policy_view``. + """ + config = _config() + engine = create_engine(_disposable_url()) + parent = sa.MetaData() + saved = None + try: + command.downgrade(config, SAVED_POLICY_RENAME_PARENT_REVISION) + parent.reflect(engine) + with engine.begin() as connection: + saved = _saved_policy_with_references(connection, parent) + command.upgrade(config, "head") + saved_policy_id = saved["saved_policy_id"] + with engine.connect() as connection: + stored = _saved_policy_row( + connection, "user_policy_associations", saved_policy_id + ) + found = _saved_policy_row(connection, "user_policies", saved_policy_id) + assert found == stored + + written_id = uuid4() + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.execute( + text( + "INSERT INTO user_policies " + "(id, user_id, policy_id, country_id, name) " + "VALUES (:id, :user_id, :policy_id, :country_id, 'Written')" + ), + { + "id": written_id, + "user_id": stored["user_id"], + "policy_id": stored["policy_id"], + "country_id": stored["country_id"], + }, + ) + written = _saved_policy_row( + connection, "user_policy_associations", written_id + ) + connection.execute( + text("UPDATE user_policies SET name = 'Changed' WHERE id = :id"), + {"id": written_id}, + ) + changed_name = _saved_policy_row( + connection, "user_policy_associations", written_id + )["name"] + connection.execute( + text("DELETE FROM user_policies WHERE id = :id"), + {"id": written_id}, + ) + remaining = connection.execute( + text( + "SELECT count(*) FROM user_policy_associations WHERE id = :id" + ), + {"id": written_id}, + ).scalar_one() + finally: + transaction.rollback() + assert written["name"] == "Written" + assert written["created_at"] is not None + assert changed_name == "Changed" + assert remaining == 0 + finally: + command.upgrade(config, "head") + if saved is not None: + _delete_saved_policy_rows(engine, parent, saved) + engine.dispose() + + +CATALOG_FORMAT_2_PARENT_REVISION = "fa7fa1035444" +CATALOG_FORMAT_2_COLUMNS = { + "tax_benefit_model_versions": ("catalog_format", "entities", "basic_inputs"), + "variables": ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + ), + "parameters": ("economy", "household"), +} + + +def _catalog_version_published_before_format_2(connection: sa.Connection) -> UUID: + """Store a model version with one variable and one parameter; return its id.""" + model_id = uuid4() + version_id = uuid4() + connection.execute( + text( + "INSERT INTO tax_benefit_models (id, name) " + "VALUES (:model_id, 'catalog-format-probe')" + ), + {"model_id": model_id}, + ) + connection.execute( + text( + "INSERT INTO tax_benefit_model_versions " + "(id, model_id, version, current_law_id, metadata_time_periods) " + "VALUES (:version_id, :model_id, '0.0.1-probe', 1, '[2026]')" + ), + {"version_id": version_id, "model_id": model_id}, + ) + connection.execute( + text( + "INSERT INTO variables " + "(id, name, entity, default_value, tax_benefit_model_version_id) " + "VALUES (:variable_id, 'probe_income', 'person', '0', :version_id)" + ), + {"variable_id": uuid4(), "version_id": version_id}, + ) + connection.execute( + text( + "INSERT INTO parameters (id, name, tax_benefit_model_version_id) " + "VALUES (:parameter_id, 'gov.probe.rate', :version_id)" + ), + {"parameter_id": uuid4(), "version_id": version_id}, + ) + return version_id + + +def _format_2_values(connection: sa.Connection, version_id: UUID) -> list[tuple]: + """Return the format 2 columns of the version's model, variable and parameter.""" + rows = [] + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + key = ( + "id" + if table_name == "tax_benefit_model_versions" + else "tax_benefit_model_version_id" + ) + rows.append( + tuple( + connection.execute( + text( + f"SELECT {', '.join(columns)} FROM {table_name} " + f"WHERE {key} = :version_id" + ), + {"version_id": version_id}, + ).one() + ) + ) + return rows + + +def test_catalog_format_2_columns_leave_earlier_versions_empty() -> None: + """A version stored before catalog format 2 keeps its format 2 fields null. + + Upgrading adds the columns as nullable without writing to stored rows, and + downgrading drops them and keeps every row. + """ + config = _config() + engine = create_engine(_disposable_url()) + version_id = None + try: + command.downgrade(config, CATALOG_FORMAT_2_PARENT_REVISION) + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + assert set(columns).isdisjoint(_columns(engine, table_name)) + with engine.begin() as connection: + version_id = _catalog_version_published_before_format_2(connection) + + command.upgrade(config, "head") + + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + stored_columns = _columns(engine, table_name) + for column_name in columns: + assert stored_columns[column_name]["nullable"] + assert stored_columns[column_name]["default"] is None + with engine.connect() as connection: + assert _format_2_values(connection, version_id) == [ + (None, None, None), + (None, None, None, None, None, None, None, None), + (None, None), + ] + + command.downgrade(config, CATALOG_FORMAT_2_PARENT_REVISION) + + for table_name, columns in CATALOG_FORMAT_2_COLUMNS.items(): + assert set(columns).isdisjoint(_columns(engine, table_name)) + with engine.connect() as connection: + counts = connection.execute( + text( + "SELECT (SELECT count(*) FROM variables " + "WHERE tax_benefit_model_version_id = :version_id), " + "(SELECT count(*) FROM parameters " + "WHERE tax_benefit_model_version_id = :version_id)" + ), + {"version_id": version_id}, + ).one() + assert tuple(counts) == (1, 1) + finally: + with engine.begin() as connection: + connection.execute( + text( + "DELETE FROM tax_benefit_models WHERE name = 'catalog-format-probe'" + ) + ) + command.upgrade(config, "head") + + +def test_countries_revision_keeps_existing_rows_and_rejects_unlisted_countries() -> ( + None +): + """Rows saved before the revision survive it, and only listed countries pass.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + def reflected(name: str) -> sa.Table: + return sa.Table(name, sa.MetaData(), autoload_with=engine) + + try: + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + with engine.begin() as connection: + connection.execute( + sa.insert(reflected("users")).values(id=user_id, primary_country="uk") + ) + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="uk", + legacy_report_id=7, + ) + ) + + command.upgrade(config, "head") + command.check(config) + _assert_countries_shape(engine) + + with engine.connect() as connection: + stored_countries = connection.execute( + text( + "SELECT users.primary_country, " + "user_report_associations.country_id FROM users " + "JOIN user_report_associations " + "ON user_report_associations.user_id = users.id " + "WHERE users.id = :user_id" + ), + {"user_id": user_id}, + ).one() + assert tuple(stored_countries) == ("uk", "uk") + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="ca", + legacy_report_id=8, + ) + ) + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute(text("DELETE FROM countries WHERE id = 'uk'")) + finally: + with engine.begin() as connection: + users = reflected("users") + connection.execute(sa.delete(users).where(users.c.id == user_id)) + command.upgrade(config, "head") + engine.dispose() + + +def test_countries_downgrade_refuses_records_of_an_unlisted_country() -> None: + """The restored US-and-UK checks stop a downgrade that would orphan records.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text("INSERT INTO users (id, primary_country) VALUES (:user_id, 'be')"), + {"user_id": user_id}, + ) + + with pytest.raises(sa.exc.IntegrityError): + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == V2_ALEMBIC_HEAD_REVISION + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = :user_id"), {"user_id": user_id} + ) + connection.execute(text("DELETE FROM countries WHERE id = 'be'")) + command.upgrade(config, "head") + engine.dispose() + + +def test_a_household_only_country_row_lets_records_name_that_country() -> None: + """Once a household-only country has a row, its records can be stored.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + household_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.connect() as connection: + transaction = connection.begin() + try: + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + connection.execute( + text( + "INSERT INTO households (id, country_id, household_data, " + "canonicalization_version, content_hash) VALUES " + "(:household_id, 'be', '{}'::jsonb, 1, :content_hash)" + ), + {"household_id": household_id, "content_hash": "c" * 64}, + ) + connection.execute( + text( + "INSERT INTO user_household_associations " + "(id, user_id, household_id, country_id) VALUES " + "(:association_id, :user_id, :household_id, 'be')" + ), + { + "association_id": uuid4(), + "user_id": user_id, + "household_id": household_id, + }, + ) + + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('xx', false, true)" + ) + ) + finally: + transaction.rollback() + finally: + engine.dispose() diff --git a/tests/integration/test_live_v2_household_calculations.py b/tests/integration/test_live_v2_household_calculations.py new file mode 100644 index 000000000..c5a62fdb6 --- /dev/null +++ b/tests/integration/test_live_v2_household_calculations.py @@ -0,0 +1,167 @@ +"""Live staging checks of ``/v2/household-calculations``. + +These run against the tagged staging candidate, through staging API v1, the +staging Simulation Entrypoint, the v2 household functions on Modal and the +staging v2 database: + +- a submit, poll and result cycle for a new US household; +- a latency check: several calculations submitted at once, so the Simulation + Entrypoint serves them in parallel, each accepted within + ``STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS`` and finished within + ``STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS``; +- a memory check: a household varied across 401 points of employment income, + with its net income calculated at each point, which must finish without + failing; a household function that runs out of memory fails it. + +The measured times are reported as a warning, so the job log keeps them. +""" + +from __future__ import annotations + +import os +import statistics +import warnings +from concurrent.futures import ThreadPoolExecutor + +import httpx + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + FinishedCalculation, + LiveHousehold, + calculate, + probe_amount, + store_household, +) + +PARALLEL_CALCULATIONS = int(os.environ.get("STAGING_HOUSEHOLD_PARALLEL_COUNT", "6")) +SUBMIT_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_SUBMIT_LIMIT_SECONDS", "10") +) +FINISH_LIMIT_SECONDS = float( + os.environ.get("STAGING_HOUSEHOLD_FINISH_LIMIT_SECONDS", "300") +) +MEMORY_CHECK_POINTS = 401 + + +class HouseholdCalculationMeasurement(UserWarning): + """Times measured by the live household calculation checks.""" + + +def _us_household(name: str, employment_income: int) -> LiveHousehold: + return LiveHousehold( + name=name, + country_id="us", + people={ + "adult": { + "age": {YEAR: 40}, + "employment_income": {YEAR: employment_income}, + } + }, + groups={ + "household": { + "state_name": {YEAR: "NY"}, + "household_net_income": {YEAR: None}, + } + }, + ) + + +def test_a_us_household_is_submitted_polled_and_read(api_client, integration_probe_id): + household = _us_household("cycle", 51_000 + probe_amount(integration_probe_id)) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + item = finished.item + assert item["status"] == "succeeded", item["error"] + assert item["household_id"] == household_id + assert item["completed_at"] is not None + result = item["result"] + net_income = result["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert isinstance(net_income, int | float) + assert result["spm_config"] is not None + assert result["spm_provenance"] is not None + assert result["policyengine_version"] + read_again = api_client.get( + f"/v2/household-calculations/{item['calculation_id']}" + ).json()["result"] + assert read_again == item + + +def test_parallel_calculations_finish_within_the_latency_limits( + api_base_url, integration_probe_id +): + run_amount = probe_amount(integration_probe_id) + households = [ + _us_household(f"parallel-{index}", 20_000 + 1_000 * index + run_amount) + for index in range(PARALLEL_CALCULATIONS) + ] + + def run(household: LiveHousehold) -> FinishedCalculation: + with httpx.Client(base_url=api_base_url, timeout=90.0) as client: + return calculate(client, household, store_household(client, household)) + + with ThreadPoolExecutor(max_workers=PARALLEL_CALCULATIONS) as pool: + outcomes = list(pool.map(run, households)) + + submit_times = [outcome.submit_seconds for outcome in outcomes] + finish_times = [outcome.total_seconds for outcome in outcomes] + warnings.warn( + HouseholdCalculationMeasurement( + f"{PARALLEL_CALCULATIONS} parallel household calculations: submit " + f"median {statistics.median(submit_times):.2f} s, longest " + f"{max(submit_times):.2f} s; finish median " + f"{statistics.median(finish_times):.1f} s, longest " + f"{max(finish_times):.1f} s" + ), + stacklevel=1, + ) + assert [outcome.item["status"] for outcome in outcomes] == ["succeeded"] * len( + outcomes + ), [outcome.item["error"] for outcome in outcomes] + assert max(submit_times) <= SUBMIT_LIMIT_SECONDS, submit_times + assert max(finish_times) <= FINISH_LIMIT_SECONDS, finish_times + + +def test_a_household_across_many_points_finishes(api_client, integration_probe_id): + household = LiveHousehold( + name="memory", + country_id="us", + people={"adult": {"age": {YEAR: 45}}}, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": {YEAR: None}, + } + }, + axes=[ + [ + { + "name": "employment_income", + "count": MEMORY_CHECK_POINTS, + "min": 0, + "max": 400_000 + probe_amount(integration_probe_id), + "period": YEAR, + } + ] + ], + ) + household_id = store_household(api_client, household) + + finished = calculate(api_client, household, household_id) + + warnings.warn( + HouseholdCalculationMeasurement( + f"Household across {MEMORY_CHECK_POINTS} points: submit " + f"{finished.submit_seconds:.2f} s, finish {finished.total_seconds:.1f} s" + ), + stacklevel=1, + ) + assert finished.item["status"] == "succeeded", finished.item["error"] + net_income = finished.item["result"]["household"]["households"]["household-1"][ + "household_net_income" + ][YEAR] + assert len(net_income) == MEMORY_CHECK_POINTS diff --git a/tests/integration/test_live_v2_household_parity.py b/tests/integration/test_live_v2_household_parity.py new file mode 100644 index 000000000..f6042f6ca --- /dev/null +++ b/tests/integration/test_live_v2_household_parity.py @@ -0,0 +1,188 @@ +"""Live staging parity of v2 household calculations with API v1's ``/calculate``. + +Each household is stored through ``/v2/households`` (and its reform through +``/v2/policies``), calculated through ``/v2/household-calculations`` (the v2 +household function on Modal) and, as the same situation and policy, through +API v1's in-process ``/calculate`` (``/calculate-full`` when it adds missing +variables) on the same candidate. Every calculated value must match, axes, +reforms and added variables included. A run-specific input makes each +household new, so no earlier calculation is reused. +""" + +from __future__ import annotations + +import pytest + +from tests.fixtures.integration.live_household_calculations import ( + YEAR, + LiveHousehold, + calculate, + calculate_with_v1, + household_differences, + probe_amount, + store_household, + store_reform, +) + +CALCULATED = {YEAR: None} + + +def parity_households(run_amount: int) -> list[LiveHousehold]: + return [ + LiveHousehold( + name="us-single-california", + country_id="us", + people={ + "adult": { + "age": {YEAR: 35}, + "employment_income": {YEAR: 42_000 + run_amount}, + # Deprecated: dropped with a warning by both calculations. + "medical_out_of_pocket_expenses": {YEAR: 1_000}, + "income_tax": CALCULATED, + } + }, + groups={ + "household": { + "state_name": {YEAR: "CA"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + expects_warnings=True, + ), + LiveHousehold( + name="us-couple-two-children-texas-axes", + country_id="us", + people={ + "first-adult": {"age": {YEAR: 38}}, + "second-adult": { + "age": {YEAR: 36}, + "employment_income": {YEAR: 15_000 + run_amount}, + }, + "first-child": {"age": {YEAR: 7}}, + "second-child": {"age": {YEAR: 3}}, + }, + groups={ + "household": { + "state_name": {YEAR: "TX"}, + "household_net_income": CALCULATED, + }, + "tax_unit": {"eitc": CALCULATED, "ctc": CALCULATED}, + "spm_unit": {"snap": CALCULATED}, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 100_000, + "period": YEAR, + } + ] + ], + ), + LiveHousehold( + name="us-single-reform-added-variables", + country_id="us", + # /calculate-full adds variables for the age year of a person + # named ``you``; the v2 path for the household's default year. + people={ + "you": { + "age": {YEAR: 29}, + "employment_income": {YEAR: 61_000 + run_amount}, + } + }, + groups={"household": {"state_name": {YEAR: "NY"}}}, + reform={"gov.irs.deductions.standard.amount.SINGLE": 20_000}, + add_missing_variables=True, + ), + LiveHousehold( + name="uk-single", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 30}, + "employment_income": {YEAR: 28_000 + run_amount}, + "income_tax": CALCULATED, + "national_insurance": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + ), + LiveHousehold( + name="uk-single-reform", + country_id="uk", + people={ + "adult": { + "age": {YEAR: 41}, + "employment_income": {YEAR: 36_000 + run_amount}, + "income_tax": CALCULATED, + } + }, + groups={"household": {"household_net_income": CALCULATED}}, + reform={"gov.hmrc.income_tax.allowances.personal_allowance.amount": 15_000}, + ), + LiveHousehold( + name="uk-couple-child-axes", + country_id="uk", + people={ + "first-adult": {"age": {YEAR: 34}}, + "second-adult": { + "age": {YEAR: 33}, + "employment_income": {YEAR: 9_000 + run_amount}, + }, + "child": {"age": {YEAR: 4}}, + }, + groups={ + "household": {"household_net_income": CALCULATED}, + "benunit": { + "universal_credit": CALCULATED, + "child_benefit": CALCULATED, + }, + }, + axes=[ + [ + { + "name": "employment_income", + "count": 5, + "min": 0, + "max": 60_000, + "period": YEAR, + } + ] + ], + ), + ] + + +PARITY_NAMES = [household.name for household in parity_households(0)] + + +@pytest.mark.parametrize("household_name", PARITY_NAMES) +def test_a_v2_household_calculation_matches_api_v1( + api_client, integration_probe_id, household_name +): + (household,) = [ + candidate + for candidate in parity_households(probe_amount(integration_probe_id)) + if candidate.name == household_name + ] + household_id = store_household(api_client, household) + policy_id = store_reform(api_client, household) + + finished = calculate(api_client, household, household_id, policy_id) + v1_answer = calculate_with_v1(api_client, household) + + assert finished.item["status"] == "succeeded", finished.item["error"] + result = finished.item["result"] + differences = household_differences(v1_answer["result"], result["household"]) + assert differences == [], "\n".join(differences[:50]) + # The v2 database orders each stored record's keys itself, so warnings, + # listed in the household's key order, may come in another order. + assert sorted(result["warnings"]) == sorted(v1_answer.get("warnings", [])) + assert bool(result["warnings"]) == household.expects_warnings + if household.country_id == "us": + assert result["spm_config"] is not None + assert result["spm_provenance"] is not None diff --git a/tests/integration/test_live_v2_metadata.py b/tests/integration/test_live_v2_metadata.py index 235304156..00ff143e2 100644 --- a/tests/integration/test_live_v2_metadata.py +++ b/tests/integration/test_live_v2_metadata.py @@ -1,8 +1,11 @@ """Read-only checks for deployed Cloud Run v2 metadata preview routes.""" +import os +import time import tomllib from pathlib import Path +import httpx import pytest REPO = Path(__file__).parents[2] @@ -44,7 +47,15 @@ def test_live_v2_openapi_describes_every_preview_resource(api_client) -> None: assert response.status_code == 200, response.text[:500] document = response.json() expected_paths = {f"/v2/{resource}" for resource in PAGED_RESOURCES} - expected_paths.update({"/v2/parameters/children", "/v2/economy-options"}) + expected_paths.update( + { + "/v2/parameters/children", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", + } + ) assert expected_paths <= document["paths"].keys() assert all("get" in document["paths"][path] for path in expected_paths) assert "V2ErrorResponse" in document["components"]["schemas"] @@ -99,8 +110,129 @@ def test_live_v2_invalid_version_returns_a_typed_error(api_client) -> None: params={"country_id": "us", "policyengine_version": "not a version"}, ) - assert response.status_code == 400, response.text[:500] + assert response.status_code == 422, response.text[:500] payload = response.json() assert payload["status"] == "error" assert payload["message"] assert "result" not in payload or payload["result"] is None + + +def _timed_get( + api_client: httpx.Client, path: str, **request_options: object +) -> tuple[httpx.Response, float]: + started = time.perf_counter() + response = api_client.get(path, **request_options) + return response, time.perf_counter() - started + + +def _record_route_times( + country_id: str, route_times: list[tuple[str, float, httpx.Response]] +) -> None: + """Append each route's time and size to the GitHub job summary, when there is one. + + The release workflow runs this test against each Cloud Run candidate, so + the summary keeps every release's staging and production route times. + """ + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + lines = [ + f"\n#### v2 catalog route times ({country_id})\n", + "| Request | Time (ms) | Body (bytes, uncompressed) |", + "| --- | ---: | ---: |", + ] + lines.extend( + f"| {label} | {seconds * 1000:.0f} | {len(response.content)} |" + for label, seconds, response in route_times + ) + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write("\n".join(lines) + "\n") + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_live_v2_catalog_parts_load_on_demand(api_client, country_id: str) -> None: + expected_version = _installed_policyengine_version() + route_times: list[tuple[str, float, httpx.Response]] = [] + + def timed_get(label: str, path: str, **request_options: object) -> httpx.Response: + response, seconds = _timed_get(api_client, path, **request_options) + route_times.append((label, seconds, response)) + return response + + # The times are recorded even when an assertion fails, so every release + # keeps the measurements it reached. + try: + _check_catalog_parts(timed_get, country_id, expected_version) + finally: + _record_route_times(country_id, route_times) + + +def _check_catalog_parts(timed_get, country_id: str, expected_version: str) -> None: + catalog = _assert_ok( + timed_get( + "/v2/model-catalog (running version)", + "/v2/model-catalog", + params={"country_id": country_id}, + ), + expected_version, + ) + assert catalog["regions"] + assert catalog["economy_options"]["region"] + assert "variables" not in catalog + pinned = {"country_id": country_id, "policyengine_version": expected_version} + + variables_response = timed_get( + "/v2/model-catalog/variables", "/v2/model-catalog/variables", params=pinned + ) + second_variables = timed_get( + "/v2/model-catalog/variables (second request; another instance may serve it)", + "/v2/model-catalog/variables", + params=pinned, + ) + variables = _assert_ok(variables_response, expected_version) + assert variables["variables"] + assert variables_response.headers["cache-control"] == "public, max-age=86400" + assert second_variables.headers["etag"] == variables_response.headers["etag"] + revalidated = timed_get( + "/v2/model-catalog/variables (revalidated)", + "/v2/model-catalog/variables", + params=pinned, + headers={"If-None-Match": variables_response.headers["etag"]}, + ) + assert revalidated.status_code == 304 + + # The two existing routes measured slow in production on 2026-10-21. + page = _assert_ok( + timed_get( + "/v2/parameters (page of 500)", + "/v2/parameters", + params={**pinned, "limit": 500}, + ), + expected_version, + ) + first_parameter = page["items"][0] + one_parameter_values = _assert_ok( + timed_get( + "/v2/parameter-values (one parameter)", + "/v2/parameter-values", + params={**pinned, "parameter_id": first_parameter["id"]}, + ), + expected_version, + ) + assert all( + value["parameter_id"] == first_parameter["id"] + for value in one_parameter_values["items"] + ) + + parameters = _assert_ok( + timed_get( + "/v2/model-catalog/parameters (1 name)", + "/v2/model-catalog/parameters", + params={**pinned, "name": first_parameter["name"]}, + ), + expected_version, + ) + assert [parameter["name"] for parameter in parameters["parameters"]] == [ + first_parameter["name"] + ] + assert parameters["missing_names"] == [] diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py index 0f9106e06..b705d68d0 100644 --- a/tests/integration/test_v2_catalog_installed.py +++ b/tests/integration/test_v2_catalog_installed.py @@ -4,13 +4,18 @@ import json import os +from collections import Counter from importlib import metadata as importlib_metadata +from typing import TYPE_CHECKING import pytest from policyengine.bundle import get_current_bundle from policyengine_api.data.v2.catalog.extraction import extract_installed_catalog +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.records import CountryCatalog + pytestmark = pytest.mark.skipif( os.environ.get("RUN_V2_CATALOG_COMPATIBILITY") != "1", reason=("loads the complete public catalog; set RUN_V2_CATALOG_COMPATIBILITY=1"), @@ -38,10 +43,9 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: ) # Counts for the 6.10.0 bundle's models: US 2.2.1 and UK 2.132.6. Keep # this exact assertion so later package changes require explicit review. - # Since 6.10.0 the UK registry also lists its 650 constituencies and 361 - # local authorities, because the bundle routes them to a local-area - # release, so regions grow by 1,011 from 6.7.5's 826 and the catalog - # adds that release's dataset. + # Regions come from the checked-in reference files: 488 US (places are + # left out) and 1,016 UK, whose constituencies and local authorities use + # the local-area dataset the release routes them to. assert catalog.entity_counts() == { "models": 2, "model_versions": 2, @@ -50,7 +54,7 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "parameters": 104_172, "parameter_values": 1_194_360, "datasets": 4, - "regions": 1_837, + "regions": 1_504, } for country_id in ("us", "uk"): @@ -73,18 +77,18 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: total_values += len(batch) assert total_values == country.entity_counts()["parameter_values"] - assert {dataset.name for dataset in catalog.country("us").datasets} == { - "populace_us_2024", - "populace_us_2024_acs_local", - } - assert {dataset.name for dataset in catalog.country("uk").datasets} == { + _check_uk_regions(catalog.country("uk")) + _check_us_regions(catalog.country("us")) + + +def _check_uk_regions(uk_catalog: CountryCatalog) -> None: + assert {dataset.name for dataset in uk_catalog.datasets} == { "microcosm_uk_2024_25", "microcosm_uk_2024_25_dense", } # The national file has no area codes, so UK constituencies and local # authorities use the local-area dataset .py routes them to, while the # UK countries filter the national file. - uk_catalog = catalog.country("uk") uk_dataset_names = {dataset.id: dataset.name for dataset in uk_catalog.datasets} uk_region_type_datasets: dict[str, set[str]] = {} for region in uk_catalog.regions: @@ -99,9 +103,7 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: } assert uk_catalog.fallback_summaries == () england = next( - region - for region in catalog.country("uk").regions - if region.code == "country/england" + region for region in uk_catalog.regions if region.code == "country/england" ) assert england.filter_field == "region" assert england.filter_strategy == "row_filter" @@ -118,11 +120,39 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "SOUTH_WEST", ] + uk_types = Counter(region.region_type for region in uk_catalog.regions) + assert uk_types == { + "national": 1, + "country": 4, + "constituency": 650, + "local_authority": 361, + } + uk_regions = {region.code: region for region in uk_catalog.regions} + aldershot = uk_regions["constituency/E14001063"] + assert ( + aldershot.filter_strategy, + aldershot.filter_field, + aldershot.filter_value, + ) == ("row_filter", "constituency_code_oa", "E14001063") + ards = uk_regions["local_authority/N09000011"] + assert (ards.label, ards.filter_field, ards.filter_value) == ( + "Ards and North Down", + "la_code_oa", + "N09000011", + ) + assert uk_regions["uk"].label == "United Kingdom" + + +def _check_us_regions(us_catalog: CountryCatalog) -> None: + assert {dataset.name for dataset in us_catalog.datasets} == { + "populace_us_2024", + "populace_us_2024_acs_local", + } # Since 6.7.5, US state and congressional-district regions use the # ACS-local dataset again, on the PolicyEngine/microcosm#1263 rebuild # (PolicyEngine/policyengine.py#649, ending #604's national-file filter). - # The national region uses the national dataset, and places fall back to it. - us_catalog = catalog.country("us") + # The national region uses the national dataset. Regions come from the + # checked-in reference files; US places cannot run yet and are left out. us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets} region_type_datasets: dict[str, set[str]] = {} for region in us_catalog.regions: @@ -133,11 +163,13 @@ def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: "national": {"populace_us_2024"}, "state": {"populace_us_2024_acs_local"}, "congressional_district": {"populace_us_2024_acs_local"}, - "place": {"populace_us_2024"}, } + assert us_catalog.fallback_summaries == () assert [ (summary.region_type, summary.count) - for summary in us_catalog.fallback_summaries - ] == [ - ("place", 333), - ] + for summary in us_catalog.omitted_region_summaries + ] == [("place", 333)] + + assert {region.code: region.label for region in us_catalog.regions}["us"] == ( + "United States" + ) diff --git a/tests/integration/test_v2_catalog_metadata_parity.py b/tests/integration/test_v2_catalog_metadata_parity.py new file mode 100644 index 000000000..079e31610 --- /dev/null +++ b/tests/integration/test_v2_catalog_metadata_parity.py @@ -0,0 +1,204 @@ +"""Parity of catalog format 2's fields with the v1 ``/{country}/metadata`` document. + +The catalog reads these fields from PolicyEngine.py's public records; the v1 +document builds them from the country package's tax-benefit system. Each new +field must agree for the US and the UK. The check normalizes the installed +public models' metadata only, so it does not depend on the certified data +release that region publication reads. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest + +from policyengine_api.data.v2.catalog.extraction import normalize_model_metadata + +if TYPE_CHECKING: + from collections.abc import Iterable, Mapping + + from policyengine_api.data.v2.catalog.extraction import ModelMetadata + +pytestmark = pytest.mark.skipif( + os.environ.get("RUN_V2_CATALOG_COMPATIBILITY") != "1", + reason=("loads the complete public catalog; set RUN_V2_CATALOG_COMPATIBILITY=1"), +) + +COUNTRY_PACKAGES = {"us": "policyengine_us", "uk": "policyengine_uk"} + + +@pytest.fixture(scope="module", params=("us", "uk")) +def country_pair(request: pytest.FixtureRequest) -> tuple[ModelMetadata, Any]: + """Return one country's normalized metadata and its v1 metadata builder.""" + import policyengine + + from policyengine_api.country import PolicyEngineCountry + + country_id = request.param + metadata = normalize_model_metadata( + country_id, + getattr(policyengine, country_id).model, + model_version_id=uuid4(), + ) + return metadata, PolicyEngineCountry(COUNTRY_PACKAGES[country_id], country_id) + + +def _report(mismatches: list[str]) -> str: + shown = "\n".join(mismatches[:20]) + return f"{len(mismatches)} mismatches, first {min(len(mismatches), 20)}:\n{shown}" + + +def _typed_enum_values( + values: Iterable[Mapping[str, Any]], +) -> list[tuple[str, Any, str]]: + """Return each enum value with its label and the label's type name. + + Comparing the type name as well keeps 1, 1.0 and True apart, which ``==`` + treats as equal. + """ + return [ + (value["value"], value["label"], type(value["label"]).__name__) + for value in values + ] + + +def test_entities_and_basic_inputs_match_v1( + country_pair: tuple[ModelMetadata, Any], +) -> None: + metadata, v1_country = country_pair + v1_entities = v1_country.metadata["entities"] + + assert [ + (entity.key, entity.plural, entity.is_person) for entity in metadata.entities + ] == [ + (key, entity["plural"], entity["is_person"]) + for key, entity in v1_entities.items() + ] + assert list(metadata.basic_inputs) == (v1_country.metadata["basicInputs"]) + + +def test_variable_flags_modules_enums_parts_units_and_periods_match_v1( + country_pair: tuple[ModelMetadata, Any], +) -> None: + metadata, v1_country = country_pair + v1_variables = v1_country.metadata["variables"] + mismatches: list[str] = [] + + assert {variable.name for variable in metadata.variables} == set(v1_variables) + for variable in metadata.variables: + v1_variable = v1_variables[variable.name] + enum_values = ( + _typed_enum_values( + {"value": item.value, "label": item.label} + for item in variable.enum_values + ) + if variable.enum_values is not None + else None + ) + v1_enum_values = ( + _typed_enum_values(v1_variable["possibleValues"]) + if "possibleValues" in v1_variable + else None + ) + comparisons = { + "is_input_variable": ( + variable.is_input_variable, + v1_variable["isInputVariable"], + ), + "hidden_input": (variable.hidden_input, v1_variable["hidden_input"]), + "module_name": (variable.module_name, v1_variable["moduleName"]), + "enum_values": (enum_values, v1_enum_values), + "unit": (variable.unit, v1_variable["unit"]), + "definition_period": ( + variable.definition_period, + v1_variable["definitionPeriod"], + ), + "adds_parameter_path": ( + variable.adds_parameter_path, + v1_variable["adds"] if isinstance(v1_variable["adds"], str) else None, + ), + "subtracts_parameter_path": ( + variable.subtracts_parameter_path, + v1_variable["subtracts"] + if isinstance(v1_variable["subtracts"], str) + else None, + ), + } + mismatches.extend( + f"{variable.name}.{field}: catalog {stored!r}, v1 {expected!r}" + for field, (stored, expected) in comparisons.items() + if stored != expected or type(stored) is not type(expected) + ) + + assert not mismatches, _report(mismatches) + + +def test_parameter_flags_match_v1(country_pair: tuple[ModelMetadata, Any]) -> None: + metadata, v1_country = country_pair + v1_parameters = { + name: entry + for name, entry in v1_country.metadata["parameters"].items() + if entry["type"] == "parameter" + } + catalog_parameters = { + parameter.name: parameter + for parameter in metadata.parameters + if parameter.name.startswith("gov") + } + mismatches: list[str] = [] + + assert set(catalog_parameters) == set(v1_parameters) + for name, parameter in catalog_parameters.items(): + expected = (v1_parameters[name]["economy"], v1_parameters[name]["household"]) + if (parameter.economy, parameter.household) != expected: + mismatches.append( + f"{name}: catalog {(parameter.economy, parameter.household)}, " + f"v1 {expected}" + ) + + assert not mismatches, _report(mismatches) + + +def test_bracket_schedules_are_nodes_with_the_stored_label_v1_shows( + country_pair: tuple[ModelMetadata, Any], +) -> None: + """Every schedule v1 lists is a catalog node; a stored label matches v1's. + + v1 falls back to the name's last part when a schedule has no stored label; + the catalog stores no label, and the routes generate one. + """ + from policyengine_core.parameters import ParameterScale + + metadata, v1_country = country_pair + nodes = {node.name: node for node in metadata.parameter_nodes} + v1_parameters = v1_country.metadata["parameters"] + schedules = [ + parameter + for parameter in v1_country.tax_benefit_system.parameters.get_descendants() + if isinstance(parameter, ParameterScale) and parameter.name.startswith("gov") + ] + mismatches: list[str] = [] + + assert schedules + for schedule in schedules: + node = nodes.get(schedule.name) + stored_label = schedule.metadata.get("label") + if node is None: + mismatches.append(f"{schedule.name}: not a catalog node") + elif node.label != stored_label: + mismatches.append( + f"{schedule.name}: catalog label {node.label!r}, " + f"stored label {stored_label!r}" + ) + elif stored_label is not None and ( + v1_parameters[schedule.name]["label"] != stored_label + ): + mismatches.append( + f"{schedule.name}: catalog label {node.label!r}, " + f"v1 label {v1_parameters[schedule.name]['label']!r}" + ) + + assert not mismatches, _report(mismatches) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 25bc9178f..a5ba64d1e 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -15,10 +15,14 @@ from sqlalchemy.pool import NullPool from sqlmodel import Session, select +from policyengine_api.data.v2.catalog import publication as publication_module +from policyengine_api.data.v2.catalog.extraction import extract_catalog from policyengine_api.data.v2.catalog.publication import ( CatalogPublicationError, publish_catalog, + unpublished_countries, ) +from policyengine_api.data.v2.catalog.records import EnumValueRecord from policyengine_api.data.v2.models import ( Dataset, DatasetVersion, @@ -30,7 +34,15 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from tests.fixtures.v2_catalog import normalized_catalog +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + add_format_2_variable_records, + bundle, + installed_version, + normalized_catalog, + region_references, + source_models, +) REPO = Path(__file__).parents[2] DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" @@ -121,6 +133,7 @@ def test_publish_and_republish_preserve_complete_catalog( second = publish_catalog(publication_engine, catalog) assert first.as_dict()["outcome"] == "ok" + assert first.omitted_region_summaries == (("us", "place", 1),) assert second.policyengine_version == first.policyengine_version assert ( _counts(publication_engine) @@ -132,9 +145,9 @@ def test_publish_and_republish_preserve_complete_catalog( "parameter_nodes": 2, "parameters": 2, "parameter_values": 4, - "datasets": 3, + "datasets": 4, "dataset_versions": 0, - "regions": 5, + "regions": 8, "simulations": 0, "reports": 0, "report_runs": 0, @@ -163,14 +176,44 @@ def test_publish_and_republish_preserve_complete_catalog( storage_path is None and not output for _, storage_path, output in datasets ) assert dict(region_defaults) == { + "congressional_district/CA-01": "populace_us_2024_acs_local", + "constituency/E14001063": "microcosm_uk_2024_25_dense", "country/england": "microcosm_uk_2024_25", - "place/CA-44000": "populace_us_2024", - "state/ca": "populace_us_ca_2024", + "country/northern_ireland": "microcosm_uk_2024_25", + "local_authority/N09000011": "microcosm_uk_2024_25_dense", + "state/ca": "populace_us_2024_acs_local", "uk": "microcosm_uk_2024_25", "us": "populace_us_2024", } +def test_unpublished_country_check_reports_a_missing_schema_specifically() -> None: + # The same server's maintenance database has no v2 schema at all. + schema_less = make_url(_disposable_url()).set(database="postgres") + engine = create_engine(schema_less, poolclass=NullPool) + try: + with pytest.raises( + CatalogPublicationError, + match=r"^the v2 Alembic revision table is absent$", + ): + unpublished_countries(engine, normalized_catalog()) + finally: + engine.dispose() + + +def test_lists_only_countries_whose_catalog_version_is_not_stored( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + newer = normalized_catalog(policyengine_version="4.21.0") + + assert unpublished_countries(publication_engine, catalog) == ("us", "uk") + publish_catalog(publication_engine, catalog) + + assert unpublished_countries(publication_engine, catalog) == () + assert unpublished_countries(publication_engine, newer) == ("us", "uk") + + @pytest.mark.parametrize( "failure_point", ["during_copy", "after_reconciliation", "after_validation"], @@ -209,13 +252,190 @@ def test_same_version_with_changed_content_fails_without_mutation( countries=(changed_us, catalog.country("uk")), ) - with pytest.raises(CatalogPublicationError, match="differs"): + with pytest.raises( + CatalogPublicationError, + match=r"^the stored us catalog version 5\.0\.4 differs from the staged " + r"catalog in its variables$", + ): publish_catalog(publication_engine, changed) assert _identifiers(publication_engine) == before_ids assert _counts(publication_engine) == before_counts +def test_stored_version_with_a_different_dataset_names_the_datasets( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + uk = catalog.country("uk") + changed = replace( + catalog, + countries=( + catalog.country("us"), + replace( + uk, + datasets=tuple( + replace(dataset, description="Another dataset selection") + for dataset in uk.datasets + ), + ), + ), + ) + + with pytest.raises( + CatalogPublicationError, + match=r"^the stored uk catalog version 5\.0\.4 differs from the staged " + r"catalog in its datasets$", + ): + publish_catalog(publication_engine, changed) + + +def _region_rows(engine: Engine) -> list[tuple[object, ...]]: + with engine.connect() as connection: + return [ + tuple(row) + for row in connection.execute( + text( + """ + SELECT id, code, label, region_type, requires_filter, + filter_field, filter_value, filter_strategy, + parent_code, state_code, state_name, default_dataset_id + FROM regions + ORDER BY id + """ + ) + ) + ] + + +def _with_changed_us_regions(catalog): + """Relabel state/ca and drop the district, as a reference-file change would.""" + us = catalog.country("us") + regions = tuple( + replace(region, label="Changed California") + if region.code == "state/ca" + else region + for region in us.regions + if region.code != "congressional_district/CA-01" + ) + return replace( + catalog, + countries=(replace(us, regions=regions), catalog.country("uk")), + ) + + +def test_stored_version_keeps_its_published_regions_and_logs_the_difference( + publication_engine: Engine, + monkeypatch: pytest.MonkeyPatch, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_regions = _region_rows(publication_engine) + before_ids = _identifiers(publication_engine) + before_counts = _counts(publication_engine) + # Alembic's logging configuration disables existing loggers, so the + # warnings are recorded where they are emitted. + warnings: list[tuple[str, dict]] = [] + monkeypatch.setattr( + publication_module.LOGGER, + "warning", + lambda message, *args, **kwargs: warnings.append((message % args, kwargs)), + ) + + evidence = publish_catalog(publication_engine, _with_changed_us_regions(catalog)) + + assert evidence.as_dict()["outcome"] == "ok" + assert _region_rows(publication_engine) == before_regions + assert _identifiers(publication_engine) == before_ids + assert _counts(publication_engine) == before_counts + # The evidence describes the stored rows, not the staged regions. + assert evidence.entity_counts == { + "models": before_counts["tax_benefit_models"], + "model_versions": before_counts["tax_benefit_model_versions"], + "variables": before_counts["variables"], + "parameter_nodes": before_counts["parameter_nodes"], + "parameters": before_counts["parameters"], + "parameter_values": before_counts["parameter_values"], + "datasets": before_counts["datasets"], + "regions": before_counts["regions"], + } + assert evidence.kept_region_summaries == ( + ("us", catalog.policyengine_version, 3, 2), + ) + assert evidence.omitted_region_summaries == () + assert evidence.fallback_summaries == () + notices = [ + (message, keywords["extra"]) + for message, keywords in warnings + if message.startswith("kept the published regions") + ] + assert [message for message, _extra in notices] == [ + "kept the published regions of the stored us catalog version " + f"{catalog.policyengine_version}: 1 staged region rows are not stored and " + "2 stored region rows are not staged" + ] + assert [extra["policyengine_attributes"] for _message, extra in notices] == [ + { + "country_id": "us", + "policyengine_version": catalog.policyengine_version, + "staged_region_rows_not_stored": 1, + "stored_region_rows_not_staged": 2, + } + ] + + +def test_stored_version_with_changed_regions_still_fails_on_other_tables( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_regions = _region_rows(publication_engine) + changed = _with_changed_us_regions(catalog) + us = changed.country("us") + changed = replace( + changed, + countries=( + replace(us, variables=(replace(us.variables[0], label="Changed"),)), + changed.country("uk"), + ), + ) + + with pytest.raises(CatalogPublicationError, match="differs"): + publish_catalog(publication_engine, changed) + + assert _region_rows(publication_engine) == before_regions + + +def test_new_version_publishes_the_reference_file_regions( + publication_engine: Engine, +) -> None: + publish_catalog(publication_engine, normalized_catalog()) + newer = _with_changed_us_regions(normalized_catalog(policyengine_version="4.21.0")) + + publish_catalog(publication_engine, newer) + + with publication_engine.connect() as connection: + newer_us_regions = dict( + connection.execute( + text( + """ + SELECT region.code, region.label + FROM regions AS region + JOIN tax_benefit_model_versions AS version + ON version.id = region.tax_benefit_model_version_id + JOIN tax_benefit_models AS model ON model.id = version.model_id + WHERE version.version = '4.21.0' AND model.name = 'policyengine-us' + """ + ) + ).all() + ) + assert newer_us_regions == { + "state/ca": "Changed California", + "us": "United States", + } + + def test_republish_preserves_existing_run_and_dataset_version_records( publication_engine: Engine, ) -> None: @@ -333,10 +553,248 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "parameter_nodes": 4, "parameters": 4, "parameter_values": 8, - "datasets": 6, + "datasets": 8, "dataset_versions": 0, - "regions": 10, + "regions": 16, "simulations": 0, "reports": 0, "report_runs": 0, } + + +def _format_2_values(engine: Engine) -> dict[str, object]: + """Return the US version's stored catalog format 2 fields.""" + with engine.connect() as connection: + version = connection.execute( + text( + "SELECT version.catalog_format, version.entities, version.basic_inputs " + "FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us'" + ) + ).one() + variable = connection.execute( + text( + "SELECT is_input_variable, hidden_input, module_name, " + "adds_parameter_path, subtracts_parameter_path, enum_values, " + "unit, definition_period " + "FROM variables WHERE name = 'employment_income' " + "AND tax_benefit_model_version_id = (" + "SELECT version.id FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us')" + ) + ).one() + parameter = connection.execute( + text( + "SELECT economy, household FROM parameters " + "WHERE name = 'gov.example.rate' " + "AND tax_benefit_model_version_id = (" + "SELECT version.id FROM tax_benefit_model_versions AS version " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us')" + ) + ).one() + return { + "version": tuple(version), + "variable": tuple(variable), + "parameter": tuple(parameter), + } + + +def _clear_format_2_fields(engine: Engine) -> None: + """Make every stored version look as if published before catalog format 2.""" + with engine.begin() as connection: + connection.execute( + text( + "UPDATE tax_benefit_model_versions SET catalog_format = NULL, " + "entities = NULL, basic_inputs = NULL" + ) + ) + connection.execute( + text( + "UPDATE variables SET is_input_variable = NULL, hidden_input = NULL, " + "module_name = NULL, adds_parameter_path = NULL, " + "subtracts_parameter_path = NULL, enum_values = NULL, " + "unit = NULL, definition_period = NULL" + ) + ) + connection.execute( + text("UPDATE parameters SET economy = NULL, household = NULL") + ) + + +def test_publication_stores_catalog_format_2_fields( + publication_engine: Engine, +) -> None: + publish_catalog(publication_engine, normalized_catalog()) + + assert _format_2_values(publication_engine) == { + "version": ( + 2, + [ + {"key": "person", "plural": "people", "is_person": True}, + {"key": "household", "plural": "households", "is_person": False}, + ], + ["employment_income"], + ), + "variable": ( + True, + False, + "household.income.employment_income", + None, + None, + None, + "currency-USD", + "year", + ), + "parameter": (True, False), + } + + +def test_version_stored_before_format_2_keeps_its_fields_empty_on_republish( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + _clear_format_2_fields(publication_engine) + before_ids = _identifiers(publication_engine) + + result = publish_catalog(publication_engine, catalog) + + assert result.as_dict()["outcome"] == "ok" + assert _identifiers(publication_engine) == before_ids + assert _format_2_values(publication_engine) == { + "version": (None, None, None), + "variable": (None, None, None, None, None, None, None, None), + "parameter": (None, None), + } + + +def test_version_stored_with_format_2_fails_when_a_format_2_field_changes( + publication_engine: Engine, +) -> None: + catalog = normalized_catalog() + publish_catalog(publication_engine, catalog) + before_counts = _counts(publication_engine) + us = catalog.country("us") + changed_parameter = replace(us.parameters[0], household=True) + changed = replace( + catalog, + countries=( + replace(us, parameters=(changed_parameter,)), + catalog.country("uk"), + ), + ) + + with pytest.raises(CatalogPublicationError, match="differs"): + publish_catalog(publication_engine, changed) + + assert _counts(publication_engine) == before_counts + assert _format_2_values(publication_engine)["parameter"] == (True, False) + + +def _catalog_with_format_2_variable_records(): + models = source_models() + add_format_2_variable_records(models["us"]) + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + region_references=region_references(), + ) + + +def _stored_us_variables(engine: Engine) -> dict[str, tuple[object, ...]]: + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT variable.name, variable.enum_values, " + "variable.adds_parameter_path, variable.subtracts_parameter_path, " + "variable.unit, variable.definition_period " + "FROM variables AS variable " + "JOIN tax_benefit_model_versions AS version " + "ON version.id = variable.tax_benefit_model_version_id " + "JOIN tax_benefit_models AS model ON model.id = version.model_id " + "WHERE model.name = 'policyengine-us'" + ) + ).all() + return {row[0]: tuple(row[1:]) for row in rows} + + +def _with_changed_us_variable(catalog, name: str, **changes: object): + us = catalog.country("us") + variables = tuple( + replace(variable, **changes) if variable.name == name else variable + for variable in us.variables + ) + return replace( + catalog, + countries=(replace(us, variables=variables), catalog.country("uk")), + ) + + +def test_enum_labels_parameter_paths_units_and_periods_round_trip_exactly( + publication_engine: Engine, +) -> None: + catalog = _catalog_with_format_2_variable_records() + + publish_catalog(publication_engine, catalog) + before_ids = _identifiers(publication_engine) + stored = _stored_us_variables(publication_engine) + republished = publish_catalog(publication_engine, catalog) + + assert stored["filing_status"] == ( + [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], + None, + None, + None, + "year", + ) + assert type(stored["filing_status"][0][1]["label"]) is int + assert stored["countable_income"] == ( + None, + "gov.example.countable_income", + "gov.example.deductions", + "currency-USD", + "month", + ) + assert republished.as_dict()["outcome"] == "ok" + assert _identifiers(publication_engine) == before_ids + + +@pytest.mark.parametrize( + ("variable_name", "changes"), + [ + ( + "filing_status", + { + "enum_values": ( + EnumValueRecord(value="SINGLE", label="Single filer"), + EnumValueRecord(value="JOINT", label=2), + ) + }, + ), + ("countable_income", {"adds_parameter_path": "gov.example.deductions"}), + ("countable_income", {"subtracts_parameter_path": None}), + ("filing_status", {"unit": "currency-USD"}), + ("countable_income", {"definition_period": "year"}), + ], +) +def test_republication_detects_a_changed_enum_label_parameter_path_unit_or_period( + publication_engine: Engine, + variable_name: str, + changes: dict[str, object], +) -> None: + catalog = _catalog_with_format_2_variable_records() + publish_catalog(publication_engine, catalog) + stored = _stored_us_variables(publication_engine) + + with pytest.raises(CatalogPublicationError, match=r"in its variables$"): + publish_catalog( + publication_engine, + _with_changed_us_variable(catalog, variable_name, **changes), + ) + + assert _stored_us_variables(publication_engine) == stored diff --git a/tests/integration/test_v2_catalog_spm_availability.py b/tests/integration/test_v2_catalog_spm_availability.py new file mode 100644 index 000000000..9df0b3564 --- /dev/null +++ b/tests/integration/test_v2_catalog_spm_availability.py @@ -0,0 +1,80 @@ +"""Record SPM availability on published catalog versions in disposable Postgres.""" + +from __future__ import annotations + +from uuid import uuid4 + +import sqlalchemy as sa +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog.spm_availability import ( + MODEL_VERSIONS, + MODELS, + record_spm_availability, +) + +AVAILABLE = {"available": True, "settings_schema": {}, "defaults": {"scenario": "x"}} +UNAVAILABLE = {"available": False} + + +def test_availability_fills_only_the_named_version_and_only_once( + disposable_v2_database_url: str, +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + model_id, version_id, other_version_id = uuid4(), uuid4(), uuid4() + model_name = f"policyengine-us-{model_id.hex[:8]}" + try: + with engine.begin() as connection: + connection.execute(sa.insert(MODELS).values(id=model_id, name=model_name)) + connection.execute( + sa.insert(MODEL_VERSIONS), + [ + { + "id": identifier, + "model_id": model_id, + "version": version, + "current_law_id": 2, + "metadata_time_periods": [2026], + } + for identifier, version in ( + (version_id, "1.450.0"), + (other_version_id, "1.449.0"), + ) + ], + ) + first = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=AVAILABLE, + ) + again = record_spm_availability( + connection, + model_name=model_name, + version="1.450.0", + availability=UNAVAILABLE, + ) + with engine.connect() as connection: + stored = dict( + connection.execute( + sa.select( + MODEL_VERSIONS.c.id, MODEL_VERSIONS.c.spm_availability + ).where(MODEL_VERSIONS.c.model_id == model_id) + ).all() + ) + finally: + with engine.begin() as connection: + connection.execute( + sa.delete(MODEL_VERSIONS).where(MODEL_VERSIONS.c.model_id == model_id) + ) + connection.execute(sa.delete(MODELS).where(MODELS.c.id == model_id)) + engine.dispose() + + assert first + assert not again + assert stored == {version_id: AVAILABLE, other_version_id: None} diff --git a/tests/integration/test_v2_country_routes.py b/tests/integration/test_v2_country_routes.py new file mode 100644 index 000000000..ee6408ddc --- /dev/null +++ b/tests/integration/test_v2_country_routes.py @@ -0,0 +1,111 @@ +"""Serve and check the v2 countries from a migrated disposable Postgres database. + +The route and the startup check run as the Public API's runtime role, so the +tests prove the role's declared privileges cover them (rule +``api.v2.runtime-privileges-tested-as-runtime-role``). +""" + +from __future__ import annotations + +import pytest +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.constants import REPO +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS +from tests.fixtures.v2_countries import LISTED_COUNTRY_ROWS + + +def test_postgres_countries_list_the_rows_the_revision_inserted( + managed_v2_roles, + v2_engine_as_role, +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + sessions = sessionmaker(runtime_engine, class_=Session) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: V2CountryService( + CountryDatabaseSession(sessions) + ), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), + dependencies=dependencies, + route_settings=settings, + ) + ) + + try: + response = client.get("/v2/countries") + finally: + runtime_engine.dispose() + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": {"items": list(LISTED_COUNTRY_ROWS)}, + } + + +def test_postgres_countries_match_the_code_list_as_the_runtime_role( + managed_v2_roles, + v2_engine_as_role, + disposable_v2_database_url: str, +) -> None: + """The startup check passes on the revision's rows and fails on an extra row.""" + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(runtime_engine, class_=Session)) + ) + owner_engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + + try: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + with owner_engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('ca', true, false)" + ) + ) + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + assert raised.value.rows_not_listed == ("ca",) + finally: + with owner_engine.begin() as connection: + connection.execute(text("DELETE FROM countries WHERE id = 'ca'")) + owner_engine.dispose() + runtime_engine.dispose() diff --git a/tests/integration/test_v2_metadata_routes.py b/tests/integration/test_v2_metadata_routes.py index 326eb78d9..7f479d183 100644 --- a/tests/integration/test_v2_metadata_routes.py +++ b/tests/integration/test_v2_metadata_routes.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 @@ -11,7 +12,7 @@ from alembic.config import Config from fastapi.testclient import TestClient from flask import Flask, jsonify -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, event, text from sqlalchemy.engine import Engine, make_url from sqlalchemy.pool import NullPool from sqlmodel import Session, select @@ -20,8 +21,11 @@ from policyengine_api.data.v2.catalog.publication import publish_catalog from policyengine_api.data.v2.models import ( Dataset, + Parameter, + ParameterValue, TaxBenefitModel, TaxBenefitModelVersion, + Variable, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies @@ -324,7 +328,7 @@ def test_postgres_resources_distinguish_invalid_and_absent_versions( params={"country_id": "us", "policyengine_version": "4.99.0"}, ) - assert invalid.status_code == 400 + assert invalid.status_code == 422 assert invalid.json()["status"] == "error" assert invalid.json()["message"] assert absent.status_code == 404 @@ -360,3 +364,362 @@ def test_postgres_parameter_collection_remains_available_without_values( assert parameters.status_code == values.status_code == 200 assert parameters.json()["result"]["items"] assert values.json()["result"]["items"] == [] + + +@pytest.mark.parametrize("country_id", ["us", "uk"]) +def test_postgres_model_catalog_serves_the_small_parts_of_a_version( + published_engine: Engine, + country_id: str, +) -> None: + client = _client(published_engine) + country = normalized_catalog().country(country_id) + + running = client.get("/v2/model-catalog", params={"country_id": country_id}) + named = client.get( + "/v2/model-catalog", + params={"country_id": country_id, "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert running.status_code == 200 + assert running.headers["cache-control"] == "public, max-age=300" + assert named.headers["cache-control"] == "public, max-age=86400" + assert running.headers["etag"] == named.headers["etag"] + result = running.json()["result"] + assert result["policyengine_version"] == POLICYENGINE_VERSION + assert result["catalog_format"] == country.model_version.catalog_format + assert result["entities"] == [ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ] + assert result["basic_inputs"] == list(country.model_version.basic_inputs) + assert [region["code"] for region in result["regions"]] == sorted( + region.code for region in country.regions + ) + assert [dataset["name"] for dataset in result["datasets"]] == sorted( + dataset.name for dataset in country.datasets + ) + assert [option["name"] for option in result["economy_options"]["region"]] == [ + region["code"] for region in result["regions"] + ] + assert "variables" not in result + assert "parameters" not in result + + revalidated = client.get( + "/v2/model-catalog", + params={"country_id": country_id}, + headers={"If-None-Match": running.headers["etag"]}, + ) + assert revalidated.status_code == 304 + + +def _us_model_version_id(engine: Engine): + with Session(engine) as session: + return session.exec( + select(TaxBenefitModelVersion.id) + .join( + TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id + ) + .where(TaxBenefitModel.name == "policyengine-us") + ).one() + + +def test_postgres_catalog_variables_carry_part_parameter_values_by_date( + published_engine: Engine, +) -> None: + model_version_id = _us_model_version_id(published_engine) + parts_id = uuid4() + with Session(published_engine) as session: + session.add( + Parameter( + id=parts_id, + tax_benefit_model_version_id=model_version_id, + name="gov.example.parts", + label="Income parts", + economy=False, + household=True, + ) + ) + session.flush() + session.add_all( + [ + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + ), + ParameterValue( + parameter_id=parts_id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + ), + Variable( + tax_benefit_model_version_id=model_version_id, + name="total_income", + entity="person", + data_type="float", + default_value=0, + adds=["employment_income"], + is_input_variable=False, + hidden_input=False, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + ), + ] + ) + session.commit() + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": POLICYENGINE_VERSION}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income", + "total_income", + ] + assert result["variables"][0]["module_name"] == ( + "household.income.employment_income" + ) + assert result["variables"][0]["unit"] == "currency-USD" + assert result["variables"][0]["definition_period"] == "year" + assert result["variables"][1]["adds_parameter_path"] == "gov.example.parts" + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2024-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + }, + { + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income", "pension_income"], + }, + ] + } + uk = client.get("/v2/model-catalog/variables", params={"country_id": "uk"}) + assert [variable["name"] for variable in uk.json()["result"]["variables"]] == [ + "employment_income" + ] + assert uk.json()["result"]["part_parameter_values"] == {} + + +def test_postgres_catalog_parameters_load_values_and_breadcrumb_labels( + published_engine: Engine, +) -> None: + client = _client(published_engine) + + response = client.get( + "/v2/model-catalog/parameters", + params=[ + ("country_id", "us"), + ("policyengine_version", POLICYENGINE_VERSION), + ("name", "gov.example.rate"), + ("name", "gov.removed.amount"), + ], + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert [parameter["name"] for parameter in result["parameters"]] == [ + "gov.example.rate" + ] + parameter = result["parameters"][0] + assert (parameter["economy"], parameter["household"]) == (True, False) + assert parameter["values"] == [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": "2025-12-31T00:00:00Z", + "value": 0.1, + }, + {"start_date": "2026-01-01T00:00:00Z", "end_date": None, "value": 0.2}, + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed.amount"] + + +def test_postgres_metadata_reads_send_no_transaction_statements( + published_engine: Engine, +) -> None: + # Each metadata read is one statement on rows a published catalog never + # changes, so the read session runs in autocommit mode: psycopg then sends + # no BEGIN before the reads and no ROLLBACK when the session closes. The + # engine has one pooled connection, so the writer below must get the same + # connection back in the ordinary, transactional mode. + from psycopg.pq import TransactionStatus + + from policyengine_api.data.v2.database import build_v2_read_session_factory + + del published_engine # publishes the catalog this pooled engine reads + pooled_engine = create_engine(_disposable_url(), pool_size=1, max_overflow=0) + read_connections: list[object] = [] + autocommit_modes: list[bool] = [] + + def record_mode(connection, _cursor, _statement, *_args) -> None: + dbapi_connection = connection.connection.dbapi_connection + read_connections.append(dbapi_connection) + autocommit_modes.append(dbapi_connection.autocommit) + + try: + event.listen(pooled_engine, "before_cursor_execute", record_mode) + service = V2MetadataService( + MetadataDatabaseSession(build_v2_read_session_factory(pooled_engine)()), + running_policyengine_version=POLICYENGINE_VERSION, + ) + result = service.list_parameters("us") + service.close() + event.remove(pooled_engine, "before_cursor_execute", record_mode) + + with Session(pooled_engine) as writer: + writer_connection = writer.connection().connection.dbapi_connection + writer.execute(text("SELECT 1")) + writer_status = writer_connection.info.transaction_status + writer_autocommit = writer_connection.autocommit + finally: + pooled_engine.dispose() + + assert [item.name for item in result.items] == ["gov.example.rate"] + assert autocommit_modes + assert all(autocommit_modes) + assert {id(connection) for connection in read_connections} == { + id(writer_connection) + } + assert writer_autocommit is False + assert writer_status == TransactionStatus.INTRANS + + +def test_postgres_production_metadata_reader_keeps_catalogs_and_responses( + published_engine: Engine, +) -> None: + # The production combination: autocommit read sessions, the process's + # selected catalogs and its whole-version response cache. + from policyengine_api.data.v2.database import build_v2_read_session_factory + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, + ) + + read_factory = build_v2_read_session_factory(published_engine) + selected_catalogs = SelectedCatalogStore() + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_metadata_reader_factory=lambda: V2MetadataService( + MetadataDatabaseSession(read_factory()), + running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, + ), + v2_metadata_response_cache=MetadataResponseCache(), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ) + ) + query = {"country_id": "us", "policyengine_version": POLICYENGINE_VERSION} + + event.listen(published_engine, "before_cursor_execute", record_statement) + try: + first_variables = client.get( + "/v2/model-catalog/variables", params={"country_id": "us"} + ) + statements_for_first = len(statements) + repeated_variables = client.get("/v2/model-catalog/variables", params=query) + statements_for_repeat = len(statements) - statements_for_first + parameters = client.get( + "/v2/parameters", params={"country_id": "us", "name": "gov.example.rate"} + ) + statements_for_parameters = ( + len(statements) - statements_for_first - statements_for_repeat + ) + finally: + event.remove(published_engine, "before_cursor_execute", record_statement) + + assert first_variables.status_code == 200 + assert repeated_variables.content == first_variables.content + assert repeated_variables.headers["etag"] == first_variables.headers["etag"] + # The running version's selection and the variables read (the fixture's + # variables name no parameter-defined parts, so no value read); then + # nothing for the explicit request, which shares the kept response; then + # only the parameter read, the running version's selection being kept. + assert statements_for_first == 2 + assert statements_for_repeat == 0 + assert statements_for_parameters == 1 + assert [item["name"] for item in parameters.json()["result"]["items"]] == [ + "gov.example.rate" + ] + assert selected_catalogs.get(("us", None)) is not None + + +@pytest.mark.parametrize( + "case", + [ + # Read for 31 December, headers written after midnight: not reusable. + ( + datetime(2025, 12, 31, 23, 59, 59, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, 1, tzinfo=UTC), + [0.1], + "public, max-age=0", + ), + ( + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + datetime(2025, 12, 31, 12, 0, tzinfo=UTC), + [0.1], + "public, max-age=43200", + ), + ( + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + datetime(2026, 1, 1, 0, 0, tzinfo=UTC), + [0.2], + "public, max-age=86400", + ), + ], +) +def test_postgres_current_values_and_their_lifetime_use_one_instant( + published_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + case: tuple[datetime, datetime, list[float], str], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant, header_instant, expected_values, expected_cache_control = case + instants = iter([read_instant, header_instant]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + + response = _client(published_engine).get( + "/v2/parameter-values", + params={ + "country_id": "us", + "policyengine_version": POLICYENGINE_VERSION, + "current": "true", + }, + ) + + assert response.status_code == 200 + assert [item["value"] for item in response.json()["result"]["items"]] == ( + expected_values + ) + assert response.headers["cache-control"] == expected_cache_control diff --git a/tests/integration/test_v2_privilege_reconciliation.py b/tests/integration/test_v2_privilege_reconciliation.py index f720c3fb9..9596a5966 100644 --- a/tests/integration/test_v2_privilege_reconciliation.py +++ b/tests/integration/test_v2_privilege_reconciliation.py @@ -23,7 +23,7 @@ check_declared_privileges, run, ) -from policyengine_api.data.v2.privileges import READ, TablePrivilege +from policyengine_api.data.v2.privileges import READ, READ_WRITE, TablePrivilege from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL if TYPE_CHECKING: @@ -62,6 +62,21 @@ def create_role() -> str: connection.execute(text(f'DROP ROLE "{role}"')) +@pytest.fixture +def temporary_view(superuser_engine) -> Iterator[str]: + """Create a view over ``regions`` for one test, then drop it.""" + view = f"policyengine_privilege_test_view_{uuid4().hex[:12]}" + with superuser_engine.begin() as connection: + connection.execute( + text(f'CREATE VIEW public."{view}" AS SELECT * FROM public.regions') + ) + try: + yield view + finally: + with superuser_engine.begin() as connection: + connection.execute(text(f'DROP VIEW public."{view}"')) + + def _has_privilege(engine: Engine, role: str, table: str, privilege: str) -> bool: with engine.connect() as connection: return bool( @@ -137,3 +152,56 @@ def test_a_missing_role_blocks_every_grant_and_fails_verification( PrivilegeEntry(role, "regions", "SELECT"), ] ) + + +def test_apply_grants_and_verify_checks_privileges_on_a_declared_view( + superuser_engine, temporary_role, temporary_view +) -> None: + """A view declared like a table is found, granted and verified.""" + role = temporary_role() + declared = {role: {temporary_view: READ_WRITE}} + with superuser_engine.begin() as connection: + plan, _defaults = apply_declared_privileges( + connection, declared, declared_views={temporary_view} + ) + missing = check_declared_privileges( + connection, declared, declared_views={temporary_view} + ) + + assert plan.missing_tables == plan.missing_views == () + assert {entry.privilege for entry in plan.grants} == { + privilege.value for privilege in READ_WRITE + } + assert missing == [] + for privilege in READ_WRITE: + assert _has_privilege(superuser_engine, role, temporary_view, privilege.value) + + +def test_a_view_declared_as_a_table_or_a_table_declared_as_a_view_blocks_grants( + superuser_engine, temporary_role, temporary_view +) -> None: + """Each declared name must exist with its declared kind. + + ``regions`` is a table, declared here as a view, and the temporary view is + declared as a table: neither is granted, and both fail verification. + """ + role = temporary_role() + declared = {role: {"regions": READ, temporary_view: READ}} + with superuser_engine.begin() as connection: + plan, _defaults = apply_declared_privileges( + connection, declared, declared_views={"regions"} + ) + missing = check_declared_privileges( + connection, declared, declared_views={"regions"} + ) + + assert plan.missing_views == ("regions",) + assert plan.missing_tables == (temporary_view,) + assert not plan.can_apply + assert not _has_privilege(superuser_engine, role, "regions", "SELECT") + assert missing == sorted( + [ + PrivilegeEntry(role, "regions", "SELECT"), + PrivilegeEntry(role, temporary_view, "SELECT"), + ] + ) diff --git a/tests/integration/test_v2_saved_policy_view.py b/tests/integration/test_v2_saved_policy_view.py new file mode 100644 index 000000000..7ffb666d5 --- /dev/null +++ b/tests/integration/test_v2_saved_policy_view.py @@ -0,0 +1,382 @@ +"""The release before the saved-policy rename keeps working through the view. + +Revision ``fa7fa1035444`` renames ``user_policies`` to +``user_policy_associations`` and keeps a temporary view under the old name. The +Public API release that still serves during the deploy, or after a rollback to +it, runs the same statements as today's services with the old table name. +These tests run today's services as the runtime role, rewrite each statement's +table name to ``user_policies`` before it is sent, as that release would +compile it, and check the result in the renamed table. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, event, select, text +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.models import ( + LegacyPolicyMapping, + LegacyUserMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserIdentity, +) +from policyengine_api.services.v2.identity.types import ( + UnauthenticatedPrincipal, + UserPrincipal, +) +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot +from policyengine_api.services.v2.user_accounts.database_session import ( + AccountDatabaseSession, +) +from policyengine_api.services.v2.user_accounts.services import V2AccountService +from policyengine_api.services.v2.user_policies.database_session import ( + UserPolicyDatabaseSession, +) +from policyengine_api.services.v2.user_policies.services import ( + V2UserPolicyService, + mirror_legacy_user_policy_in_session, +) +from policyengine_api.services.v2.user_policies.types import ( + LegacyUserPolicySnapshot, + UserPolicyCreateFields, + UserPolicyUpdateInput, +) +from policyengine_api.services.v2.user_reports.database_session import ( + UserReportDatabaseSession, +) +from policyengine_api.services.v2.user_reports.services import V2UserReportService +from policyengine_api.services.v2.user_reports.types import UserReportCreateFields +from tests.unit.v2.user_service_support import default_legacy_references + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Engine + +TABLE_BEFORE_THE_RENAME = "user_policies" +TABLE_AFTER_THE_RENAME = "user_policy_associations" +# The renamed table where a statement names it as a table, not inside a label +# such as ``user_policy_associations_id``. +RENAMED_TABLE_REFERENCE = re.compile(rf"\b{TABLE_AFTER_THE_RENAME}\b") +MIRRORED_LEGACY_USER_ID = "auth0|saved-policy-view" + + +@pytest.fixture +def cleanup_engine(disposable_v2_database_url) -> Iterator[Engine]: + engine = create_engine(disposable_v2_database_url) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture +def previous_release_statements() -> list[str]: + """Every statement on saved policies, as sent with the earlier table name.""" + return [] + + +@pytest.fixture +def sessions( + managed_v2_roles, v2_engine_as_role, cleanup_engine, previous_release_statements +) -> Iterator[sessionmaker[Session]]: + """Open sessions as the runtime role that name saved policies as before. + + Each statement that names ``user_policy_associations`` is sent naming + ``user_policies`` instead. Results are matched to columns by position, so + the rewritten labels do not matter. + """ + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime) + + @event.listens_for(runtime_engine, "before_cursor_execute", retval=True) + def name_saved_policies_as_before_the_rename( + _connection, _cursor, statement, parameters, _context, _executemany + ): + if RENAMED_TABLE_REFERENCE.search(statement): + statement = RENAMED_TABLE_REFERENCE.sub(TABLE_BEFORE_THE_RENAME, statement) + previous_release_statements.append(statement) + return statement, parameters + + factory = sessionmaker(runtime_engine, class_=Session, expire_on_commit=False) + issuer = f"https://saved-policy-view-{uuid4().hex[:12]}.example.test/" + factory.kw["info"] = {"issuer": issuer, "browser_user_ids": []} + try: + yield factory + finally: + with cleanup_engine.begin() as connection: + owned = select(UserIdentity.user_id).where(UserIdentity.issuer == issuer) + user_ids = [ + *connection.execute(owned).scalars(), + *factory.kw["info"]["browser_user_ids"], + ] + if user_ids: + connection.execute(delete(User).where(User.id.in_(user_ids))) + runtime_engine.dispose() + + +def _browser(sessions) -> UnauthenticatedPrincipal: + user_id = uuid4() + sessions.kw["info"]["browser_user_ids"].append(user_id) + return UnauthenticatedPrincipal(user_id=user_id) + + +def _saved_policy_row(cleanup_engine, association_id: UUID) -> dict | None: + with cleanup_engine.connect() as connection: + row = ( + connection.execute( + text(f"SELECT * FROM {TABLE_AFTER_THE_RENAME} WHERE id = :id"), + {"id": association_id}, + ) + .mappings() + .one_or_none() + ) + return None if row is None else dict(row) + + +def _statement_kinds(statements: list[str]) -> set[str]: + """Return the kind of each statement: its first word.""" + return {statement.split(maxsplit=1)[0].upper() for statement in statements} + + +def test_runtime_role_holds_its_saved_policy_privileges_on_the_view( + sessions, managed_v2_roles +) -> None: + with sessions() as session: + held = { + privilege: session.execute( + text("SELECT has_table_privilege(:role, :view, :privilege)"), + { + "role": managed_v2_roles.runtime, + "view": f"public.{TABLE_BEFORE_THE_RENAME}", + "privilege": privilege, + }, + ).scalar_one() + for privilege in ("SELECT", "INSERT", "UPDATE", "DELETE") + } + + assert held == dict.fromkeys(("SELECT", "INSERT", "UPDATE", "DELETE"), True) + + +def test_previous_release_saves_reads_renames_and_deletes_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + browser = _browser(sessions) + policies = V2UserPolicyService( + UserPolicyDatabaseSession(sessions), default_legacy_references() + ) + + created = policies.create_user_policy( + principal=browser, + association_input=UserPolicyCreateFields( + country_id="us", legacy_policy_id=1, name="Saved" + ), + ) + stored_after_create = _saved_policy_row(cleanup_engine, created.id) + read = policies.get_user_policy( + principal=browser, country_id="us", association_id=created.id + ) + page = policies.list_user_policies(principal=browser, country_id="us") + renamed = policies.patch_user_policy( + principal=browser, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(name="Renamed"), + ) + stored_after_rename = _saved_policy_row(cleanup_engine, created.id) + policies.delete_user_policy( + principal=browser, country_id="us", association_id=created.id + ) + + assert stored_after_create is not None + assert stored_after_create["user_id"] == browser.user_id + assert stored_after_create["created_at"] == created.created_at + assert read.id == created.id + assert [association.id for association in page.items] == [created.id] + assert renamed.name == "Renamed" + assert stored_after_rename is not None + assert stored_after_rename["name"] == "Renamed" + assert _saved_policy_row(cleanup_engine, created.id) is None + assert _statement_kinds(previous_release_statements) == { + "INSERT", + "SELECT", + "UPDATE", + "DELETE", + } + + +def test_previous_release_merges_saved_policies_and_checks_reports_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + """Sign-in moves a browser's saved policies, and reports name them.""" + browser = _browser(sessions) + signed_in = UserPrincipal( + issuer=sessions.kw["info"]["issuer"], subject="google-oauth2|view-merge" + ) + references = default_legacy_references() + policies = V2UserPolicyService(UserPolicyDatabaseSession(sessions), references) + reports = V2UserReportService(UserReportDatabaseSession(sessions), references) + accounts = V2AccountService(AccountDatabaseSession(sessions)) + + account_id = accounts.get_account(principal=signed_in, country_id="us").id + saved = policies.create_user_policy( + principal=browser, + association_input=UserPolicyCreateFields(country_id="us", legacy_policy_id=1), + ) + report = reports.create_user_report( + principal=browser, + association_input=UserReportCreateFields( + country_id="us", + legacy_report_id=615, + user_policy_id=saved.id, + baseline_user_policy_id=saved.id, + ), + ) + accounts.upgrade_account( + principal=signed_in, + unauthenticated_user_id=browser.user_id, + country_id="us", + ) + exported = accounts.export_account(principal=signed_in, country_id="us") + + stored = _saved_policy_row(cleanup_engine, saved.id) + assert stored is not None + assert stored["user_id"] == account_id + assert report.user_policy_id == saved.id + assert [policy.id for policy in exported.policies] == [saved.id] + # The sign-in merge moves every saved policy of the browser's user. + assert any( + re.match( + rf"UPDATE {TABLE_BEFORE_THE_RENAME} SET .*\buser_id=.* WHERE " + rf"{TABLE_BEFORE_THE_RENAME}\.user_id = ", + statement, + ) + for statement in previous_release_statements + ) + + +def _seed_catalog(cleanup_engine) -> tuple[UUID, str]: + parameter_name = f"gov.saved_policy_view.rate_{uuid4().hex[:8]}" + with Session(cleanup_engine) as session, session.begin(): + # The policy copy selects the catalog by this model name. + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, + version=POLICYENGINE_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + session.add(Parameter(name=parameter_name, tax_benefit_model_version=version)) + session.flush() + return model.id, parameter_name + + +def _delete_mirrored_rows(cleanup_engine, model_id: UUID) -> None: + with cleanup_engine.begin() as connection: + user_ids = select(LegacyUserMapping.user_id).where( + LegacyUserMapping.legacy_user_id == MIRRORED_LEGACY_USER_ID + ) + mirrored_user_ids = connection.execute(user_ids).scalars().all() + connection.execute( + delete(LegacyUserMapping).where( + LegacyUserMapping.legacy_user_id == MIRRORED_LEGACY_USER_ID + ) + ) + # Deleting the user deletes its saved policies and their v1 mappings. + connection.execute(delete(User).where(User.id.in_(mirrored_user_ids))) + policy_ids = select(Policy.id).where(Policy.tax_benefit_model_id == model_id) + for model in (LegacyPolicyMapping, ParameterValue): + connection.execute(delete(model).where(model.policy_id.in_(policy_ids))) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == model_id) + ) + version_ids = select(TaxBenefitModelVersion.id).where( + TaxBenefitModelVersion.model_id == model_id + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id.in_(version_ids) + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.model_id == model_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == model_id) + ) + + +def _legacy_saved_policy(reform_label: str) -> LegacyUserPolicySnapshot: + return LegacyUserPolicySnapshot.model_validate( + { + "country_id": "us", + "legacy_user_policy_id": 901, + "reform_id": 801, + "reform_label": reform_label, + "baseline_id": 1, + "baseline_label": "Current law", + "user_id": MIRRORED_LEGACY_USER_ID, + "year": "2026", + "geography": "us", + "dataset": "enhanced_cps_2024", + "number_of_provisions": 1, + "api_version": "1.0.0", + "added_date": 1, + "updated_date": 2, + "budgetary_impact": None, + "type": None, + } + ) + + +def test_previous_release_copies_v1_saved_policies_through_the_view( + sessions, cleanup_engine, previous_release_statements +) -> None: + """The Stage 10 saved-policy copy inserts and then updates through the view.""" + model_id, parameter_name = _seed_catalog(cleanup_engine) + reform = LegacyPolicySnapshot( + country_id="us", + legacy_policy_id=801, + label="Ignored legacy label", + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + policy_json={parameter_name: {"2026": 0.2}}, + source_policy_hash=f"view-{uuid4().hex}", + ) + try: + with sessions.begin() as session: + first = mirror_legacy_user_policy_in_session( + session, _legacy_saved_policy("Copied"), reform, source_revision=1 + ) + with sessions.begin() as session: + second = mirror_legacy_user_policy_in_session( + session, + _legacy_saved_policy("Copied again"), + reform, + source_revision=2, + changed_fields=frozenset({"reform_label"}), + ) + stored = _saved_policy_row(cleanup_engine, first.association_id) + finally: + _delete_mirrored_rows(cleanup_engine, model_id) + + assert first.association_created is True + assert second.association_id == first.association_id + assert second.association_created is False + assert stored is not None + assert stored["name"] == "Copied again" + assert {"INSERT", "SELECT", "UPDATE"} <= _statement_kinds( + previous_release_statements + ) diff --git a/tests/unit/routes/test_migration_context_logging.py b/tests/unit/routes/test_migration_context_logging.py index 641c8c34b..efc3d27bf 100644 --- a/tests/unit/routes/test_migration_context_logging.py +++ b/tests/unit/routes/test_migration_context_logging.py @@ -717,3 +717,79 @@ def test_compute_routes_in_the_same_groups_keep_their_compute_labels(): assert context["route_group"] == "simulation" assert context["sim_flow"] == "economy" assert context["sim_compute"] is not None + + +def test_household_calculations_log_their_actual_sources_and_compute(monkeypatch): + monkeypatch.setenv("DB_READ_HOUSEHOLD", "invalid-for-native-v2") + monkeypatch.setenv("DB_WRITE_HOUSEHOLD", "invalid-for-native-v2") + monkeypatch.setenv("SIM_ENTRYPOINT", "old_gateway_direct") + monkeypatch.delenv("SIM_COMPUTE_HOUSEHOLD", raising=False) + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + for method, path in ( + ("POST", "/v2/household-calculations"), + ( + "GET", + "/v2/household-calculations/00000000-0000-0000-0000-000000000001", + ), + ): + log_migration_request( + ServedRequest( + request_id=f"request-{method}", + method=method, + path=path, + status_code=202 if method == "POST" else 200, + started_at=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + + submit_context, read_context = ( + call.args[0]["migration"] for call in mock_logger.log_struct.call_args_list + ) + computed_on_v2 = { + "route_group": "household", + "route_impl": "fastapi_native", + "sim_flow": "household", + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", + } + assert submit_context == { + **submit_context, + **computed_on_v2, + "db_write": None, + "db_read": "supabase", + } + assert read_context == { + **read_context, + **computed_on_v2, + "db_write": None, + "db_read": None, + } + + +def test_household_calculations_ignore_compute_flags_they_do_not_use(monkeypatch): + monkeypatch.setenv("SIM_ENTRYPOINT", "invalid-unused") + monkeypatch.setenv("SIM_COMPUTE_HOUSEHOLD", "invalid-unused") + + with patch("policyengine_api.migration_logging.logger") as mock_logger: + log_migration_request( + ServedRequest( + request_id="request-submit", + method="POST", + path="/v2/household-calculations", + status_code=202, + started_at=None, + route_impl=RouteImplementation.FASTAPI_NATIVE, + ) + ) + + context = mock_logger.log_struct.call_args.args[0]["migration"] + assert "migration_flag_error" not in context + assert context == { + **context, + "route_group": "household", + "db_read": "supabase", + "sim_entrypoint": "cloud_run_simulation_entrypoint", + "sim_compute": "v2_primary", + } diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 797c82848..23e3fb846 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -3,6 +3,7 @@ import os import re import subprocess +import tomllib from pathlib import Path import pytest @@ -218,9 +219,11 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "alembic -c alembic-v2.ini upgrade head" in workflow assert "test_alembic_v2_lifecycle.sh" not in workflow assert "test_v2_catalog_installed.py" in workflow + assert "test_v2_catalog_metadata_parity.py" in workflow assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow + assert "test_v2_country_routes.py" in workflow assert "test_v2_policy_persistence.py" in workflow assert "test_v1_policy_dual_write.py" in workflow assert "test_mysql_policy_dual_write.py" in workflow @@ -257,6 +260,41 @@ def test_v2_integration_check_grants_and_verifies_privileges_before_tests(): assert schema_upgrade < grant < verify < first_test +def _integration_modules_using(pattern: str) -> list[str]: + return sorted( + path.relative_to(REPO).as_posix() + for path in (REPO / "tests" / "integration").glob("test_*.py") + if re.search(pattern, path.read_text(encoding="utf-8")) + ) + + +def test_every_managed_role_integration_module_runs_in_ci_and_is_in_its_rule(): + """Integration modules that use the managed v2 roles run in the workflow. + + Those that run services as a managed role (``v2_engine_as_role``) are also + in the scope of rule ``api.v2.runtime-privileges-tested-as-runtime-role``. + """ + workflow = _workflow("v2-integration-check.yml") + rules = tomllib.loads((REPO / "rules" / "registry.toml").read_text()) + (runtime_role_rule,) = ( + rule + for rule in rules["rule"] + if rule["id"] == "api.v2.runtime-privileges-tested-as-runtime-role" + ) + managed_role_modules = _integration_modules_using( + r"\b(managed_v2_roles|v2_engine_as_role)\b" + ) + role_session_modules = _integration_modules_using(r"\bv2_engine_as_role\b") + + assert "tests/integration/test_v2_saved_policy_view.py" in role_session_modules + assert [module for module in managed_role_modules if module not in workflow] == [] + assert [ + module + for module in role_session_modules + if module not in runtime_role_rule["scope"] + ] == [] + + def test_release_migration_fails_closed_before_tests_and_cloud_run_deploy(): workflow = _workflow("push.yml") orchestration_script = ( diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index ee7a48b9f..a9d0dc878 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -1894,6 +1894,12 @@ def _assert_live_tests_run_in_parallel_after_readiness( assert "tests/integration/test_live_budget_window_cache.py" in live_test_script assert "tests/integration/test_live_calculate.py" in live_test_script assert "tests/integration/test_live_economy.py" in live_test_script + assert "tests/integration/test_live_v2_household_calculations.py" in ( + live_test_script + ) + assert "tests/integration/test_live_v2_household_parity.py" in live_test_script + # The household calculation checks wait on Modal; 30 minutes leaves room. + assert "timeout-minutes: 30" in parallel_live_tests assert "tests/integration/test_live_v2_policies.py" not in parallel_live_tests assert ( "API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..90eec1780 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -13,8 +13,8 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 16, - "request_count": 89, + "workflow_count": 18, + "request_count": 95, "db_entity_count": 6, "sim_flow_count": 3, } @@ -25,12 +25,14 @@ def test_migration_contract_payload_summarizes_route_contracts(): "user_policy_associations_v2", "household_save_read", "household_resources_v2", + "household_calculations_v2", "user_household_associations_v2", "user_simulation_associations_v2", "user_report_associations_v2", "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", diff --git a/tests/unit/test_migration_flags.py b/tests/unit/test_migration_flags.py index e110da879..4ca5dd68f 100644 --- a/tests/unit/test_migration_flags.py +++ b/tests/unit/test_migration_flags.py @@ -181,6 +181,11 @@ def test_explicit_migration_context_rejects_invalid_database_sources( "policy", ), ("/v2/user-households", "household"), + ("/v2/household-calculations", "household"), + ( + "/v2/household-calculations/00000000-0000-0000-0000-000000000001", + "household", + ), ("/v2/user-simulations", "simulation"), ("/v2/user-reports/00000000-0000-0000-0000-000000000001", "report"), ("/v2/me", "user_profile"), diff --git a/tests/unit/test_simulation_entrypoint_environment_script.py b/tests/unit/test_simulation_entrypoint_environment_script.py new file mode 100644 index 000000000..3f5ca8e04 --- /dev/null +++ b/tests/unit/test_simulation_entrypoint_environment_script.py @@ -0,0 +1,256 @@ +"""Tests for .github/scripts/check_simulation_entrypoint_environment.sh.""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / ".github/scripts/check_simulation_entrypoint_environment.sh" +PUSH_WORKFLOW = REPO / ".github/workflows/push.yml" +CHECK_COMMAND = "bash .github/scripts/check_simulation_entrypoint_environment.sh" +SIMULATION_URL = "https://simulation.example.test" +STAGING_REVISION = "policyengine-simulation-entry-staging-00070-abc" +PRODUCTION_REVISION = "policyengine-simulation-entry-00122-low" +HEALTHY_BODY = '{"status":"healthy"}' + + +@dataclass(frozen=True) +class FakeCurlReply: + """What the fake curl prints, and the exit code it returns.""" + + output: str + exit_code: int = 0 + + +# Cloud Run's sample container answers with an HTML page and no revision header. +SAMPLE_CONTAINER_RESPONSE = FakeCurlReply( + "HTTP/2 200\r\ncontent-type: text/html; charset=utf-8\r\n\r\n" +) +# curl --fail exits with 22 when the server answers with an HTTP error. +FAILED_REQUEST = FakeCurlReply("", exit_code=22) + + +def _entrypoint_response(revision: str) -> FakeCurlReply: + return FakeCurlReply( + "HTTP/2 200\r\n" + "content-type: application/json\r\n" + f"x-policyengine-simulation-revision: {revision}\r\n" + "\r\n" + f"{HEALTHY_BODY}" + ) + + +def _install_fake_curl(tmp_path: Path, reply: FakeCurlReply) -> Path: + """Install a fake curl that records its arguments and gives ``reply``.""" + fake_bin_directory = tmp_path / "bin" + fake_bin_directory.mkdir() + response_file = tmp_path / "response.txt" + response_file.write_text(reply.output, encoding="utf-8") + calls_file = tmp_path / "curl-calls.log" + fake_curl = fake_bin_directory / "curl" + fake_curl.write_text( + "#!/usr/bin/env bash\n" + f'echo "$*" >> "{calls_file}"\n' + f'cat "{response_file}"\n' + f"exit {reply.exit_code}\n", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + return fake_bin_directory + + +def _run_check( + tmp_path: Path, + curl_reply: FakeCurlReply, + *, + deployment_environment: str, + sim_entrypoint: str = "cloud_run_simulation_entrypoint", + simulation_url: str = SIMULATION_URL, +) -> tuple[subprocess.CompletedProcess[str], list[str]]: + fake_bin_directory = _install_fake_curl(tmp_path, curl_reply) + result = subprocess.run( + ["bash", str(SCRIPT)], + cwd=REPO, + env={ + "PATH": f"{fake_bin_directory}{os.pathsep}{os.environ['PATH']}", + "DEPLOYMENT_ENVIRONMENT": deployment_environment, + "SIM_ENTRYPOINT": sim_entrypoint, + "SIMULATION_ENTRYPOINT_URL": simulation_url, + "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", + }, + text=True, + capture_output=True, + check=False, + ) + calls_file = tmp_path / "curl-calls.log" + curl_calls = calls_file.read_text().splitlines() if calls_file.exists() else [] + return result, curl_calls + + +@pytest.mark.parametrize( + ("deployment_environment", "revision"), + [("staging", STAGING_REVISION), ("production", PRODUCTION_REVISION)], +) +def test_accepts_the_simulation_service_of_the_same_environment( + tmp_path: Path, deployment_environment: str, revision: str +) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(revision), + deployment_environment=deployment_environment, + ) + + assert result.returncode == 0, result.stderr + assert len(curl_calls) == 1 + assert curl_calls[0].endswith(f"{SIMULATION_URL}/health") + assert revision in result.stdout + + +def test_requests_health_once_without_a_doubled_slash(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="staging", + simulation_url=f"{SIMULATION_URL}/", + ) + + assert result.returncode == 0, result.stderr + assert curl_calls[0].endswith(f" {SIMULATION_URL}/health") + + +def test_rejects_staging_calling_the_production_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "it must call policyengine-simulation-entry-staging" in result.stderr + assert PRODUCTION_REVISION in result.stderr + + +def test_rejects_production_calling_the_staging_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="production", + ) + + assert result.returncode == 1 + assert "it must call policyengine-simulation-entry\n" in result.stderr + assert STAGING_REVISION in result.stderr + + +def test_rejects_a_url_that_names_no_entrypoint_revision(tmp_path: Path) -> None: + result, _ = _run_check( + tmp_path, SAMPLE_CONTAINER_RESPONSE, deployment_environment="staging" + ) + + assert result.returncode == 1 + assert "did not name a Simulation Entrypoint revision" in result.stderr + + +def test_rejects_a_revision_name_without_a_cloud_run_suffix(tmp_path: Path) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response("policyengine-simulation-entry-staging"), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "is not a Cloud Run revision name" in result.stderr + + +def test_fails_when_the_health_request_fails(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, FAILED_REQUEST, deployment_environment="staging" + ) + + assert result.returncode == 22 + assert len(curl_calls) == 1 + + +def test_rejects_an_unknown_deployment_environment(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="preview", + ) + + assert result.returncode == 2 + assert "DEPLOYMENT_ENVIRONMENT must be staging or production" in result.stderr + assert curl_calls == [] + + +def test_rejects_a_missing_simulation_url(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="staging", + simulation_url="", + ) + + assert result.returncode == 1 + assert "SIMULATION_ENTRYPOINT_URL is required" in result.stderr + assert curl_calls == [] + + +def test_does_not_check_the_shared_old_gateway(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + sim_entrypoint="old_gateway_direct", + ) + + assert result.returncode == 0, result.stderr + assert curl_calls == [] + assert "old_gateway_direct" in result.stdout + + +@pytest.mark.parametrize( + ("job_name", "deployment_environment"), + [ + ("ensure-staging-model-version-aligns-with-sim-api", "staging"), + ("ensure-production-model-version-aligns-with-sim-api", "production"), + ], +) +def test_each_release_checks_its_own_simulation_service_first( + job_name: str, deployment_environment: str +) -> None: + """Each version-alignment job fails if it reaches the other environment. + + The check runs right after checkout, before the job asks the simulation + service which bundle it supports, so a crossed environment fails the + release instead of validating against the wrong service. + """ + job = yaml.safe_load(PUSH_WORKFLOW.read_text(encoding="utf-8"))["jobs"][job_name] + + assert job["environment"] == deployment_environment + assert job["env"]["DEPLOYMENT_ENVIRONMENT"] == deployment_environment + assert job["env"]["SIM_ENTRYPOINT"] == "${{ vars.SIM_ENTRYPOINT }}" + assert ( + job["env"]["SIMULATION_ENTRYPOINT_URL"] + == "${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + ) + step_commands = [step.get("run", "") for step in job["steps"]] + assert step_commands.count(CHECK_COMMAND) == 1 + check_position = step_commands.index(CHECK_COMMAND) + bundle_check_position = step_commands.index( + "bash .github/check-policyengine-bundle-supported.sh" + ) + assert ( + job["steps"][check_position - 1].get("uses", "").startswith("actions/checkout@") + ) + assert check_position < bundle_check_position diff --git a/tests/unit/test_startup_country_check.py b/tests/unit/test_startup_country_check.py new file mode 100644 index 000000000..64bd0f219 --- /dev/null +++ b/tests/unit/test_startup_country_check.py @@ -0,0 +1,229 @@ +"""The startup check that the v2 countries table matches the country list in code.""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest +from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError + +from policyengine_api import startup_country_check +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.startup_country_check import check_countries_table_at_startup +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + +V2_RUNTIME_SECRET_RESOURCE = "projects/example/secrets/v2-runtime/versions/latest" + + +@pytest.fixture(autouse=True) +def enabled_startup_check_logger(monkeypatch: pytest.MonkeyPatch) -> None: + """Re-enable the check's logger, which Alembic's logging setup disables. + + Tests that run Alembic commands load its logging configuration, which + disables every logger that already exists, including this module's. + """ + monkeypatch.setattr(startup_country_check.logger, "disabled", False) + + +class FakeCountryListVerifier: + """Record each verification, then raise ``error`` if one is set.""" + + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.verified_country_lists: list[tuple[str, ...]] = [] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + self.verified_country_lists.append(tuple(listed_country_ids)) + if self.error is not None: + raise self.error + + +class CountingVerifierFactory: + """Return one verifier and count how often startup asks for it.""" + + def __init__(self, verifier: FakeCountryListVerifier) -> None: + self.verifier = verifier + self.calls = 0 + + def __call__(self) -> FakeCountryListVerifier: + self.calls += 1 + return self.verifier + + +def test_startup_without_a_v2_runtime_database_skips_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={"FLASK_DEBUG": "1"}, verifier_factory=factory + ) + + assert factory.calls == 0 + + +def test_blank_selection_variables_skip_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={ + "K_SERVICE": "", + "V2_RUNTIME_DATABASE_URL": " ", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": "", + }, + verifier_factory=factory, + ) + + assert factory.calls == 0 + + +@pytest.mark.parametrize( + "environ", + [ + {"K_SERVICE": "policyengine-api"}, + {"V2_RUNTIME_DATABASE_URL": "postgresql+psycopg://configured"}, + {"V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": V2_RUNTIME_SECRET_RESOURCE}, + ], + ids=["cloud-run", "runtime-url", "runtime-secret"], +) +def test_a_selected_v2_runtime_database_is_checked_against_the_code_list( + environ: dict[str, str], +) -> None: + verifier = FakeCountryListVerifier() + + check_countries_table_at_startup( + environ=environ, verifier_factory=CountingVerifierFactory(verifier) + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + + +@pytest.mark.parametrize( + "error", + [ + CountryListMismatchError(listed_without_row=("us",), rows_not_listed=()), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception('relation "countries" does not exist'), + ), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception("permission denied for table countries"), + ), + ], + ids=["mismatch", "missing-table", "missing-privilege"], +) +def test_a_failed_check_stops_startup(error: Exception) -> None: + verifier = FakeCountryListVerifier(error=error) + + with pytest.raises(type(error)): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + +@pytest.mark.parametrize( + "message", + [ + "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE could not be resolved", + ], + ids=["invalid-url", "secret-unavailable"], +) +def test_unusable_v2_configuration_is_logged_and_left_to_readiness( + message: str, + caplog: pytest.LogCaptureFixture, +) -> None: + """Readiness already reports not ready for it, and recovers when it does.""" + + def unusable_configuration() -> FakeCountryListVerifier: + raise V2ConfigurationError(message) + + with caplog.at_level(logging.WARNING): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=unusable_configuration, + ) + + assert [record.levelno for record in caplog.records] == [logging.WARNING] + assert message in caplog.records[0].getMessage() + + +@pytest.mark.parametrize( + "error_type", + [OperationalError, InterfaceError], + ids=["operational", "interface"], +) +def test_an_unreachable_database_is_logged_and_startup_continues( + error_type: type[OperationalError | InterfaceError], + caplog: pytest.LogCaptureFixture, +) -> None: + verifier = FakeCountryListVerifier( + error=error_type( + "SELECT countries.id FROM countries", + {}, + Exception("connection to db.secret-host failed: password=hunter2"), + ) + ) + + with caplog.at_level(logging.WARNING): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + assert [record.levelno for record in caplog.records] == [logging.WARNING] + message = caplog.records[0].getMessage() + assert "countries table" in message + assert error_type.__name__ in message + assert "secret-host" not in message + assert "hunter2" not in message + + +def test_importing_the_asgi_application_runs_the_check(tmp_path: Path) -> None: + """The check runs at import; an unusable v2 runtime URL is only logged.""" + environment = os.environ.copy() + environment.update( + { + "POLICYENGINE_API_STARTUP_WARMUP": "0", + "V2_RUNTIME_DATABASE_URL": ( + "postgresql+psycopg://runtime:hunter2@localhost:5432/postgres" + "?sslmode=require" + ), + "V2_SUPABASE_PROJECT_REF": "abcdefghijklmnopqrst", + "V2_SUPABASE_ENVIRONMENT": "test-startup", + } + ) + for name in ("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", "K_SERVICE"): + environment.pop(name, None) + + result = subprocess.run( + [sys.executable, "-c", "import policyengine_api.asgi"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + # The application's log handler writes structured records to stdout. + output = result.stdout + result.stderr + assert result.returncode == 0, result.stderr + assert "Skipped the countries table check" in output + assert "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" in output + assert "hunter2" not in output diff --git a/tests/unit/test_supported_countries.py b/tests/unit/test_supported_countries.py new file mode 100644 index 000000000..8c4391366 --- /dev/null +++ b/tests/unit/test_supported_countries.py @@ -0,0 +1,146 @@ +"""The Public API's one country list, and the lists that derive from it.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from typing import get_args + +import pytest +from pydantic import TypeAdapter, ValidationError + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog import extraction +from policyengine_api.data.v2.catalog.region_references import ( + REGION_REFERENCE_DIRECTORY, +) +from policyengine_api.query_parameters import CountryId +from policyengine_api.supported_countries import ( + SUPPORTED_COUNTRY_IDS, + SupportedCountryId, +) + +# Runs in a fresh interpreter: it adds a country to the one list before any +# module that derives from it is imported, then reports what each derived +# list contains. PolicyEngine.py is replaced by a stand-in, because importing +# the real package builds every country model. +EXTENDED_LIST_SCRIPT = """ +import json +import sys +from types import ModuleType, SimpleNamespace +from typing import Literal, get_args + +import policyengine_api.supported_countries as supported_countries + +supported_countries.SupportedCountryId = Literal["us", "uk", "zz"] +supported_countries.SUPPORTED_COUNTRY_IDS = get_args( + supported_countries.SupportedCountryId +) + +installed_policyengine = ModuleType("policyengine") +for country_id in ("us", "uk", "zz", "ca"): + setattr( + installed_policyengine, + country_id, + SimpleNamespace(model=f"{country_id} model"), + ) +installed_bundle = ModuleType("policyengine.bundle") +installed_bundle.get_current_bundle = dict +sys.modules["policyengine"] = installed_policyengine +sys.modules["policyengine.bundle"] = installed_bundle + +from pydantic import TypeAdapter + +from policyengine_api.data.v2.catalog import catalog_selection, extraction +from policyengine_api.query_parameters import CountryId +from scripts import process_v1_household_mirror_event + +captured_arguments = {} + + +def capture_extraction(**arguments): + captured_arguments.update(arguments) + + +extraction.extract_catalog = capture_extraction +# A new country brings its own region reference file; stand in for it here. +extraction.load_region_references = lambda country_id: f"{country_id} regions" +extraction.extract_installed_catalog() +country_option = next( + action + for action in process_v1_household_mirror_event._parser()._actions + if action.dest == "country_id" +) +print( + json.dumps( + { + "country_type": TypeAdapter(CountryId).validate_python("ZZ"), + "extracted_countries": list(extraction.SUPPORTED_COUNTRIES), + "required_dependencies": list(extraction.REQUIRED_DEPENDENCIES), + "installed_models": captured_arguments["models"], + "dual_write_countries": sorted(catalog_selection.SUPPORTED_V2_COUNTRY_IDS), + "household_copy_countries": list(country_option.choices), + } + ) +) +""" + + +def test_the_country_list_is_the_us_and_the_uk_in_catalog_order() -> None: + assert SUPPORTED_COUNTRY_IDS == ("us", "uk") + assert get_args(SupportedCountryId) == SUPPORTED_COUNTRY_IDS + + +def test_the_v2_country_type_accepts_exactly_the_listed_countries() -> None: + country_id_adapter = TypeAdapter(CountryId) + + for country_id in SUPPORTED_COUNTRY_IDS: + assert country_id_adapter.validate_python(country_id) == country_id + assert country_id_adapter.validate_python(country_id.upper()) == country_id + for unlisted_country_id in ("ca", "ng", "il", ""): + with pytest.raises(ValidationError): + country_id_adapter.validate_python(unlisted_country_id) + + +def test_every_country_list_follows_a_country_added_to_the_one_list() -> None: + result = subprocess.run( + [sys.executable, "-c", EXTENDED_LIST_SCRIPT], + cwd=REPO, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + derived_lists = json.loads(result.stdout.strip().splitlines()[-1]) + assert derived_lists == { + "country_type": "zz", + "extracted_countries": ["us", "uk", "zz"], + "required_dependencies": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "policyengine-zz", + ], + # The installed package's other countries are not extracted. + "installed_models": { + "us": "us model", + "uk": "uk model", + "zz": "zz model", + }, + "dual_write_countries": ["uk", "us", "zz"], + "household_copy_countries": ["us", "uk", "zz"], + } + + +def test_catalog_settings_cover_exactly_the_listed_countries() -> None: + # Each listed country needs reviewed catalog settings, so adding a country + # without them fails here rather than during publication. + listed_countries = set(SUPPORTED_COUNTRY_IDS) + + assert { + path.stem for path in REGION_REFERENCE_DIRECTORY.glob("*.csv") + } == listed_countries + assert set(extraction.CURRENT_LAW_IDS) == listed_countries + assert set(extraction.METADATA_TIME_PERIODS) == listed_countries diff --git a/tests/unit/v2/household_calculation_support.py b/tests/unit/v2/household_calculation_support.py new file mode 100644 index 000000000..b28ab2ba0 --- /dev/null +++ b/tests/unit/v2/household_calculation_support.py @@ -0,0 +1,248 @@ +"""Fakes and records shared by the household calculation tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any +from uuid import UUID + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.households.types import HouseholdRead +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.types import ( + PolicyParameterValueRead, + PolicyRead, +) +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError + +HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-0000000000a1") +POLICY_ID = UUID("00000000-0000-4000-8000-0000000000b1") +CALCULATION_ID = UUID("00000000-0000-4000-8000-0000000000c1") +SIMULATION_ID = UUID("00000000-0000-4000-8000-0000000000d1") +CATALOG_VERSION_ID = UUID("00000000-0000-4000-8000-0000000000e1") +POLICYENGINE_VERSION = "6.10.0" +CREATED_AT = datetime(2026, 10, 27, 12, 0, tzinfo=UTC) +SPM_CHOICE = {"scenario": "ce_trend", "geography_kind": "national"} +HOUSEHOLD_DATA = { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}, + "memberships": { + "household": "household-1", + "family": "family-1", + "tax_unit": "tax_unit-1", + "spm_unit": "spm_unit-1", + "marital_unit": "marital_unit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {"state_name": {"2026": "CA"}}}], + "family": [{"id": "family-1", "values": {}}], + "tax_unit": [{"id": "tax_unit-1", "values": {}}], + "spm_unit": [{"id": "spm_unit-1", "values": {}}], + "marital_unit": [{"id": "marital_unit-1", "values": {}}], + "spm": SPM_CHOICE, +} +SITUATION = { + "people": {"person-1": {"age": {"2026": 40}, "employment_income": {"2026": 30000}}}, + "households": { + "household-1": {"members": ["person-1"], "state_name": {"2026": "CA"}} + }, + "families": {"family-1": {"members": ["person-1"]}}, + "tax_units": {"tax_unit-1": {"members": ["person-1"]}}, + "spm_units": {"spm_unit-1": {"members": ["person-1"]}}, + "marital_units": {"marital_unit-1": {"members": ["person-1"]}}, +} +SPM_CONFIG = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +SPM_PROVENANCE = {"schema_version": "canonical-spm-provenance-v2", "years": ["2026"]} +BUNDLE = { + "policyengine_version": POLICYENGINE_VERSION, + "core_package_version": "3.33.0", + "country_package_name": "policyengine-us", + "country_package_version": "1.450.0", + "bundle_manifest_sha256": "a" * 64, +} + + +def simulation_calculation(**changes: object) -> dict[str, Any]: + """Return the simulation service's answer for a running calculation.""" + answer: dict[str, Any] = { + "calculation_id": str(CALCULATION_ID), + "simulation_id": str(SIMULATION_ID), + "country": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "options": {}, + "catalog_version_id": str(CATALOG_VERSION_ID), + "year": 2026, + "attempt": 1, + "trigger": "initial", + "status": "running", + "created_at": CREATED_AT.isoformat(), + "dispatched_at": CREATED_AT.isoformat(), + "completed_at": None, + "result": None, + "error": None, + } + answer.update(changes) + return answer + + +def succeeded_calculation() -> dict[str, Any]: + """Return the simulation service's answer once the calculation succeeded.""" + return simulation_calculation( + status="succeeded", + completed_at=CREATED_AT.isoformat(), + result={ + "household": { + **SITUATION, + "households": { + "household-1": { + "members": ["person-1"], + "state_name": {"2026": "CA"}, + "household_net_income": {"2026": 27000.5}, + } + }, + }, + "warnings": ["The input `age_group` is deprecated and was ignored."], + "spm_config": SPM_CONFIG, + "spm_provenance": SPM_PROVENANCE, + "bundle": BUNDLE, + }, + ) + + +def household_read(*, default_year: int | None = 2026) -> HouseholdRead: + """Return the stored US household.""" + return HouseholdRead( + id=HOUSEHOLD_ID, + country_id="us", + default_year=default_year, + household_data=HOUSEHOLD_DATA, + created_at=CREATED_AT, + updated_at=CREATED_AT, + ) + + +def policy_read() -> PolicyRead: + """Return the stored US reform with one parameter value.""" + return PolicyRead( + id=POLICY_ID, + country_id="us", + tax_benefit_model_id=CATALOG_VERSION_ID, + tax_benefit_model_version_id=CATALOG_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT, + parameter_values=( + PolicyParameterValueRead( + id=POLICY_ID, + parameter_id=POLICY_ID, + parameter_name="gov.irs.credits.ctc.amount.base[0].amount", + value=3000, + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ), + ) + + +@dataclass +class FakeCountries: + """The countries table: the US with every calculation, the UK households only.""" + + rows: tuple[CountryRead, ...] = ( + CountryRead(id="uk", household_enabled=True, society_wide_enabled=False), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ) + + def list_countries(self) -> list[CountryRead]: + """Return every row.""" + return list(self.rows) + + +class FakeHouseholds: + """Stored households by country and ID.""" + + def __init__(self) -> None: + """Hold the stored US household.""" + self.households = {("us", HOUSEHOLD_ID): household_read()} + self.calls: list[tuple[str, UUID]] = [] + + def get_household(self, *, country_id: str, household_id: UUID) -> HouseholdRead: + """Return the household or raise HouseholdNotFoundError.""" + self.calls.append((country_id, household_id)) + household = self.households.get((country_id, household_id)) + if household is None: + msg = f"Household {household_id} not found." + raise HouseholdNotFoundError(msg) + return household + + +class FakePolicies: + """Stored policies by country and ID.""" + + def __init__(self) -> None: + """Hold the stored US reform.""" + self.policies = {("us", POLICY_ID): policy_read()} + + def get_policy(self, *, country_id: str, policy_id: UUID) -> PolicyRead: + """Return the policy or raise PolicyNotFoundError.""" + policy = self.policies.get((country_id, policy_id)) + if policy is None: + msg = f"Policy {policy_id} not found." + raise PolicyNotFoundError(msg) + return policy + + +class FakeCatalogs: + """Each country's current catalog version, and whether it was closed.""" + + def __init__(self) -> None: + """Start open.""" + self.closed = False + + def get_model_by_country(self, country_id: str) -> SimpleNamespace: + """Return the running catalog version's ID and PolicyEngine.py version.""" + return SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + model_version=SimpleNamespace(id=CATALOG_VERSION_ID), + country_id=country_id, + ) + + def close(self) -> None: + """Record the close.""" + self.closed = True + + +class FakeSimulation: + """The simulation service's household routes, as scripted answers.""" + + def __init__(self) -> None: + """Answer submissions with a running calculation.""" + self.submissions: list[dict[str, Any]] = [] + self.answer: dict[str, Any] = simulation_calculation() + self.read_answers: dict[UUID, dict[str, Any]] = {} + self.error: Exception | None = None + + def submit(self, submission: dict[str, Any]) -> dict[str, Any]: + """Record the submission and answer it.""" + self.submissions.append(submission) + if self.error is not None: + raise self.error + return self.answer + + def read(self, calculation_id: UUID) -> dict[str, Any] | None: + """Return the scripted calculation, or None.""" + if self.error is not None: + raise self.error + return self.read_answers.get(calculation_id) diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 4c0e6ee09..fbea9c3ef 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -40,6 +40,7 @@ V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) POST_BASELINE_TABLE_NAMES = frozenset( { + "countries", "legacy_household_mappings", "legacy_policy_mappings", "legacy_user_mappings", @@ -54,6 +55,18 @@ f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" "postgres?sslmode=require" ) +# The saved-policy rename and its parent, the Stage 12A revision. +SAVED_POLICY_RENAME_REVISION = "fa7fa1035444" +SAVED_POLICY_RENAME_PARENT_REVISION = "fbb7488ebe8a" +# The only hand-written statements in any v2 revision: the approved temporary +# view under the saved-policy table's earlier name, and its removal on +# downgrade. +SAVED_POLICY_VIEW_CREATION = ( + "CREATE VIEW user_policies AS SELECT id, created_at, updated_at, user_id, " + "policy_id, country_id, name, description, legacy_policy_id, baseline, " + "provenance FROM user_policy_associations" +) +SAVED_POLICY_VIEW_REMOVAL = "DROP VIEW user_policies" DISPOSABLE_URL = ( "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" f"{DISPOSABLE_DATABASE_NAME}" @@ -235,6 +248,10 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + "fb1a3e40d3d6", + "f5c1d9b4685b", + SAVED_POLICY_RENAME_REVISION, + SAVED_POLICY_RENAME_PARENT_REVISION, "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -495,7 +512,6 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -565,6 +581,97 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None ) in upgrade +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policy_associations", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) +# The generated downgrade restores each check in PostgreSQL's reflected form +# of `` IN ('us', 'uk')``. +US_AND_UK = "ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])" +REPLACED_COUNTRY_CHECKS = { + "ck_users_primary_country": ("users", f"primary_country::text = {US_AND_UK}"), + "ck_households_country": ("households", f"country_id::text = {US_AND_UK}"), + "ck_policies_country": ("policies", f"country_id::text = {US_AND_UK}"), + "ck_user_household_associations_country": ( + "user_household_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_policy_associations_country": ( + "user_policy_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_simulation_associations_country": ( + "user_simulation_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_report_associations_country": ( + "user_report_associations", + f"country_id::text = {US_AND_UK}", + ), +} + + +def _countries_revision_source() -> str: + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + countries_revision = next( + revision + for revision in script.walk_revisions() + if revision.down_revision == "f5c1d9b4685b" + ) + assert countries_revision.revision == "fb1a3e40d3d6" + assert countries_revision.path is not None + return Path(countries_revision.path).read_text(encoding="utf-8") + + +def test_countries_revision_inserts_us_and_uk_and_replaces_country_checks() -> None: + revision = _countries_revision_source() + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert revision.count("op.create_table(") == 1 + assert 'op.create_table(\n "countries"' in upgrade + assert "ck_countries_society_wide_requires_household" in upgrade + assert 'op.drop_table("countries")' in downgrade + # The one hand-written operation, an approved exception to generated-only + # revisions: the US and UK rows, inserted before the foreign keys that + # existing rows must satisfy. The generated downgrade drops them with the + # table. + assert revision.count("Approved hand-written data operation") == 1 + assert revision.count("Post-generation correction") == 0 + assert revision.count("op.bulk_insert(") == 1 + assert "op.execute(" not in revision + inserted_rows = upgrade[ + upgrade.index("op.bulk_insert(") : upgrade.index("op.create_foreign_key(") + ] + for country_id in ("uk", "us"): + assert f'"id": "{country_id}"' in inserted_rows + assert inserted_rows.count('"household_enabled": True') == 2 + assert inserted_rows.count('"society_wide_enabled": True') == 2 + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + key_name = f"fk_{table_name}_{column_name}_countries" + assert key_name in upgrade + assert key_name in downgrade + assert upgrade.count('ondelete="RESTRICT"') == len(COUNTRY_REFERENCE_COLUMNS) + for check_name, (table_name, expression) in REPLACED_COUNTRY_CHECKS.items(): + assert check_name in upgrade + assert f'"{table_name}"' in upgrade + assert check_name in downgrade + assert expression in downgrade + assert "op.drop_column(" not in upgrade + assert "op.add_column(" not in upgrade + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" @@ -739,3 +846,112 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "current_heads != previous_heads" in env_source assert "current_heads == script_heads" in env_source assert "validate_v2_head_schema(connection, target_metadata)" in env_source + + +def _executed_statements(body: str) -> list[str]: + """Return the SQL each ``op.execute`` call in a revision body runs.""" + return [ + "".join(re.findall(r'"([^"]*)"', call)) + for call in re.findall(r"op\.execute\((.*?)\)\n", body, flags=re.DOTALL) + ] + + +def test_saved_policy_rename_revision_renames_the_table_in_place() -> None: + """The rename keeps every saved reform: no table is created or dropped. + + Autogeneration replaces a renamed table with a new empty one; the revision + records its corrections, and every other operation is generated except the + approved temporary view under the old name, created after the rename and + dropped before the downgrade renames the table back. + """ + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + rename = script.get_revision(SAVED_POLICY_RENAME_REVISION) + revision = Path(rename.path).read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + rename_up = 'op.rename_table("user_policies", "user_policy_associations")' + rename_down = 'op.rename_table("user_policy_associations", "user_policies")' + + assert rename.down_revision == SAVED_POLICY_RENAME_PARENT_REVISION + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert rename_up in upgrade + assert rename_down in downgrade + for body in (upgrade, downgrade): + assert "op.create_table(" not in body + assert "op.drop_table(" not in body + assert "op.bulk_insert(" not in body + assert body.count("Approved hand-written exception:") == 1 + assert _executed_statements(upgrade) == [SAVED_POLICY_VIEW_CREATION] + assert _executed_statements(downgrade) == [SAVED_POLICY_VIEW_REMOVAL] + assert upgrade.index(rename_up) < upgrade.index("op.execute(") + assert downgrade.index("op.execute(") < downgrade.index(rename_down) + assert upgrade.count("Post-generation correction:") == 1 + assert downgrade.count("Post-generation correction:") == 1 + + +def test_no_other_v2_revision_runs_hand_written_sql() -> None: + """Only the saved-policy rename's approved view uses ``op.execute``.""" + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + revisions_with_sql = { + revision.revision + for revision in script.walk_revisions() + if "op.execute(" in Path(revision.path).read_text(encoding="utf-8") + } + + assert revisions_with_sql == {SAVED_POLICY_RENAME_REVISION} + + +def test_catalog_format_2_revision_adds_only_nullable_columns() -> None: + """The revision after the saved-policy rename adds catalog format 2's columns. + + It is generated, adds each column as nullable so versions published earlier + keep them empty, drops them on downgrade, and writes no data. + """ + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + children = [ + revision + for revision in script.walk_revisions() + if revision.down_revision == SAVED_POLICY_RENAME_REVISION + ] + assert len(children) == 1 + revision = Path(children[0].path).read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + added_columns = { + "tax_benefit_model_versions": ("catalog_format", "entities", "basic_inputs"), + "variables": ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + ), + "parameters": ("economy", "household"), + } + column_count = sum(len(columns) for columns in added_columns.values()) + assert upgrade.count("op.add_column(") == column_count + assert upgrade.count("nullable=True") == column_count + assert downgrade.count("op.drop_column(") == column_count + for table_name, columns in added_columns.items(): + for column_name in columns: + assert f'"{column_name}"' in upgrade + assert f'op.drop_column("{table_name}", "{column_name}")' in downgrade + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert "server_default" not in body diff --git a/tests/unit/v2/test_catalog_extraction.py b/tests/unit/v2/test_catalog_extraction.py index f88319cd8..66ace2ced 100644 --- a/tests/unit/v2/test_catalog_extraction.py +++ b/tests/unit/v2/test_catalog_extraction.py @@ -3,7 +3,7 @@ from __future__ import annotations import ast -import json +from dataclasses import replace from datetime import UTC, datetime from importlib.metadata import PackageNotFoundError from pathlib import Path @@ -22,6 +22,7 @@ bundle, installed_version, normalized_catalog, + region_references, source_models, ) @@ -39,9 +40,12 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: assert {dataset.name for dataset in us.datasets} == { "populace_us_2024", - "populace_us_ca_2024", + "populace_us_2024_acs_local", + } + assert {dataset.name for dataset in uk.datasets} == { + "microcosm_uk_2024_25", + "microcosm_uk_2024_25_dense", } - assert {dataset.name for dataset in uk.datasets} == {"microcosm_uk_2024_25"} assert all( not dataset.is_output_dataset for dataset in (*us.datasets, *uk.datasets) ) @@ -52,74 +56,138 @@ def test_extracts_canonical_policyengine_catalog_and_dataset_defaults() -> None: region.code: us_datasets[region.default_dataset_id] for region in us.regions } assert us_defaults == { - "place/CA-44000": "populace_us_2024", - "state/ca": "populace_us_ca_2024", + "congressional_district/CA-01": "populace_us_2024_acs_local", + "state/ca": "populace_us_2024_acs_local", "us": "populace_us_2024", } + assert us.fallback_summaries == () assert [ - (summary.region_type, summary.count) for summary in us.fallback_summaries + (summary.region_type, summary.count) for summary in us.omitted_region_summaries ] == [("place", 1)] - assert all(region.default_dataset_id == uk.datasets[0].id for region in uk.regions) - + assert uk.omitted_region_summaries == () + uk_datasets = {dataset.id: dataset.name for dataset in uk.datasets} + assert { + region.code: uk_datasets[region.default_dataset_id] for region in uk.regions + } == { + "uk": "microcosm_uk_2024_25", + "country/england": "microcosm_uk_2024_25", + "country/northern_ireland": "microcosm_uk_2024_25", + "constituency/E14001063": "microcosm_uk_2024_25_dense", + "local_authority/N09000011": "microcosm_uk_2024_25_dense", + } + # UK nations filter the national dataset by design, and constituencies and + # local authorities use the dataset the release routes them to. + assert uk.fallback_summaries == () -UK_DENSE_URI = "hf://policyengine/uk/microcosm_uk_2024_25_dense.h5@fixture" +def test_regions_take_labels_from_references_and_filters_from_scoping_rules() -> None: + catalog = normalized_catalog() + regions = { + region.code: ( + region.label, + region.region_type, + region.parent_code, + region.requires_filter, + region.filter_strategy, + region.filter_field, + region.filter_value, + region.state_code, + region.state_name, + ) + for country_id in ("us", "uk") + for region in catalog.country(country_id).regions + } -def _uk_area_region(code: str, region_type: str, filter_field: str) -> SimpleNamespace: - """Return a UK area region routed to the local-area dataset, as .py 6.10 does.""" - return SimpleNamespace( - code=code, - label=code.split("/", maxsplit=1)[1], - region_type=region_type, - parent_code="uk", - dataset_path=UK_DENSE_URI, - scoping_strategy=SimpleNamespace( - strategy_type="row_filter", - variable_name=filter_field, - variable_value=code.split("/", maxsplit=1)[1], - additional_filters={}, + assert regions == { + "us": ("United States", "national", None, False, None, None, None, None, None), + "state/ca": ( + "California", + "state", + "us", + True, + "row_filter", + "state_fips", + "6", + "CA", + "California", ), - requires_filter=True, - state_code=None, - state_name=None, - ) + "congressional_district/CA-01": ( + "California's 1st congressional district", + "congressional_district", + "state/ca", + True, + "row_filter", + "congressional_district_geoid", + "601", + "CA", + "California", + ), + "uk": ("United Kingdom", "national", None, False, None, None, None, None, None), + "country/england": ( + "England", + "country", + "uk", + True, + "row_filter", + "region", + '["NORTH_EAST","LONDON"]', + None, + None, + ), + "constituency/E14001063": ( + "Aldershot", + "constituency", + "country/england", + True, + "row_filter", + "constituency_code_oa", + "E14001063", + None, + None, + ), + "country/northern_ireland": ( + "Northern Ireland", + "country", + "uk", + True, + "row_filter", + "region", + "NORTHERN_IRELAND", + None, + None, + ), + "local_authority/N09000011": ( + "Ards and North Down", + "local_authority", + "country/northern_ireland", + True, + "row_filter", + "la_code_oa", + "N09000011", + None, + None, + ), + } -def test_uk_area_regions_use_the_dataset_their_path_names() -> None: +def test_sub_national_regions_without_a_release_dataset_use_the_default() -> None: models = source_models() - models["uk"].region_registry.extend( - [ - _uk_area_region( - "constituency/E14001063", "constituency", "constituency_code_oa" - ), - _uk_area_region( - "local_authority/E07000092", "local_authority", "la_code_oa" - ), - ] - ) + del models["us"].release_manifest.region_datasets["state"] + del models["us"].release_manifest.region_datasets["congressional_district"] - uk = extract_catalog( + us = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, installed_version=installed_version, - ).country("uk") + ).country("us") - assert {dataset.name for dataset in uk.datasets} == { - "microcosm_uk_2024_25", - "microcosm_uk_2024_25_dense", - } - uk_datasets = {dataset.id: dataset.name for dataset in uk.datasets} - assert { - region.code: uk_datasets[region.default_dataset_id] for region in uk.regions - } == { - "uk": "microcosm_uk_2024_25", - "country/england": "microcosm_uk_2024_25", - "constituency/E14001063": "microcosm_uk_2024_25_dense", - "local_authority/E07000092": "microcosm_uk_2024_25_dense", - } - # A UK country filters the national dataset by design; it is no fallback. - assert uk.fallback_summaries == () + assert {dataset.name for dataset in us.datasets} == {"populace_us_2024"} + assert all(region.default_dataset_id == us.datasets[0].id for region in us.regions) + assert [ + (summary.region_type, summary.count) for summary in us.fallback_summaries + ] == [("congressional_district", 1), ("state", 1)] def test_durable_ids_are_deterministic_and_stable_at_the_intended_scope() -> None: @@ -179,62 +247,56 @@ def item(self): assert normalize_json_value(float("-inf")) == "-Infinity" -@pytest.mark.parametrize("value", [("NORTH_EAST", "LONDON"), ["NORTH_EAST", "LONDON"]]) -def test_serializes_region_membership_filters_in_the_existing_text_field( - value: object, -) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value - - catalog = extract_catalog( - bundle=bundle(), - policyengine_version=POLICYENGINE_VERSION, - models=models, - installed_version=installed_version, - ) - england = next( - region - for region in catalog.country("uk").regions - if region.code == "country/england" +@pytest.mark.parametrize("value", ['["NORTH_EAST","LONDON"]', "ENGLAND", "6"]) +def test_keeps_reference_filter_values_as_text(value: str) -> None: + references = region_references() + england = references["uk"][1] + references["uk"] = ( + references["uk"][0], + replace(england, filter_value=value), + references["uk"][2], ) - assert isinstance(england.filter_value, str) - assert json.loads(england.filter_value) == ["NORTH_EAST", "LONDON"] - - -@pytest.mark.parametrize("value", ["ENGLAND", 6, True, 1.5]) -def test_preserves_existing_scalar_region_filter_serialization(value: object) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value - catalog = extract_catalog( + region_references=references, bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, - models=models, + models=source_models(), installed_version=installed_version, ) - england = next( + england_record = next( region for region in catalog.country("uk").regions if region.code == "country/england" ) - assert england.filter_value == str(value) + assert england_record.filter_value == value @pytest.mark.parametrize( "value", - [[], [["LONDON"]], [None], [object()], [float("nan")], ["x" * 256], "x" * 256], + ["[]", '[["LONDON"]]', "[null]", '[{"a": 1}]', "[NaN]", "[", '{"a": 1}'], ) -def test_rejects_invalid_or_oversized_region_membership_values(value: object) -> None: - models = source_models() - models["uk"].region_registry[1].scoping_strategy.variable_value = value +def test_rejects_filter_value_lists_that_are_not_flat_json_scalars( + value: str, +) -> None: + references = region_references() + england = references["uk"][1] + references["uk"] = ( + references["uk"][0], + replace(england, filter_value=value), + references["uk"][2], + ) - with pytest.raises(CatalogExtractionError, match="filter value"): + with pytest.raises( + CatalogExtractionError, + match=r"^region 'country/england' has an unsupported filter value$", + ): extract_catalog( + region_references=references, bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, - models=models, + models=source_models(), installed_version=installed_version, ) @@ -250,6 +312,7 @@ def test_rejects_missing_or_mismatched_manifest_dependencies() -> None: del missing["packages"]["policyengine-core"] with pytest.raises(CatalogExtractionError, match="omits policyengine-core"): extract_catalog( + region_references=region_references(), bundle=missing, policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -258,6 +321,7 @@ def test_rejects_missing_or_mismatched_manifest_dependencies() -> None: with pytest.raises(CatalogExtractionError, match="does not match"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -271,6 +335,7 @@ def absent(_name: str) -> str: with pytest.raises(CatalogExtractionError, match="is absent"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=source_models(), @@ -282,6 +347,7 @@ def absent(_name: str) -> str: def test_rejects_placeholder_policyengine_versions(version: str) -> None: with pytest.raises(CatalogExtractionError, match="placeholder"): extract_catalog( + region_references=region_references(), bundle=bundle(policyengine_version=version), policyengine_version=version, models=source_models(policyengine_version=version), @@ -294,16 +360,74 @@ def test_rejects_incomplete_public_model_catalogs() -> None: del models["uk"] with pytest.raises(CatalogExtractionError, match="absent for: uk"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, installed_version=installed_version, ) + +def _use_another_certified_default(model: SimpleNamespace, dataset_name: str) -> None: + """Make the release default to another dataset it certifies, for every region.""" + manifest = model.release_manifest + manifest.default_dataset = dataset_name + manifest.datasets[dataset_name] = SimpleNamespace( + path=f"{dataset_name}.h5", + revision="fixture", + sha256="0" * 64, + repo_id=None, + repo_type=None, + ) + for template in manifest.region_datasets.values(): + template.path_template = f"{dataset_name}.h5" + + +def test_each_country_uses_the_default_dataset_its_release_names() -> None: + """The release's own default is used; the Public API fixes none of its own.""" models = source_models() - models["us"].release_manifest.default_dataset = "unreviewed_us_2025" - with pytest.raises(CatalogExtractionError, match="reviewed selection"): + _use_another_certified_default(models["us"], "populace_us_2025") + _use_another_certified_default(models["uk"], "microcosm_uk_2025_26") + + catalog = extract_catalog( + region_references=region_references(), + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) + + for country_id, default_dataset in ( + ("us", "populace_us_2025"), + ("uk", "microcosm_uk_2025_26"), + ): + country = catalog.country(country_id) + dataset_names = {dataset.id: dataset.name for dataset in country.datasets} + national = next( + region for region in country.regions if region.region_type == "national" + ) + assert dataset_names[national.default_dataset_id] == default_dataset + assert set(dataset_names.values()) == {default_dataset} + + +@pytest.mark.parametrize( + ("default_dataset", "message"), + [ + (None, "^us release manifest names no default dataset$"), + ("", "^us release manifest names no default dataset$"), + ("populace_us_2025", "^us release does not certify populace_us_2025$"), + ], +) +def test_a_missing_or_uncertified_release_default_fails( + default_dataset: str | None, + message: str, +) -> None: + models = source_models() + models["us"].release_manifest.default_dataset = default_dataset + + with pytest.raises(CatalogExtractionError, match=message): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -321,6 +445,7 @@ def test_rejects_inconsistent_country_model_identity(identity_source: str) -> No with pytest.raises(CatalogExtractionError, match="identity does not match"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -335,6 +460,7 @@ def test_ignores_only_the_unnamed_structural_parameter_root() -> None: ) catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -352,6 +478,7 @@ def test_uses_and_validates_public_name_indexed_mappings() -> None: models["uk"].variables_by_name["wrong_name"] = variable with pytest.raises(CatalogExtractionError, match="does not match record name"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -359,33 +486,25 @@ def test_uses_and_validates_public_name_indexed_mappings() -> None: ) -def test_rejects_duplicate_keys_invalid_intervals_and_unknown_region_types() -> None: +def test_rejects_unindexed_mappings_and_invalid_parameter_intervals() -> None: duplicate = source_models() duplicate["us"].parameters_by_name = [] with pytest.raises(CatalogExtractionError, match="name-indexed mapping"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=duplicate, installed_version=installed_version, ) - unknown_region = source_models() - unknown_region["us"].region_registry[1].region_type = "province" - with pytest.raises(CatalogExtractionError, match="unsupported type"): - extract_catalog( - bundle=bundle(), - policyengine_version=POLICYENGINE_VERSION, - models=unknown_region, - installed_version=installed_version, - ) - invalid_interval = source_models() invalid_interval["us"].parameters_by_name["gov.example.rate"].parameter_values[ 0 ].end_date = datetime(2026, 1, 1, tzinfo=UTC) with pytest.raises(CatalogExtractionError, match="canonical inclusive intervals"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=invalid_interval, @@ -399,6 +518,7 @@ def test_rejects_parameter_values_that_are_not_oldest_to_newest() -> None: parameter.parameter_values.reverse() with pytest.raises(CatalogExtractionError, match="oldest to newest"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -419,6 +539,7 @@ def test_collapses_equal_parameter_values_at_the_same_effective_date() -> None: ) catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -446,6 +567,7 @@ def test_rejects_conflicting_parameter_values_at_the_same_effective_date() -> No with pytest.raises(CatalogExtractionError, match="conflicting values"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -467,6 +589,7 @@ def test_rejects_repeated_parameter_values_with_inconsistent_intervals() -> None with pytest.raises(CatalogExtractionError, match="inconsistent intervals"): extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -491,6 +614,7 @@ def test_preserves_consecutive_day_parameter_value_intervals() -> None: ] catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, diff --git a/tests/unit/v2/test_catalog_extraction_failures.py b/tests/unit/v2/test_catalog_extraction_failures.py index fa46c3f80..b09bff9fd 100644 --- a/tests/unit/v2/test_catalog_extraction_failures.py +++ b/tests/unit/v2/test_catalog_extraction_failures.py @@ -3,9 +3,10 @@ from __future__ import annotations import math +from dataclasses import replace from datetime import date from enum import Enum -from typing import TYPE_CHECKING +from types import SimpleNamespace import pytest @@ -14,19 +15,19 @@ extract_catalog, normalize_json_value, ) +from policyengine_api.data.v2.catalog.region_references import RegionReference from tests.fixtures.v2_catalog import ( POLICYENGINE_VERSION, bundle, installed_version, + region_references, source_models, ) -if TYPE_CHECKING: - from types import SimpleNamespace - def _extract(models: dict[str, SimpleNamespace]) -> None: extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, @@ -46,97 +47,276 @@ def _break_manifest_version(models: dict[str, SimpleNamespace]) -> None: models["us"].release_manifest.policyengine_version = "9.9.9" -def _remove_region_registry(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry = None +@pytest.mark.parametrize( + ("break_models", "message"), + [ + ( + _break_package_version, + "^us public model does not match the PolicyEngine.py manifest$", + ), + ( + _break_manifest_country, + "^us public model has an inconsistent release manifest$", + ), + ( + _break_manifest_version, + "^us public model was not certified by the installed PolicyEngine.py " + "version$", + ), + ], +) +def test_inconsistent_public_models_fail_with_a_specific_message( + break_models, + message: str, +) -> None: + models = source_models() + break_models(models) + with pytest.raises(CatalogExtractionError, match=message): + _extract(models) -def _mislabel_region_registry(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry.country_id = "uk" +References = dict[str, tuple[RegionReference, ...]] -def _duplicate_region_codes(models: dict[str, SimpleNamespace]) -> None: - registry = models["us"].region_registry - registry.append(registry[1]) + +def _replace_reference( + references: References, country_id: str, code: str, **changes: str | None +) -> None: + references[country_id] = tuple( + replace(reference, **changes) if reference.code == code else reference + for reference in references[country_id] + ) + + +def _point_national_region_at_another_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["uk"].release_manifest.region_datasets[ + "national" + ].path_template = "frs_2024_25.h5" + models["uk"].release_manifest.datasets["frs_2024_25"] = SimpleNamespace( + path="frs_2024_25.h5", revision="fixture", repo_id=None, repo_type=None + ) + + +def _name_an_uncertified_region_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets["state"].path_template = "missing.h5" + + +def _name_a_per_region_dataset_template( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets[ + "state" + ].path_template = "populace_{state_code}.h5" -def _remove_national_region(models: dict[str, SimpleNamespace]) -> None: - registry = models["us"].region_registry - del registry[0] +def _stop_routing_constituencies( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["uk"].release_manifest.region_datasets["constituency"] -def _give_national_region_another_dataset(models: dict[str, SimpleNamespace]) -> None: - models["uk"].region_registry[0].dataset_path = "hf://policyengine/uk/other.h5" +def _stop_routing_local_authorities( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["uk"].release_manifest.region_datasets["local_authority"] -def _give_filter_an_unsupported_value(models: dict[str, SimpleNamespace]) -> None: - models["us"].region_registry[1].scoping_strategy.variable_value = {"state": "CA"} +def _drop_a_filter_value( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference(references, "us", "state/ca", filter_value=None) -def _give_filter_additional_filters(models: dict[str, SimpleNamespace]) -> None: - models["uk"].region_registry[1].scoping_strategy.additional_filters = { - "region": "LONDON" - } +def _give_the_national_region_a_filter_value( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference(references, "us", "us", filter_value="0") + + +def _place_a_district_under_a_place( + _models: dict[str, SimpleNamespace], references: References +) -> None: + _replace_reference( + references, + "us", + "congressional_district/CA-01", + parent_code="place/CA-44000", + ) + + +def _remove_a_country_from_the_references( + _models: dict[str, SimpleNamespace], references: References +) -> None: + del references["uk"] + + +def _remove_the_dataset_map( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.datasets = None + + +def _remove_the_release_default_dataset( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["us"].release_manifest.datasets["populace_us_2024"] + + +def _remove_the_region_dataset_map( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets = None + + +def _remove_a_region_dataset_path( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.region_datasets["state"].path_template = None + + +def _remove_an_artifact_path( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.datasets["populace_us_2024"].path = None + + +def _remove_every_artifact_revision( + models: dict[str, SimpleNamespace], _references: References +) -> None: + manifest = models["us"].release_manifest + manifest.datasets["populace_us_2024"].revision = None + manifest.data_package.release_manifest_revision = None + manifest.data_package.version = None + + +def _drop_the_region_dataset_attribute( + models: dict[str, SimpleNamespace], _references: References +) -> None: + del models["us"].release_manifest.region_datasets + + +def _remove_every_artifact_repository( + models: dict[str, SimpleNamespace], _references: References +) -> None: + models["us"].release_manifest.data_package.repo_id = None @pytest.mark.parametrize( - ("break_models", "message"), + ("break_sources", "message"), [ + (_remove_the_dataset_map, "^us release manifest has no dataset map$"), ( - _break_package_version, - "^us public model does not match the PolicyEngine.py manifest$", + _drop_the_region_dataset_attribute, + "^us release manifest lacks datasets, region_datasets or data_package$", ), ( - _break_manifest_country, - "^us public model has an inconsistent release manifest$", + _remove_the_release_default_dataset, + "^us release does not certify populace_us_2024$", ), ( - _break_manifest_version, - "^us public model was not certified by the installed PolicyEngine.py " - "version$", + _remove_the_region_dataset_map, + "^us release manifest has no region dataset map$", + ), + ( + _remove_a_region_dataset_path, + "^us state region dataset must be a non-empty string$", + ), + ( + _remove_an_artifact_path, + "^us release artifact populace_us_2024 path must be a non-empty string$", + ), + ( + _remove_every_artifact_revision, + "^us release artifact populace_us_2024 revision must be a non-empty " + "string$", + ), + ( + _remove_every_artifact_repository, + "^us release artifact populace_us_2024 repository must be a non-empty " + "string$", + ), + ( + _point_national_region_at_another_dataset, + "^uk release routes national regions to frs_2024_25, not to its default " + "dataset microcosm_uk_2024_25$", + ), + ( + _name_an_uncertified_region_dataset, + "^us release names no certified dataset at path 'missing.h5' for state " + "regions$", + ), + ( + _name_a_per_region_dataset_template, + r"^us state regions name a per-region dataset template " + r"'populace_\{state_code\}.h5', which the catalog does not support$", + ), + ( + _stop_routing_constituencies, + "^the uk release routes no dataset to constituency regions, so region " + "'constituency/E14001063' cannot select its households$", ), - (_remove_region_registry, "^us public region registry is absent$"), - (_mislabel_region_registry, "^us public region registry is absent$"), ( - _duplicate_region_codes, - r"^duplicate region natural keys: \['state/ca'\]$", + _stop_routing_local_authorities, + "^the uk release routes no dataset to local_authority regions, so region " + "'local_authority/N09000011' cannot select its households$", ), - (_remove_national_region, "^us national region is absent$"), + (_drop_a_filter_value, "^region 'state/ca' has no filter value$"), ( - _give_national_region_another_dataset, - "^uk national region does not use the reviewed dataset$", + _give_the_national_region_a_filter_value, + "^region 'us' has a filter value, but a national region selects every " + "household$", ), ( - _give_filter_an_unsupported_value, - "^region 'state/ca' has an unsupported filter value$", + _place_a_district_under_a_place, + "^region 'congressional_district/CA-01' cannot be published because its " + "parent 'place/CA-44000' is left out$", ), ( - _give_filter_additional_filters, - "^region 'country/england' has unsupported additional filters$", + _remove_a_country_from_the_references, + "^region references are absent for: uk$", ), ], ) -def test_inconsistent_public_models_fail_with_a_specific_message( - break_models, +def test_inconsistent_regions_fail_with_a_specific_message( + break_sources, message: str, ) -> None: models = source_models() - break_models(models) + references = region_references() + break_sources(models, references) with pytest.raises(CatalogExtractionError, match=message): - _extract(models) + extract_catalog( + region_references=references, + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) -def test_national_region_without_a_dataset_path_uses_the_reviewed_dataset() -> None: +def test_national_region_without_a_release_dataset_uses_the_reviewed_dataset() -> None: models = source_models() - models["us"].region_registry[0].dataset_path = None + del models["us"].release_manifest.region_datasets["national"] - extract_catalog( + catalog = extract_catalog( + region_references=region_references(), bundle=bundle(), policyengine_version=POLICYENGINE_VERSION, models=models, installed_version=installed_version, ) + us = catalog.country("us") + national = next(region for region in us.regions if region.code == "us") + datasets = {dataset.id: dataset.name for dataset in us.datasets} + assert datasets[national.default_dataset_id] == "populace_us_2024" + assert us.fallback_summaries == () + class _ScalarWrapper: """A numpy-like scalar whose item() returns a plain Python value.""" diff --git a/tests/unit/v2/test_catalog_initialization.py b/tests/unit/v2/test_catalog_initialization.py index e77681fcd..22f01b72b 100644 --- a/tests/unit/v2/test_catalog_initialization.py +++ b/tests/unit/v2/test_catalog_initialization.py @@ -8,11 +8,16 @@ import pytest from policyengine_api.data.v2.catalog import initialization +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError from policyengine_api.data.v2.catalog.publication import PublicationEvidence +from policyengine_api.data.v2.catalog.region_data_verification import ( + RegionDataVerificationError, +) from policyengine_api.data.v2.settings import ( V2_DATA_WRITE_DATABASE_URL, V2_SUPABASE_ENVIRONMENT, V2_SUPABASE_PROJECT_REF, + V2ConfigurationError, ) from tests.fixtures.v2_catalog import ( DEPENDENCY_VERSIONS, @@ -29,6 +34,7 @@ V2_SUPABASE_ENVIRONMENT: "test-foundation", "V2_RUNTIME_DATABASE_URL": "invalid-runtime-value", "V2_MIGRATION_DATABASE_URL": "invalid-migration-value", + "SIMULATION_ENTRYPOINT_URL": "https://simulation.example", } @@ -65,22 +71,159 @@ def publish(selected_engine, catalog): observed["version"] = catalog.policyengine_version return _evidence() + def verify(selected_engine, catalog, environ): + assert "engine" not in observed + observed["verified"] = ( + selected_engine, + catalog.policyengine_version, + environ, + ) + + def record_spm(selected_engine, catalog, environ): + assert "engine" in observed + observed["spm"] = (selected_engine, catalog.policyengine_version, environ) + result = initialization.initialize_catalog( ENVIRONMENT, extractor=normalized_catalog, engine_builder=build, - publisher=publish, + steps=initialization.PublicationSteps( + verify_region_data=verify, + publish=publish, + record_spm_availability=record_spm, + ), ) assert result == _evidence() assert observed == { "username": "data-writer", + "verified": (engine, POLICYENGINE_VERSION, ENVIRONMENT), "engine": engine, "version": POLICYENGINE_VERSION, + "spm": (engine, POLICYENGINE_VERSION, ENVIRONMENT), } assert engine.disposed +def test_region_data_failure_publishes_nothing_and_disposes_engine() -> None: + engine = FakeEngine() + + def reject(_engine, _catalog, _environ): + msg = "the certified dataset d has no household column c" + raise RegionDataVerificationError(msg) + + def refuse_publication(_engine, _catalog): + msg = "publication must not start" + raise AssertionError(msg) + + with pytest.raises(RegionDataVerificationError, match="no household column"): + initialization.initialize_catalog( + ENVIRONMENT, + extractor=normalized_catalog, + engine_builder=lambda _settings: engine, + steps=initialization.PublicationSteps( + verify_region_data=reject, publish=refuse_publication + ), + ) + assert engine.disposed + + +def test_installed_verification_reads_artifacts_with_the_hugging_face_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed = {} + + class RecordingInspector: + def __init__(self, *, token): + observed["token"] = token + + def record(catalog, inspector, *, countries): + observed["call"] = (catalog.policyengine_version, type(inspector), countries) + + monkeypatch.setattr(initialization, "HuggingFaceDataInspector", RecordingInspector) + monkeypatch.setattr(initialization, "verify_region_data", record) + + initialization.verify_installed_region_data( + normalized_catalog(), ("uk",), {"HUGGING_FACE_TOKEN": "read-token"} + ) + assert observed == { + "token": "read-token", + "call": (POLICYENGINE_VERSION, RecordingInspector, ("uk",)), + } + + initialization.verify_installed_region_data(normalized_catalog(), ("us",), {}) + assert observed == { + "token": None, + "call": (POLICYENGINE_VERSION, RecordingInspector, ("us",)), + } + + +@pytest.mark.parametrize("environ", [{}, {"HUGGING_FACE_TOKEN": ""}]) +def test_uk_verification_requires_the_hugging_face_token_before_reading( + monkeypatch: pytest.MonkeyPatch, + environ: dict[str, str], +) -> None: + def refuse(*_args, **_kwargs): + msg = "no artifact may be read without the token" + raise AssertionError(msg) + + monkeypatch.setattr(initialization, "HuggingFaceDataInspector", refuse) + monkeypatch.setattr(initialization, "verify_region_data", refuse) + + with pytest.raises( + V2ConfigurationError, + match=r"^HUGGING_FACE_TOKEN must be set to read the certified uk data, " + r"whose repository is private$", + ): + initialization.verify_installed_region_data( + normalized_catalog(), ("us", "uk"), environ + ) + + +def test_default_verification_checks_only_countries_not_yet_published( + monkeypatch: pytest.MonkeyPatch, +) -> None: + engine = FakeEngine() + catalog = normalized_catalog() + observed = {} + + def find_unpublished(selected_engine, selected_catalog): + observed["found"] = (selected_engine, selected_catalog) + return ("uk",) + + def verify(selected_catalog, countries, environ): + observed["verified"] = (selected_catalog, countries, environ) + + monkeypatch.setattr(initialization, "unpublished_countries", find_unpublished) + monkeypatch.setattr(initialization, "verify_installed_region_data", verify) + + initialization.verify_unpublished_region_data(engine, catalog, ENVIRONMENT) + + assert observed == { + "found": (engine, catalog), + "verified": (catalog, ("uk",), ENVIRONMENT), + } + + +def test_main_reports_a_region_data_failure_with_its_message( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def fail(): + msg = ( + "the certified dataset d has no household column c, which 1 uk region " + "filters on (constituency/x)" + ) + raise RegionDataVerificationError(msg) + + monkeypatch.setattr(initialization, "initialize_catalog", fail) + + assert initialization.main() == 1 + failure = capsys.readouterr() + assert '"type": "RegionDataVerificationError"' in failure.err + assert "which 1 uk region filters on" in failure.err + + def test_extraction_failure_opens_no_engine() -> None: def fail_extraction(): msg = "extraction failed" @@ -142,3 +285,42 @@ def test_initializer_is_not_imported_by_application_startup_modules() -> None: module.startswith("policyengine_api.data.v2.catalog.initialization") for module in imported ) + + +def test_region_data_errors_are_not_reported_as_extraction_errors() -> None: + # Extraction errors are raised before any database access; verification + # runs after reading the stored versions and reads certified data over the + # network. + assert not issubclass(RegionDataVerificationError, CatalogExtractionError) + + +def test_main_reports_a_missing_hugging_face_token_with_its_message( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def verify_without_token(): + initialization.verify_installed_region_data(normalized_catalog(), ("uk",), {}) + + monkeypatch.setattr(initialization, "initialize_catalog", verify_without_token) + + assert initialization.main() == 1 + failure = capsys.readouterr() + assert '"type": "V2ConfigurationError"' in failure.err + assert "HUGGING_FACE_TOKEN must be set to read the certified uk data" in ( + failure.err + ) + + +def test_a_missing_simulation_service_url_stops_before_extraction() -> None: + def refuse_extraction(): + msg = "extraction must not start" + raise AssertionError(msg) + + environment = { + name: value + for name, value in ENVIRONMENT.items() + if name != "SIMULATION_ENTRYPOINT_URL" + } + + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + initialization.initialize_catalog(environment, extractor=refuse_extraction) diff --git a/tests/unit/v2/test_catalog_metadata_fields.py b/tests/unit/v2/test_catalog_metadata_fields.py new file mode 100644 index 000000000..6b8cc3ddc --- /dev/null +++ b/tests/unit/v2/test_catalog_metadata_fields.py @@ -0,0 +1,490 @@ +"""Catalog format 2: the metadata v3 reads, taken from .py's public records.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import pytest + +from policyengine_api.data.v2.catalog.extraction import ( + CatalogExtractionError, + extract_catalog, +) +from policyengine_api.data.v2.catalog.records import ( + CATALOG_FORMAT, + EntityRecord, + EnumValueRecord, + NormalizedCatalog, +) +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + bundle, + installed_version, + normalized_catalog, + region_references, + source_models, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + +def _extract(models: dict[str, SimpleNamespace]) -> NormalizedCatalog: + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + region_references=region_references(), + ) + + +def _add_variable(model: SimpleNamespace, **fields: object) -> None: + variable = SimpleNamespace( + label=None, + entity="person", + description=None, + data_type=float, + possible_values=None, + default_value=0.0, + adds=None, + subtracts=None, + is_input_variable=False, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit=None, + definition_period="year", + ) + for name, value in fields.items(): + setattr(variable, name, value) + model.variables.append(variable) + model.variables_by_name[variable.name] = variable + + +def _add_parameter(model: SimpleNamespace, name: str, *, value: object) -> None: + parameter = SimpleNamespace( + name=name, + label=None, + description=None, + data_type=type(value), + unit=None, + economy=True, + household=True, + parameter_values=[ + SimpleNamespace( + value=value, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ], + ) + model.parameters.append(parameter) + model.parameters_by_name[name] = parameter + + +def _add_parameter_node(model: SimpleNamespace, name: str, label: str | None) -> None: + node = SimpleNamespace(name=name, label=label, description=None) + model.parameter_nodes.append(node) + model.parameter_nodes_by_name[name] = node + + +def test_catalog_format_2_publishes_entities_and_basic_inputs() -> None: + catalog = normalized_catalog() + + for country_id in ("us", "uk"): + model_version = catalog.country(country_id).model_version + assert model_version.catalog_format == CATALOG_FORMAT == 2 + assert model_version.entities == ( + EntityRecord(key="person", plural="people", is_person=True), + EntityRecord(key="household", plural="households", is_person=False), + ) + assert model_version.basic_inputs == ("employment_income",) + + +def test_variables_carry_flags_module_unit_period_and_unset_parts() -> None: + variable = normalized_catalog().country("us").variables[0] + + assert variable.name == "employment_income" + assert variable.is_input_variable is True + assert variable.hidden_input is False + assert variable.module_name == "household.income.employment_income" + assert variable.adds_parameter_path is None + assert variable.subtracts_parameter_path is None + assert variable.enum_values is None + assert variable.unit == "currency-USD" + assert variable.definition_period == "year" + + +def test_enum_values_keep_their_declared_labels_in_order() -> None: + models = source_models() + _add_variable( + models["us"], + name="filing_status", + data_type="Enum", + possible_values=["SINGLE", "JOINT", "EXEMPT"], + default_value="SINGLE", + is_input_variable=True, + enum_values=[ + SimpleNamespace(value="SINGLE", label="Single"), + SimpleNamespace(value="JOINT", label=2), + SimpleNamespace(value="EXEMPT", label=None), + ], + ) + + variables = { + variable.name: variable for variable in _extract(models).country("us").variables + } + + assert variables["filing_status"].enum_values == ( + EnumValueRecord(value="SINGLE", label="Single"), + EnumValueRecord(value="JOINT", label=2), + EnumValueRecord(value="EXEMPT", label=None), + ) + assert isinstance(variables["filing_status"].enum_values[1].label, int) + + +def test_parameter_named_parts_record_the_parameter_path_beside_the_list() -> None: + models = source_models() + _add_parameter( + models["us"], + "gov.example.countable_income", + value=["employment_income"], + ) + _add_parameter(models["us"], "gov.example.deductions", value=["employment_income"]) + _add_variable( + models["us"], + name="countable_income", + adds=["employment_income"], + subtracts=["employment_income"], + adds_parameter_path="gov.example.countable_income", + subtracts_parameter_path="gov.example.deductions", + ) + + variables = { + variable.name: variable for variable in _extract(models).country("us").variables + } + + countable_income = variables["countable_income"] + assert countable_income.adds_parameter_path == "gov.example.countable_income" + assert countable_income.subtracts_parameter_path == "gov.example.deductions" + assert countable_income.adds == ["employment_income"] + assert countable_income.subtracts == ["employment_income"] + + +def test_parameters_carry_their_economy_and_household_flags() -> None: + parameter = normalized_catalog().country("us").parameters[0] + + assert parameter.name == "gov.example.rate" + assert (parameter.economy, parameter.household) == (True, False) + + +def test_bracket_schedules_are_stored_as_nodes_with_their_stored_label() -> None: + models = source_models() + _add_parameter_node(models["uk"], "gov.example.rates", "Example rates") + _add_parameter_node(models["uk"], "gov.example.rates[0]", None) + _add_parameter_node(models["uk"], "gov.example.thresholds", None) + _add_parameter_node(models["uk"], "gov.example.thresholds[0]", None) + _add_parameter(models["uk"], "gov.example.rates[0].rate", value=0.2) + _add_parameter(models["uk"], "gov.example.thresholds[0].threshold", value=0) + + nodes = {node.name: node for node in _extract(models).country("uk").parameter_nodes} + + assert nodes["gov.example.rates"].label == "Example rates" + assert nodes["gov.example.thresholds"].label is None + assert "gov.example.rates[0]" in nodes + + +def _remove_entities(models: dict[str, SimpleNamespace]) -> None: + del models["us"].entities + + +def _make_entity_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities[0].is_person = "yes" + + +def _remove_person_entity(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities = models["us"].entities[1:] + + +def _repeat_entity_key(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities.append( + SimpleNamespace(key="household", plural="homes", is_person=False) + ) + + +def _remove_basic_inputs(models: dict[str, SimpleNamespace]) -> None: + del models["us"].basic_inputs + + +def _make_entities_null(models: dict[str, SimpleNamespace]) -> None: + models["us"].entities = None + + +def _make_basic_inputs_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].basic_inputs = "employment_income" + + +def _remove_variable_enum_values(models: dict[str, SimpleNamespace]) -> None: + del models["us"].variables[0].enum_values + + +def _make_enum_label_nan(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label=float("nan"))] + + +def _name_unknown_basic_input(models: dict[str, SimpleNamespace]) -> None: + models["us"].basic_inputs = ["employment_income", "unpublished_input"] + + +def _remove_variable_unit(models: dict[str, SimpleNamespace]) -> None: + del models["us"].variables[0].unit + + +def _make_unit_a_number(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].unit = 1 + + +def _make_definition_period_a_list(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].definition_period = ["year"] + + +def _make_input_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].is_input_variable = "true" + + +def _make_hidden_flag_number(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].hidden_input = 0 + + +def _name_unknown_adds_parameter(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].adds_parameter_path = "gov.example.missing" + + +def _name_unknown_subtracts_parameter(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].subtracts_parameter_path = "gov.example.missing" + + +def _mismatch_enum_values(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE", "JOINT"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label="Single")] + + +def _give_enum_labels_without_values(models: dict[str, SimpleNamespace]) -> None: + models["us"].variables[0].enum_values = [ + SimpleNamespace(value="SINGLE", label="Single") + ] + + +def _make_enum_label_an_object(models: dict[str, SimpleNamespace]) -> None: + variable = models["us"].variables[0] + variable.possible_values = ["SINGLE"] + variable.enum_values = [SimpleNamespace(value="SINGLE", label={"text": "Single"})] + + +def _make_economy_flag_missing(models: dict[str, SimpleNamespace]) -> None: + del models["us"].parameters[0].economy + + +def _make_economy_flag_null(models: dict[str, SimpleNamespace]) -> None: + models["us"].parameters[0].economy = None + + +def _make_household_flag_text(models: dict[str, SimpleNamespace]) -> None: + models["us"].parameters[0].household = "false" + + +@pytest.mark.parametrize( + ("break_models", "message"), + [ + ( + _remove_entities, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: model version entities$", + ), + (_make_entities_null, "^us public model entities must be a sequence$"), + (_make_entity_flag_text, "^us entity person is_person must be a boolean$"), + ( + _remove_person_entity, + "^us public model must have exactly one person entity$", + ), + (_repeat_entity_key, "^us entity key 'household' is repeated$"), + ( + _remove_basic_inputs, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: model version basic_inputs$", + ), + ( + _make_basic_inputs_text, + "^us public model basic_inputs must be a sequence$", + ), + ( + _remove_variable_enum_values, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: variable enum_values$", + ), + ( + _make_enum_label_nan, + "^variable employment_income enum label for SINGLE is not " + "JSON-compatible: NaN is not JSON-compatible$", + ), + ( + _name_unknown_basic_input, + "^us basic input 'unpublished_input' is not a published variable$", + ), + ( + _remove_variable_unit, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: variable unit$", + ), + ( + _make_unit_a_number, + "^variable employment_income unit must be a string or null$", + ), + ( + _make_definition_period_a_list, + "^variable employment_income definition_period must be a string or null$", + ), + ( + _make_input_flag_text, + "^variable employment_income is_input_variable must be a boolean or null$", + ), + ( + _make_hidden_flag_number, + "^variable employment_income hidden_input must be a boolean or null$", + ), + ( + _name_unknown_adds_parameter, + "^variable employment_income adds_parameter_path 'gov.example.missing' " + "is not a published parameter$", + ), + ( + _name_unknown_subtracts_parameter, + "^variable employment_income subtracts_parameter_path " + "'gov.example.missing' is not a published parameter$", + ), + ( + _mismatch_enum_values, + "^variable employment_income enum values do not match its possible values$", + ), + ( + _give_enum_labels_without_values, + "^variable employment_income enum values do not match its possible values$", + ), + ( + _make_enum_label_an_object, + "^variable employment_income enum label for SINGLE must be a JSON scalar$", + ), + ( + _make_economy_flag_missing, + "^the installed PolicyEngine.py does not publish catalog format 2's " + "fields for us: parameter economy$", + ), + ( + _make_economy_flag_null, + "^parameter gov.example.rate economy must be a boolean$", + ), + ( + _make_household_flag_text, + "^parameter gov.example.rate household must be a boolean$", + ), + ], +) +def test_inconsistent_format_2_metadata_fails_with_a_specific_message( + break_models: Callable[[dict[str, SimpleNamespace]], None], + message: str, +) -> None: + models = source_models() + break_models(models) + + with pytest.raises(CatalogExtractionError, match=message): + _extract(models) + + +def test_a_release_without_format_2_fields_fails_naming_each_missing_field() -> None: + """A release without the fields fails before any record is checked, naming each. + + PolicyEngine.py 6.7.5 and the locked 6.10.0, for example, publish only the + variables' unit and definition period of these fields. + """ + models = source_models() + model = models["us"] + del model.entities, model.basic_inputs + for variable in model.variables: + for field_name in ( + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + ): + delattr(variable, field_name) + for parameter in model.parameters: + del parameter.economy, parameter.household + + with pytest.raises( + CatalogExtractionError, + match=r"^the installed PolicyEngine.py does not publish catalog format 2's " + r"fields for us: model version entities, basic_inputs; variable " + r"is_input_variable, hidden_input, module_name, adds_parameter_path, " + r"subtracts_parameter_path, enum_values, unit, definition_period; parameter " + r"economy, household$", + ): + _extract(models) + + +def test_an_empty_basic_input_list_is_stored_empty() -> None: + models = source_models() + models["us"].basic_inputs = [] + + assert _extract(models).country("us").model_version.basic_inputs == () + + +FORMAT_2_VARIABLE_TEXT_FIELDS = ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "unit", + "definition_period", +) + + +def _variable_column_length(field_name: str) -> int: + from policyengine_api.data.v2.models import Variable + + return Variable.__table__.c[field_name].type.length + + +def test_variable_text_limits_are_the_variable_column_lengths() -> None: + """Extraction checks each format 2 text field against its column's length.""" + from policyengine_api.data.v2.catalog.extraction import VARIABLE_TEXT_LIMITS + + column_lengths = { + field_name: _variable_column_length(field_name) + for field_name in FORMAT_2_VARIABLE_TEXT_FIELDS + } + + assert column_lengths == VARIABLE_TEXT_LIMITS + + +@pytest.mark.parametrize("field_name", FORMAT_2_VARIABLE_TEXT_FIELDS) +def test_an_over_long_format_2_text_field_stops_extraction(field_name: str) -> None: + length = _variable_column_length(field_name) + models = source_models() + setattr(models["us"].variables[0], field_name, "x" * (length + 1)) + + with pytest.raises( + CatalogExtractionError, + match=rf"^variable employment_income {field_name} exceeds {length} characters$", + ): + _extract(models) diff --git a/tests/unit/v2/test_catalog_publication.py b/tests/unit/v2/test_catalog_publication.py index a3cbe0d65..47cb262ba 100644 --- a/tests/unit/v2/test_catalog_publication.py +++ b/tests/unit/v2/test_catalog_publication.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from io import StringIO from pathlib import Path from unittest.mock import patch @@ -213,12 +214,42 @@ def test_publication_uses_mapped_tables_and_sqlalchemy_statements() -> None: assert all("VALUES" not in statement for statement in compiled_inserts) catalog = normalized_catalog() - comparisons = publication_reconciliation._comparison_pairs(catalog.country("us")) - assert all( - isinstance(statement, sa.sql.Select) and not isinstance(statement, TextClause) - for pair in comparisons - for statement in pair + format_1_comparisons, format_2_comparisons = ( + publication_reconciliation._comparison_pairs( + catalog.country("us"), with_format_2=with_format_2 + ) + for with_format_2 in (False, True) ) + for comparisons in (format_1_comparisons, format_2_comparisons): + assert [table_name for table_name, _staged, _stored in comparisons] == [ + "model version", + "variables", + "parameter nodes", + "parameters", + "parameter values", + "datasets", + ] + statements = [ + statement for _table_name, *pair in comparisons for statement in pair + ] + list( + publication_reconciliation._region_comparison_pair(catalog.country("us")) + ) + assert all( + isinstance(statement, sa.sql.Select) + and not isinstance(statement, TextClause) + for statement in statements + ) + assert all( + len(staged.selected_columns) == len(stored.selected_columns) + for _table_name, staged, stored in comparisons + ) + # Format 2 compares model versions, variables and parameters on more fields. + assert [ + len(format_2[1].selected_columns) - len(format_1[1].selected_columns) + for format_1, format_2 in zip( + format_1_comparisons, format_2_comparisons, strict=True + ) + ] == [3, 8, 0, 2, 0, 0] class Result: def scalar_one(self): @@ -251,6 +282,8 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: entity_counts={"parameters": 12}, fallback_summaries=(("us", "state", 3),), elapsed_seconds=2.3456, + omitted_region_summaries=(("us", "place", 333),), + kept_region_summaries=(("uk", "6.7.5", 5, 1_015),), ).as_dict() assert evidence == { @@ -261,6 +294,17 @@ def test_completion_evidence_contains_only_reviewed_non_secret_fields() -> None: "fallback_summaries": [ {"country_id": "us", "region_type": "state", "count": 3} ], + "omitted_region_summaries": [ + {"country_id": "us", "region_type": "place", "count": 333} + ], + "kept_regions": [ + { + "country_id": "uk", + "policyengine_version": "6.7.5", + "stored_regions": 5, + "staged_regions": 1_015, + } + ], "elapsed_seconds": 2.346, } assert ( @@ -293,3 +337,55 @@ def test_fallback_summary_emits_one_non_secret_warning() -> None: rendered_message = message_template % (logged_summaries,) assert "regional dataset fallback summary" in rendered_message assert "postgresql://" not in rendered_message + + +def _kept_region_records( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + differences: tuple[int, int], +) -> list[logging.LogRecord]: + monkeypatch.setattr( + publication, + "count_region_differences", + lambda _connection, _country: differences, + ) + monkeypatch.setattr(publication, "count_stored_regions", lambda *_args: 4) + # An earlier test's Alembic logging setup may have disabled existing loggers. + monkeypatch.setattr(publication.LOGGER, "disabled", False) + us = normalized_catalog().country("us") + with caplog.at_level(logging.WARNING, logger=publication.LOGGER.name): + summary = publication._kept_region_summary(object(), us) + assert summary == (None if differences == (0, 0) else ("us", "5.0.4", 4, 3)) + return [ + record for record in caplog.records if record.name == publication.LOGGER.name + ] + + +def test_kept_region_warning_prints_country_version_and_counts_without_logging_setup( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + [record] = _kept_region_records(monkeypatch, caplog, (1, 2)) + + # The seed command configures no logging, so Python's last-resort handler + # prints the record; this is the text the job output shows. + assert logging.lastResort is not None + assert logging.lastResort.format(record) == ( + "kept the published regions of the stored us catalog version 5.0.4: " + "1 staged region rows are not stored and 2 stored region rows are not " + "staged" + ) + assert record.levelno == logging.WARNING + assert record.policyengine_attributes == { + "country_id": "us", + "policyengine_version": "5.0.4", + "staged_region_rows_not_stored": 1, + "stored_region_rows_not_staged": 2, + } + + +def test_matching_kept_regions_log_nothing( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + assert _kept_region_records(monkeypatch, caplog, (0, 0)) == [] diff --git a/tests/unit/v2/test_catalog_spm_availability.py b/tests/unit/v2/test_catalog_spm_availability.py new file mode 100644 index 000000000..c8cd1ac1b --- /dev/null +++ b/tests/unit/v2/test_catalog_spm_availability.py @@ -0,0 +1,243 @@ +"""Each catalog version's SPM availability, from the simulation service's versions.""" + +from __future__ import annotations + +from contextlib import contextmanager +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import httpx +import pytest + +from policyengine_api.data.v2.catalog import spm_availability +from policyengine_api.data.v2.catalog.publication_types import CatalogPublicationError +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.spm import SPMSelection + +if TYPE_CHECKING: + from collections.abc import Iterator + +POLICYENGINE_VERSION = "6.10.0" +DEFAULTS = { + "forecast_content_sha256": "3d" * 32, + "scenario": "ce_trend", + "geography_kind": "county", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +ROUTES = {"latest": POLICYENGINE_VERSION, POLICYENGINE_VERSION: "policyengine-v6-10-0"} +VERSIONS = { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": DEFAULTS, + } + }, +} + + +def test_a_us_version_with_a_certified_capability_offers_spm() -> None: + availability = spm_availability.spm_availability( + "us", POLICYENGINE_VERSION, VERSIONS + ) + + assert availability == { + "available": True, + "settings_schema": SPMSelection.model_json_schema(), + "defaults": DEFAULTS, + } + + +@pytest.mark.parametrize( + ("country_id", "policyengine_version", "versions"), + [ + ("uk", POLICYENGINE_VERSION, VERSIONS), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES}), + ("us", POLICYENGINE_VERSION, {"policyengine": ROUTES, "spm_capabilities": []}), + ( + "us", + POLICYENGINE_VERSION, + { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v0", + "defaults": DEFAULTS, + } + }, + }, + ), + ( + "us", + POLICYENGINE_VERSION, + { + "policyengine": ROUTES, + "spm_capabilities": { + POLICYENGINE_VERSION: { + "contract_version": "canonical-spm-v1", + "defaults": {**DEFAULTS, "scenario": None}, + } + }, + }, + ), + ], + ids=[ + "uk", + "no-capability", + "malformed-capabilities", + "other-contract", + "unpinned-defaults", + ], +) +def test_anything_else_offers_no_spm( + country_id: str, policyengine_version: str, versions: object +) -> None: + assert spm_availability.spm_availability( + country_id, policyengine_version, versions + ) == {"available": False} + + +@pytest.mark.parametrize( + "versions", + [ + {**VERSIONS, "policyengine": {"latest": "6.11.0", "6.11.0": "app"}}, + {"spm_capabilities": VERSIONS["spm_capabilities"]}, + [], + ], + ids=["version-not-served", "no-route-map", "not-an-object"], +) +def test_a_us_version_the_service_does_not_serve_is_not_known_yet( + versions: object, +) -> None: + assert ( + spm_availability.spm_availability("us", POLICYENGINE_VERSION, versions) is None + ) + + +class RecordingEngine: + """An engine whose transactions are only counted.""" + + def __init__(self) -> None: + """Start with no transaction.""" + self.transactions = 0 + + @contextmanager + def begin(self) -> Iterator[str]: + """Count one transaction and hand out a placeholder connection.""" + self.transactions += 1 + yield "connection" + + +def test_publication_records_only_what_is_known( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[tuple[str, dict]] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + (model_name, dict(availability)) + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id=country_id, + model=SimpleNamespace(name=f"policyengine-{country_id}"), + model_version=SimpleNamespace(version="1.0.0"), + ) + for country_id in ("uk", "us") + ], + ) + not_served = {"policyengine": {"latest": "6.11.0", "6.11.0": "app"}} + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=lambda _url: not_served, + ) + + assert recorded == [("policyengine-uk", {"available": False})] + + +def test_the_versions_come_from_the_simulation_services_public_route() -> None: + requests: list[httpx.Request] = [] + + def answer(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=VERSIONS) + + with httpx.Client(transport=httpx.MockTransport(answer)) as client: + versions = spm_availability.fetch_simulation_versions( + "https://simulation.example/", client=client + ) + + assert versions == VERSIONS + assert [str(request.url) for request in requests] == [ + "https://simulation.example/versions" + ] + assert "authorization" not in requests[0].headers + + +def test_an_unreadable_versions_answer_stops_the_recording() -> None: + with ( + httpx.Client( + transport=httpx.MockTransport(lambda _request: httpx.Response(503)) + ) as client, + pytest.raises(CatalogPublicationError, match="versions"), + ): + spm_availability.fetch_simulation_versions( + "https://simulation.example", client=client + ) + + +def test_recording_needs_the_simulation_service_url() -> None: + with pytest.raises(V2ConfigurationError, match="SIMULATION_ENTRYPOINT_URL"): + spm_availability.simulation_entrypoint_url({}) + + assert ( + spm_availability.simulation_entrypoint_url( + {"SIMULATION_ENTRYPOINT_URL": " https://simulation.example/ "} + ) + == "https://simulation.example" + ) + + +def test_an_unreadable_versions_answer_leaves_every_version_unrecorded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorded: list[str] = [] + monkeypatch.setattr( + spm_availability, + "record_spm_availability", + lambda _connection, *, model_name, version, availability: recorded.append( + model_name + ), + ) + catalog = SimpleNamespace( + policyengine_version=POLICYENGINE_VERSION, + countries=[ + SimpleNamespace( + country_id="us", + model=SimpleNamespace(name="policyengine-us"), + model_version=SimpleNamespace(version="1.0.0"), + ) + ], + ) + + def unreadable(_url: str) -> object: + msg = "the simulation service's versions could not be read" + raise CatalogPublicationError(msg) + + spm_availability.record_published_spm_availability( + RecordingEngine(), + catalog, + {"SIMULATION_ENTRYPOINT_URL": "https://simulation.example"}, + fetch_versions=unreadable, + ) + + assert recorded == [] diff --git a/tests/unit/v2/test_catalog_staging_rows.py b/tests/unit/v2/test_catalog_staging_rows.py index f43f2772f..e8b6f0bd4 100644 --- a/tests/unit/v2/test_catalog_staging_rows.py +++ b/tests/unit/v2/test_catalog_staging_rows.py @@ -3,13 +3,16 @@ from __future__ import annotations import inspect +from dataclasses import replace from psycopg.types.json import Jsonb from policyengine_api.data.v2.catalog.publication_staging import ( + STAGE_CATALOG_VARIABLES, STAGING_TABLES, _catalog_rows, ) +from policyengine_api.data.v2.catalog.records import EnumValueRecord from tests.fixtures.v2_catalog import normalized_catalog @@ -49,6 +52,15 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: country.model_version.description, country.model_version.current_law_id, ("jsonb", country.model_version.metadata_time_periods), + 2, + ( + "jsonb", + [ + {"key": "person", "plural": "people", "is_person": True}, + {"key": "household", "plural": "households", "is_person": False}, + ], + ), + ("jsonb", ["employment_income"]), ) ] assert rows["stage_catalog_variables"] == [ @@ -66,6 +78,14 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: ("jsonb", record.default_value), None if record.adds is None else ("jsonb", record.adds), None if record.subtracts is None else ("jsonb", record.subtracts), + record.is_input_variable, + record.hidden_input, + record.module_name, + record.adds_parameter_path, + record.subtracts_parameter_path, + None, + record.unit, + record.definition_period, ) for record in country.variables ] @@ -82,6 +102,8 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: record.description, record.data_type, record.unit, + record.economy, + record.household, ) for record in country.parameters ] @@ -121,3 +143,24 @@ def test_catalog_rows_project_every_record_field_in_column_order() -> None: ] assert len(rows["stage_catalog_parameter_values"]) == 2 assert len(rows["stage_catalog_regions"]) == 3 + + +def test_enum_values_are_staged_as_value_and_label_objects_in_order() -> None: + country = normalized_catalog().country("us") + variable = replace( + country.variables[0], + possible_values=["SINGLE", "JOINT"], + enum_values=( + EnumValueRecord(value="SINGLE", label="Single"), + EnumValueRecord(value="JOINT", label=2), + ), + ) + country = replace(country, variables=(variable,)) + + (row,) = (_plain(row) for row in _catalog_rows(country)[STAGE_CATALOG_VARIABLES]) + + staged_column_names = [column.name for column in STAGE_CATALOG_VARIABLES.columns] + assert row[staged_column_names.index("enum_values")] == ( + "jsonb", + [{"value": "SINGLE", "label": "Single"}, {"value": "JOINT", "label": 2}], + ) diff --git a/tests/unit/v2/test_certified_data_inspection.py b/tests/unit/v2/test_certified_data_inspection.py new file mode 100644 index 000000000..a0b375909 --- /dev/null +++ b/tests/unit/v2/test_certified_data_inspection.py @@ -0,0 +1,234 @@ +"""Reading household column names from certified HDF5 datasets.""" + +from __future__ import annotations + +import ast +import importlib.metadata +import tomllib +from pathlib import Path + +import h5py +import numpy as np +import pandas as pd +import pytest +from huggingface_hub import HfFileSystem +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +from policyengine_api.data.v2.catalog.certified_data_inspection import ( + CertifiedDataReadError, + HuggingFaceDataInspector, + household_columns_in_hdf5, + huggingface_path, +) +from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +HOUSEHOLDS = pd.DataFrame( + { + "household_id": [1, 2], + "household_weight": [1.5, 2.5], + "state_fips": [6, 36], + } +) + + +def _artifact( + *, + name: str = "populace_us_2024_acs_local", + repo_type: str = "dataset", + revision: str = "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z", +) -> CertifiedArtifact: + return CertifiedArtifact( + name=name, + repo_id="policyengine/populace-us", + repo_type=repo_type, + path=f"{name}.h5", + revision=revision, + sha256=None, + ) + + +def _columns(path: Path) -> frozenset[str]: + with h5py.File(path, "r") as h5: + return household_columns_in_hdf5(h5) + + +def test_reads_columns_of_a_fixed_format_household_frame(tmp_path: Path) -> None: + path = tmp_path / "fixed.h5" + HOUSEHOLDS.to_hdf(path, key="household", format="fixed") + + assert _columns(path) == {"household_id", "household_weight", "state_fips"} + + +def test_reads_data_columns_of_a_table_format_household_frame(tmp_path: Path) -> None: + path = tmp_path / "table.h5" + frame = HOUSEHOLDS.assign(region=pd.Categorical(["LONDON", "WALES"])) + frame.to_hdf(path, key="household", format="table", data_columns=True) + + assert _columns(path) == { + "household_id", + "household_weight", + "state_fips", + "region", + } + + +# PyTables warns that year keys are not Python identifiers; policyengine-uk's +# multi-year files use the same keys. +@pytest.mark.filterwarnings("ignore:object name is not a valid Python identifier") +def test_keeps_only_columns_every_year_of_a_multi_year_file_has( + tmp_path: Path, +) -> None: + path = tmp_path / "years.h5" + HOUSEHOLDS.to_hdf(path, key="household/2024", format="table", data_columns=True) + HOUSEHOLDS.drop(columns="state_fips").to_hdf( + path, key="household/2025", format="table", data_columns=True + ) + + assert _columns(path) == {"household_id", "household_weight"} + + +def test_reads_variable_names_of_a_core_format_file(tmp_path: Path) -> None: + path = tmp_path / "core.h5" + with h5py.File(path, "w") as h5: + h5.create_dataset("state_fips/2024", data=np.array([6, 36])) + h5.create_dataset("household_weight/2024", data=np.array([1.5, 2.5])) + + assert _columns(path) == {"state_fips", "household_weight"} + + +def test_refuses_a_table_whose_columns_are_stored_in_value_blocks( + tmp_path: Path, +) -> None: + path = tmp_path / "blocks.h5" + HOUSEHOLDS.to_hdf(path, key="household", format="table") + + with pytest.raises( + CertifiedDataReadError, + match=r"^household table household stores columns in value blocks, whose " + r"names cannot be read without unpickling$", + ): + _columns(path) + + +def test_refuses_a_household_group_that_holds_no_frame(tmp_path: Path) -> None: + path = tmp_path / "empty.h5" + with h5py.File(path, "w") as h5: + h5.create_group("household") + + with pytest.raises( + CertifiedDataReadError, match=r"^household group holds no data frame$" + ): + _columns(path) + + +def test_refuses_a_household_entry_that_is_not_a_group(tmp_path: Path) -> None: + path = tmp_path / "flat.h5" + with h5py.File(path, "w") as h5: + h5.create_dataset("household", data=np.array([1, 2])) + + with pytest.raises( + CertifiedDataReadError, match=r"^household entry is not an HDF5 group$" + ): + _columns(path) + + +def test_default_file_system_is_hugging_face_with_the_token() -> None: + inspector = HuggingFaceDataInspector(token="read-token") + + file_system = inspector._file_system + assert isinstance(file_system, HfFileSystem) + assert file_system.token == "read-token" + + +def test_names_dataset_and_model_repository_files_at_their_revision() -> None: + assert huggingface_path(_artifact()) == ( + "datasets/policyengine/populace-us@" + "populace-us-2024-buildo-acs-local-766b7c7-20261017T030317Z/" + "populace_us_2024_acs_local.h5" + ) + assert huggingface_path( + _artifact(name="microcosm_uk_2024_25", repo_type="model", revision="1.0.0") + ) == ("policyengine/populace-us@1.0.0/microcosm_uk_2024_25.h5") + assert huggingface_path(_artifact(revision="refs/pr/1")).startswith( + "datasets/policyengine/populace-us@refs%2Fpr%2F1/" + ) + + +class FakeFileSystem: + """Serves local files for Hugging Face paths and records how it was built.""" + + def __init__(self, files: dict[str, Path]) -> None: + self.files = files + self.opened: list[str] = [] + + def open(self, path: str, mode: str, *, block_size: int, cache_type: str): + assert (mode, block_size, cache_type) == ("rb", 1 << 20, "blockcache") + self.opened.append(path) + return self.files[path].open("rb") + + +def test_inspector_reads_artifacts_through_an_authenticated_file_system( + tmp_path: Path, +) -> None: + dataset_path = tmp_path / "dataset.h5" + HOUSEHOLDS.to_hdf(dataset_path, key="household", format="fixed") + local_area_path = tmp_path / "local_area.h5" + HOUSEHOLDS.assign(la_code_oa=["N09000011", "E07000110"]).to_hdf( + local_area_path, key="household", format="table", data_columns=True + ) + dataset = _artifact() + local_area = _artifact(name="microcosm_uk_2024_25_dense", repo_type="dataset") + file_system = FakeFileSystem( + { + huggingface_path(dataset): dataset_path, + huggingface_path(local_area): local_area_path, + } + ) + tokens: list[str | None] = [] + + def build_file_system(token: str | None) -> FakeFileSystem: + tokens.append(token) + return file_system + + inspector = HuggingFaceDataInspector( + token="read-token", file_system_factory=build_file_system + ) + + assert "state_fips" in inspector.household_columns(dataset) + assert "la_code_oa" in inspector.household_columns(local_area) + assert tokens == ["read-token"] + assert file_system.opened == [ + huggingface_path(dataset), + huggingface_path(local_area), + ] + + +def test_imports_only_packages_the_project_depends_on_directly() -> None: + repo = Path(__file__).parents[3] + module = repo / "policyengine_api/data/v2/catalog/certified_data_inspection.py" + tree = ast.parse(module.read_text(encoding="utf-8")) + top_level_imports = { + alias.name.split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } | { + (node.module or "").split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) and node.level == 0 + } + distributions = importlib.metadata.packages_distributions() + third_party = { + canonicalize_name(distributions[name][0]) + for name in top_level_imports + if name in distributions and name != "policyengine_api" + } + project = tomllib.loads((repo / "pyproject.toml").read_text(encoding="utf-8")) + direct = { + canonicalize_name(Requirement(requirement).name) + for requirement in project["project"]["dependencies"] + } + + assert third_party == {"h5py", "huggingface-hub"} + assert third_party <= direct diff --git a/tests/unit/v2/test_country_routes.py b/tests/unit/v2/test_country_routes.py new file mode 100644 index 000000000..7fd2e2b91 --- /dev/null +++ b/tests/unit/v2/test_country_routes.py @@ -0,0 +1,198 @@ +"""Route coverage for the native v2 countries list.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from flask import Flask, jsonify +from sqlalchemy.exc import OperationalError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, + infer_route_group, +) +from policyengine_api.services.v2.countries.types import CountryRead + +UNITED_KINGDOM = CountryRead(id="uk", household_enabled=True, society_wide_enabled=True) +UNITED_STATES = CountryRead(id="us", household_enabled=True, society_wide_enabled=True) +HOUSEHOLD_ONLY_COUNTRY = CountryRead( + id="be", household_enabled=True, society_wide_enabled=False +) + + +class FakeCountryService: + """Return fixed countries, or raise ``error``, and count the calls.""" + + def __init__( + self, + countries: list[CountryRead], + *, + error: Exception | None = None, + ) -> None: + self.countries = countries + self.error = error + self.list_calls = 0 + + def list_countries(self) -> list[CountryRead]: + self.list_calls += 1 + if self.error is not None: + raise self.error + return self.countries + + +def _client(service: FakeCountryService) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app(flask_app, dependencies=dependencies, route_settings=settings), + raise_server_exceptions=False, + ) + return client, flask_calls + + +def test_countries_list_every_country_with_its_capability() -> None: + service = FakeCountryService( + [HOUSEHOLD_ONLY_COUNTRY, UNITED_KINGDOM, UNITED_STATES] + ) + client, flask_calls = _client(service) + + response = client.get("/v2/countries") + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "items": [ + { + "id": "be", + "household_enabled": True, + "society_wide_enabled": False, + }, + { + "id": "uk", + "household_enabled": True, + "society_wide_enabled": True, + }, + { + "id": "us", + "household_enabled": True, + "society_wide_enabled": True, + }, + ] + }, + } + assert service.list_calls == 1 + assert flask_calls["count"] == 0 + + +def test_countries_list_needs_no_user_credential() -> None: + client, _flask_calls = _client(FakeCountryService([UNITED_STATES])) + + response = client.get("/v2/countries", headers={"Authorization": ""}) + + assert response.status_code == 200 + + +def test_countries_list_rejects_query_parameters() -> None: + service = FakeCountryService([UNITED_STATES]) + client, _flask_calls = _client(service) + + response = client.get("/v2/countries?country_id=us") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert service.list_calls == 0 + + +@pytest.mark.parametrize( + "error", + [ + V2ConfigurationError("V2_RUNTIME_DATABASE_URL is required"), + OperationalError("SELECT", {}, Exception("connection refused")), + ], + ids=["configuration", "database"], +) +def test_countries_list_reports_an_unavailable_database_as_503( + error: Exception, +) -> None: + client, _flask_calls = _client(FakeCountryService([], error=error)) + + response = client.get("/v2/countries") + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 countries are unavailable", + } + + +def test_countries_list_hides_unexpected_failures_behind_a_500() -> None: + client, _flask_calls = _client( + FakeCountryService([], error=RuntimeError("secret detail")) + ) + + response = client.get("/v2/countries") + + assert response.status_code == 500 + assert response.json() == { + "status": "error", + "message": "V2 countries query failed", + } + + +def test_countries_list_supports_get_only() -> None: + service = FakeCountryService([UNITED_STATES]) + client, flask_calls = _client(service) + + response = client.post("/v2/countries", json={}) + + assert response.status_code == 405 + assert service.list_calls == 0 + assert flask_calls["count"] == 0 + + +def test_countries_list_is_documented_in_the_v2_openapi_document() -> None: + client, _flask_calls = _client(FakeCountryService([])) + + document = client.get("/v2/openapi.json").json() + operation = document["paths"]["/v2/countries"]["get"] + + assert operation.get("parameters", []) == [] + assert operation["responses"]["200"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/CountryListResponse" + } + assert {"422", "500", "503"} <= operation["responses"].keys() + item_schema = document["components"]["schemas"]["CountryItem"] + assert item_schema["required"] == [ + "id", + "household_enabled", + "society_wide_enabled", + ] + assert item_schema["additionalProperties"] is False + + +def test_countries_requests_are_logged_with_the_metadata_route_group() -> None: + assert infer_route_group("/v2/countries") == "metadata" diff --git a/tests/unit/v2/test_country_service.py b/tests/unit/v2/test_country_service.py new file mode 100644 index 000000000..253278599 --- /dev/null +++ b/tests/unit/v2/test_country_service.py @@ -0,0 +1,85 @@ +"""SQLite-backed coverage for the v2 countries service: the list and the check.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import V2_METADATA, Country +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS +from tests.fixtures.v2_countries import add_listed_countries + + +def _service_with_countries(*extra_countries: Country) -> V2CountryService: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + sessions = sessionmaker(engine, class_=Session) + with sessions() as session: + add_listed_countries(session) + session.add_all(extra_countries) + session.commit() + return V2CountryService(CountryDatabaseSession(sessions)) + + +def test_countries_are_listed_by_id_with_their_capability_flags() -> None: + service = _service_with_countries( + Country(id="be", household_enabled=True, society_wide_enabled=False) + ) + + countries = service.list_countries() + + assert countries == [ + CountryRead(id="be", household_enabled=True, society_wide_enabled=False), + CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ] + + +def test_an_empty_countries_table_lists_no_countries() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + assert service.list_countries() == [] + + +def test_rows_for_exactly_the_listed_countries_pass_verification() -> None: + service = _service_with_countries() + + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + +def test_a_row_for_an_unlisted_country_fails_verification() -> None: + service = _service_with_countries( + Country(id="ca", household_enabled=True, society_wide_enabled=False) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.rows_not_listed == ("ca",) + assert raised.value.listed_without_row == () + + +def test_an_empty_countries_table_fails_verification() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.listed_without_row == tuple(sorted(SUPPORTED_COUNTRY_IDS)) diff --git a/tests/unit/v2/test_country_validators.py b/tests/unit/v2/test_country_validators.py new file mode 100644 index 000000000..d56db5c00 --- /dev/null +++ b/tests/unit/v2/test_country_validators.py @@ -0,0 +1,68 @@ +"""The comparison of stored countries rows with the country list in code.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, + validate_countries_match_code_list, +) + + +def test_the_same_countries_in_any_order_match() -> None: + validate_countries_match_code_list( + stored_country_ids=["us", "uk"], + listed_country_ids=("uk", "us"), + ) + + +def test_a_listed_country_without_a_row_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["uk"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("us",) + assert raised.value.rows_not_listed == () + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: us; rows not listed: none" + ) + + +def test_a_row_for_an_unlisted_country_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ca", "uk", "us"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == () + assert raised.value.rows_not_listed == ("ca",) + + +def test_every_difference_is_reported_in_order() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ng", "ca"], + listed_country_ids=("us", "uk"), + ) + + assert raised.value.listed_without_row == ("uk", "us") + assert raised.value.rows_not_listed == ("ca", "ng") + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: uk, us; rows not listed: ca, ng" + ) + + +def test_an_empty_table_misses_every_listed_country() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=[], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("uk", "us") diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index a00667a49..fbfa92c72 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -1,11 +1,14 @@ """Tests for lazy process-owned v2 engine and SQLModel Session factories.""" from collections.abc import Iterator +from pathlib import Path import pytest -from sqlmodel import Session +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select from policyengine_api.data.v2 import database +from policyengine_api.data.v2.models import V2_METADATA, TaxBenefitModel from policyengine_api.data.v2.settings import ( V2_MIGRATION_DATABASE_URL, V2_RUNTIME_DATABASE_URL, @@ -125,3 +128,66 @@ def test_forked_process_state_builds_a_new_engine( assert child_engine is not parent_engine assert database._engine_state.engine_pid == parent_pid + 1 + + +def test_read_sessions_share_the_runtime_pool_without_transactions() -> None: + settings = load_v2_runtime_database_settings(_environment()) + + read_factory = database.get_v2_read_session_factory(settings) + runtime_engine = database.get_v2_engine(settings) + + with read_factory() as session: + bind = session.get_bind() + assert bind.get_execution_options()["isolation_level"] == "AUTOCOMMIT" + assert bind.pool is runtime_engine.pool + assert database.get_v2_read_session_factory(settings) is read_factory + assert runtime_engine.get_execution_options().get("isolation_level") is None + + +def test_read_session_factory_is_rebuilt_with_the_engine() -> None: + settings = load_v2_runtime_database_settings(_environment()) + first = database.get_v2_read_session_factory(settings) + + database.close_v2_database() + + assert database.get_v2_read_session_factory(settings) is not first + + +def test_read_sessions_refuse_to_write_rows() -> None: + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + V2_METADATA.create_all(engine) + read_factory = database.build_v2_read_session_factory(engine) + + with read_factory() as session: + session.add(TaxBenefitModel(name="policyengine-test")) + with pytest.raises(database.ReadOnlySessionError): + session.flush() + + with Session(engine) as session: + assert session.exec(select(TaxBenefitModel)).all() == [] + engine.dispose() + + +def test_only_the_metadata_reader_opens_read_sessions() -> None: + package = Path(database.__file__).parents[2] + users = sorted( + str(module.relative_to(package)) + for module in package.rglob("*.py") + if module.name != "database.py" + and ( + "get_v2_read_session_factory" in module.read_text(encoding="utf-8") + or "build_v2_read_session_factory" in module.read_text(encoding="utf-8") + ) + ) + + assert users == ["fastapi_routes/dependencies.py"] + dependencies_source = (package / "fastapi_routes/dependencies.py").read_text( + encoding="utf-8" + ) + reader_factory = dependencies_source.split( + "def _default_v2_metadata_reader_factory" + )[1].split("\ndef ")[0] + assert "get_v2_read_session_factory" in reader_factory + assert dependencies_source.count("get_v2_read_session_factory") == 2 diff --git a/tests/unit/v2/test_household_calculation_routes.py b/tests/unit/v2/test_household_calculation_routes.py new file mode 100644 index 000000000..2bbd63642 --- /dev/null +++ b/tests/unit/v2/test_household_calculation_routes.py @@ -0,0 +1,329 @@ +"""The /v2/household-calculations routes: submit a stored household, then poll it.""" + +from __future__ import annotations + +from uuid import uuid4 + +import httpx +import pytest +from fastapi.testclient import TestClient +from flask import Flask + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.household_calculations.simulation_client import ( + SimulationHouseholdClient, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + HOUSEHOLD_ID, + POLICY_ID, + SPM_CONFIG, + SPM_PROVENANCE, + FakeCatalogs, + FakeCountries, + FakeHouseholds, + FakePolicies, + FakeSimulation, + succeeded_calculation, +) + +PATH = "/v2/household-calculations" + + +class App: + """A client of the native app serving household calculations over fakes.""" + + def __init__(self, countries: FakeCountries | None = None) -> None: + """Build the fakes and the app.""" + self.simulation = FakeSimulation() + self.households = FakeHouseholds() + self.service = V2HouseholdCalculationService( + countries=countries or FakeCountries(), + households=self.households, + policies=FakePolicies(), + catalog_reader_factory=FakeCatalogs, + simulation_factory=lambda: self.simulation, + ) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_household_calculation_service_factory=lambda: self.service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + self.client = TestClient( + create_asgi_app( + Flask(__name__), dependencies=dependencies, route_settings=settings + ), + raise_server_exceptions=False, + ) + + +def body(**changes: object) -> dict[str, object]: + """Return a request calculating the stored household under the reform.""" + request: dict[str, object] = { + "country_id": "us", + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + } + request.update(changes) + return request + + +def test_a_calculation_is_accepted_without_an_account() -> None: + app = App() + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 202 + payload = response.json() + assert payload["status"] == "ok" + assert payload["result"]["calculation_id"] == str(CALCULATION_ID) + assert payload["result"]["status"] == "running" + assert payload["result"]["household_id"] == str(HOUSEHOLD_ID) + assert payload["result"]["result"] is None + + +def test_a_calculation_with_axes_sends_them() -> None: + app = App() + axes = [[{"name": "employment_income", "count": 3, "min": 0, "max": 50000}]] + + response = app.client.post(PATH, json=body(axes=axes)) + + assert response.status_code == 202 + assert app.simulation.submissions[0]["options"]["axes"] == axes + + +def test_a_succeeded_us_calculation_reads_back_its_spm_choice() -> None: + app = App() + app.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + response = app.client.get(f"{PATH}/{CALCULATION_ID}") + + assert response.status_code == 200 + result = response.json()["result"] + assert result["status"] == "succeeded" + assert result["result"]["spm_config"] == SPM_CONFIG + assert result["result"]["spm_provenance"] == SPM_PROVENANCE + assert result["result"]["warnings"] == [ + "The input `age_group` is deprecated and was ignored." + ] + + +def test_a_country_without_household_calculations_gets_422_naming_it() -> None: + app = App( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 422 + assert "us" in response.json()["message"] + assert app.simulation.submissions == [] + + +def test_a_household_only_country_is_accepted() -> None: + app = App() + stored = app.households.households[("us", HOUSEHOLD_ID)] + app.households.households[("uk", HOUSEHOLD_ID)] = stored.__class__( + **{ + **stored.__dict__, + "country_id": "uk", + "household_data": { + "people": [ + { + "id": "person-1", + "values": {}, + "memberships": { + "household": "household-1", + "benunit": "benunit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {}}], + "benunit": [{"id": "benunit-1", "values": {}}], + }, + } + ) + + response = app.client.post(PATH, json=body(country_id="uk", policy_id=None)) + + assert response.status_code == 202 + + +@pytest.mark.parametrize( + "changes", + [{"household_id": str(uuid4())}, {"policy_id": str(uuid4())}], + ids=["household", "policy"], +) +def test_an_unknown_household_or_policy_is_not_found(changes) -> None: + response = App().client.post(PATH, json=body(**changes)) + + assert response.status_code == 404 + + +def test_an_unknown_calculation_is_not_found() -> None: + assert App().client.get(f"{PATH}/{uuid4()}").status_code == 404 + + +@pytest.mark.parametrize( + ("error", "status_code"), + [ + (SimulationServiceUnavailableError("unreachable"), 503), + (SimulationServiceRefusedError("no household function"), 503), + ], + ids=["unreachable", "refused"], +) +def test_a_simulation_service_that_cannot_calculate_answers_503( + error, status_code +) -> None: + app = App() + app.simulation.error = error + + response = app.client.post(PATH, json=body()) + + assert response.status_code == status_code + + +@pytest.mark.parametrize( + "changes", + [ + {"axes": [[{"name": "employment_income", "count": 0, "min": 0, "max": 1}]]}, + { + "axes": [ + [ + {"name": "employment_income", "count": 3, "min": 0, "max": 1}, + {"name": "age", "count": 4, "min": 20, "max": 60}, + ] + ] + }, + { + "axes": [ + [{"name": "employment_income", "count": 101, "min": 0, "max": 1}], + [{"name": "age", "count": 100, "min": 20, "max": 60}], + ] + }, + {"country_id": "ca"}, + {"unexpected": True}, + {"year": 2026}, + {"force": True}, + ], + ids=[ + "axis-count", + "parallel-axes-of-different-counts", + "too-many-axis-points", + "country", + "unknown-field", + "year", + "force", + ], +) +def test_a_malformed_request_is_refused_before_anything_is_read(changes) -> None: + app = App() + + response = app.client.post(PATH, json=body(**changes)) + + assert response.status_code == 422 + assert app.simulation.submissions == [] + + +def test_a_stored_household_with_a_group_no_person_belongs_to_gets_422() -> None: + """The simulation service's 422 reaches the caller with its reason.""" + refusal = ( + f"the household {HOUSEHOLD_ID} cannot be calculated: " + "benunits.benunit-2.members: List should have at least 1 item after " + "validation" + ) + + def answer(request: httpx.Request) -> httpx.Response: + del request + return httpx.Response(422, json={"detail": refusal}) + + app = App() + app.service = V2HouseholdCalculationService( + countries=FakeCountries(), + households=app.households, + policies=FakePolicies(), + catalog_reader_factory=FakeCatalogs, + simulation_factory=lambda: SimulationHouseholdClient( + "https://simulation.example", + httpx_args={"transport": httpx.MockTransport(answer)}, + ), + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 422 + assert refusal in response.json()["message"] + + +def test_axis_groups_within_the_point_limit_are_accepted() -> None: + app = App() + axes = [ + [ + {"name": "employment_income", "count": 100, "min": 0, "max": 1}, + {"name": "self_employment_income", "count": 100, "min": 0, "max": 1}, + ], + [{"name": "age", "count": 100, "min": 20, "max": 60}], + ] + + response = app.client.post(PATH, json=body(axes=axes)) + + assert response.status_code == 202 + + +@pytest.mark.parametrize( + "refusal_body", + [ + {"detail": None}, + {"detail": {"reason": "x"}}, + {"detail": 7}, + {"detail": ["the household is refused"]}, + {"detail": [{"loc": ["body", "options"], "msg": "Field required"}]}, + ], + ids=["null", "object", "number", "list-of-strings", "entry-without-type"], +) +def test_a_simulation_refusal_of_another_shape_answers_503(refusal_body) -> None: + def answer(request: httpx.Request) -> httpx.Response: + del request + return httpx.Response(422, json=refusal_body) + + app = App() + app.service = V2HouseholdCalculationService( + countries=FakeCountries(), + households=app.households, + policies=FakePolicies(), + catalog_reader_factory=FakeCatalogs, + simulation_factory=lambda: SimulationHouseholdClient( + "https://simulation.example", + httpx_args={"transport": httpx.MockTransport(answer)}, + ), + ) + + response = app.client.post(PATH, json=body()) + + assert response.status_code == 503 diff --git a/tests/unit/v2/test_household_calculation_service.py b/tests/unit/v2/test_household_calculation_service.py new file mode 100644 index 000000000..c87c820db --- /dev/null +++ b/tests/unit/v2/test_household_calculation_service.py @@ -0,0 +1,307 @@ +"""Sending a stored household's calculation to the simulation service; reading it.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.household_calculations.services import ( + V2HouseholdCalculationService, +) +from policyengine_api.services.v2.household_calculations.types import ( + HouseholdCalculationInput, + SimulationServiceUnavailableError, +) +from policyengine_api.services.v2.household_calculations.validators import ( + HouseholdCalculationCountryError, + HouseholdCalculationNotFoundError, + HouseholdCalculationYearError, +) +from policyengine_api.services.v2.households.validators import HouseholdNotFoundError +from policyengine_api.services.v2.policies.validators import PolicyNotFoundError +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + HOUSEHOLD_ID, + POLICY_ID, + POLICYENGINE_VERSION, + SPM_CONFIG, + SPM_PROVENANCE, + FakeCatalogs, + FakeCountries, + FakeHouseholds, + FakePolicies, + FakeSimulation, + household_read, + simulation_calculation, + succeeded_calculation, +) + +AXES = [[{"name": "employment_income", "count": 5, "min": 0, "max": 80000}]] + + +class Parts: + """The service's collaborators, kept for the assertions.""" + + def __init__(self, countries: FakeCountries | None = None) -> None: + """Build every fake.""" + self.countries = countries or FakeCountries() + self.households = FakeHouseholds() + self.policies = FakePolicies() + self.catalog_readers: list[FakeCatalogs] = [] + self.simulation = FakeSimulation() + self.service = V2HouseholdCalculationService( + countries=self.countries, + households=self.households, + policies=self.policies, + catalog_reader_factory=self.open_catalog_reader, + simulation_factory=lambda: self.simulation, + ) + + def open_catalog_reader(self) -> FakeCatalogs: + """Open and keep a new catalog reader.""" + reader = FakeCatalogs() + self.catalog_readers.append(reader) + return reader + + +def calculation_input(**changes: object) -> HouseholdCalculationInput: + """Return a calculation of the stored US household under the stored reform.""" + values: dict[str, object] = { + "country_id": "us", + "household_id": HOUSEHOLD_ID, + "policy_id": POLICY_ID, + } + values.update(changes) + return HouseholdCalculationInput.model_validate(values) + + +def test_a_calculation_sends_the_household_and_policy_ids_and_the_version() -> None: + """The simulation service reads the household, policy and catalog version.""" + parts = Parts() + + calculation = parts.service.submit_calculation( + calculation_input(axes=AXES, add_missing_variables=True) + ) + + (submission,) = parts.simulation.submissions + assert submission == { + "country": "us", + "policyengine_version": POLICYENGINE_VERSION, + "household_id": str(HOUSEHOLD_ID), + "policy_id": str(POLICY_ID), + "options": { + "axes": AXES, + "add_missing_variables": True, + }, + "force": False, + "trigger": "initial", + } + assert calculation.calculation_id == CALCULATION_ID + assert calculation.status == "running" + assert calculation.result is None + (catalog_reader,) = parts.catalog_readers + assert catalog_reader.closed + + +def test_current_law_sends_no_policy() -> None: + parts = Parts() + + parts.service.submit_calculation(calculation_input(policy_id=None)) + + (submission,) = parts.simulation.submissions + assert submission["policy_id"] is None + assert submission["options"] == {"axes": None, "add_missing_variables": False} + + +def test_an_axis_without_a_period_is_sent_as_given() -> None: + """The simulation service varies the household's default year for it.""" + parts = Parts() + + parts.service.submit_calculation(calculation_input(axes=AXES)) + + assert parts.simulation.submissions[0]["options"]["axes"] == AXES + + +def test_a_household_without_a_default_year_is_refused() -> None: + parts = Parts() + parts.households.households[("us", HOUSEHOLD_ID)] = household_read( + default_year=None + ) + + with pytest.raises(HouseholdCalculationYearError): + parts.service.submit_calculation(calculation_input()) + assert parts.simulation.submissions == [] + + +def test_a_country_without_household_calculations_is_refused_by_name() -> None: + parts = Parts( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + + with pytest.raises(HouseholdCalculationCountryError, match="us"): + parts.service.submit_calculation(calculation_input()) + + assert parts.households.calls == [] + assert parts.simulation.submissions == [] + + +def test_a_household_only_country_is_accepted() -> None: + parts = Parts() + uk_household = household_read() + parts.households.households[("uk", HOUSEHOLD_ID)] = uk_household.__class__( + **{ + **uk_household.__dict__, + "country_id": "uk", + "household_data": { + "people": [ + { + "id": "person-1", + "values": {"age": {"2026": 30}}, + "memberships": { + "household": "household-1", + "benunit": "benunit-1", + }, + } + ], + "household": [{"id": "household-1", "values": {}}], + "benunit": [{"id": "benunit-1", "values": {}}], + }, + } + ) + + parts.service.submit_calculation(calculation_input(country_id="uk", policy_id=None)) + + (submission,) = parts.simulation.submissions + assert submission["country"] == "uk" + assert submission["policy_id"] is None + + +def test_an_unknown_household_or_policy_sends_nothing() -> None: + parts = Parts() + + with pytest.raises(HouseholdNotFoundError): + parts.service.submit_calculation(calculation_input(household_id=uuid4())) + with pytest.raises(PolicyNotFoundError): + parts.service.submit_calculation(calculation_input(policy_id=uuid4())) + + assert parts.simulation.submissions == [] + + +def test_a_us_result_includes_the_spm_choice_applied_and_its_receipt() -> None: + parts = Parts() + parts.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + calculation = parts.service.get_calculation(CALCULATION_ID) + + assert calculation.status == "succeeded" + assert calculation.result is not None + assert calculation.result.spm_config == SPM_CONFIG + assert calculation.result.spm_provenance == SPM_PROVENANCE + assert calculation.result.policyengine_version == POLICYENGINE_VERSION + assert calculation.result.household["households"]["household-1"][ + "household_net_income" + ] == {"2026": 27000.5} + + +def test_a_failed_calculation_reads_back_its_error() -> None: + parts = Parts() + detail = {"code": "invalid_inputs", "message": "Unrecognized inputs: incme"} + parts.simulation.read_answers[CALCULATION_ID] = simulation_calculation( + status="failed", + error={ + "code": "household_calculation_failed", + "message": "The household calculation failed: invalid_inputs.", + "detail": detail, + }, + ) + + calculation = parts.service.get_calculation(CALCULATION_ID) + + assert calculation.status == "failed" + assert calculation.error is not None + assert calculation.error.code == "household_calculation_failed" + assert calculation.error.detail == detail + + +def test_an_unknown_calculation_is_not_found() -> None: + with pytest.raises(HouseholdCalculationNotFoundError): + Parts().service.get_calculation(uuid4()) + + +def test_an_unreachable_simulation_service_is_reported() -> None: + parts = Parts() + parts.simulation.error = SimulationServiceUnavailableError("unreachable") + + with pytest.raises(SimulationServiceUnavailableError): + parts.service.submit_calculation(calculation_input()) + + +def test_a_refused_submission_and_a_read_open_no_catalog_reader() -> None: + parts = Parts() + parts.simulation.read_answers[CALCULATION_ID] = succeeded_calculation() + + with pytest.raises(HouseholdNotFoundError): + parts.service.submit_calculation(calculation_input(household_id=uuid4())) + parts.service.get_calculation(CALCULATION_ID) + + assert parts.catalog_readers == [] + + +def _unconfigured_simulation() -> FakeSimulation: + message = "The simulation service is not configured." + raise SimulationServiceUnavailableError(message) + + +def test_requests_refused_by_the_public_api_need_no_simulation_service() -> None: + parts = Parts( + FakeCountries( + rows=( + CountryRead( + id="us", household_enabled=False, society_wide_enabled=False + ), + ) + ) + ) + unconfigured = V2HouseholdCalculationService( + countries=parts.countries, + households=parts.households, + policies=parts.policies, + catalog_reader_factory=parts.open_catalog_reader, + simulation_factory=_unconfigured_simulation, + ) + available_country = V2HouseholdCalculationService( + countries=FakeCountries(), + households=parts.households, + policies=parts.policies, + catalog_reader_factory=parts.open_catalog_reader, + simulation_factory=_unconfigured_simulation, + ) + + with pytest.raises(HouseholdCalculationCountryError): + unconfigured.submit_calculation(calculation_input()) + with pytest.raises(HouseholdNotFoundError): + available_country.submit_calculation(calculation_input(household_id=uuid4())) + with pytest.raises(SimulationServiceUnavailableError): + available_country.submit_calculation(calculation_input()) + + +@pytest.mark.parametrize( + "period", + ["2025", "2025-03", None], + ids=["explicit-year", "explicit-month", "null-period"], +) +def test_an_axis_period_is_forwarded_unchanged(period) -> None: + parts = Parts() + axes = [[{**AXES[0][0], "period": period}]] + + parts.service.submit_calculation(calculation_input(axes=axes)) + + assert parts.simulation.submissions[0]["options"]["axes"] == axes diff --git a/tests/unit/v2/test_household_calculation_simulation_client.py b/tests/unit/v2/test_household_calculation_simulation_client.py new file mode 100644 index 000000000..8d84b8c8f --- /dev/null +++ b/tests/unit/v2/test_household_calculation_simulation_client.py @@ -0,0 +1,289 @@ +"""The simulation service's household calculation routes, through its client. + +These tests need a simulation client with the household calculation routes. +""" + +from __future__ import annotations + +import json +from uuid import uuid4 + +import httpx +import pytest + +from policyengine_api.libs.gateway_auth import GatewayAuthError +from policyengine_api.services.v2.household_calculations.simulation_client import ( + SimulationHouseholdClient, +) +from policyengine_api.services.v2.household_calculations.types import ( + SimulationServiceInvalidRequestError, + SimulationServiceRefusedError, + SimulationServiceUnavailableError, +) +from tests.unit.v2.household_calculation_support import ( + CALCULATION_ID, + HOUSEHOLD_ID, + POLICYENGINE_VERSION, + simulation_calculation, + succeeded_calculation, +) + +BASE_URL = "https://simulation.example" +SUBMISSION = { + "country": "us", + "policyengine_version": POLICYENGINE_VERSION, + "household_id": str(HOUSEHOLD_ID), + "policy_id": None, + "options": { + "axes": [[{"name": "employment_income", "count": 3, "min": 0, "max": 1}]], + "add_missing_variables": False, + }, + "force": False, + "trigger": "initial", +} + + +def client_answering( + status_code: int, payload: object, requests: list +) -> SimulationHouseholdClient: + """Return a client whose requests get one fixed answer.""" + + def answer(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(status_code, json=payload) + + return SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(answer)} + ) + + +def test_a_submission_is_posted_and_its_calculation_returned() -> None: + requests: list[httpx.Request] = [] + client = client_answering(202, simulation_calculation(), requests) + + calculation = client.submit(SUBMISSION) + + assert calculation == simulation_calculation() + (request,) = requests + assert request.method == "POST" + assert str(request.url) == f"{BASE_URL}/v2/household-calculations" + assert json.loads(request.content) == SUBMISSION + + +def test_a_calculation_is_read_by_id() -> None: + requests: list[httpx.Request] = [] + client = client_answering(200, succeeded_calculation(), requests) + + assert client.read(CALCULATION_ID) == succeeded_calculation() + assert str(requests[0].url) == ( + f"{BASE_URL}/v2/household-calculations/{CALCULATION_ID}" + ) + + +def test_an_unknown_calculation_reads_as_none() -> None: + client = client_answering(404, {"detail": "No such household calculation."}, []) + + assert client.read(uuid4()) is None + + +def test_a_refusal_carries_the_services_reason() -> None: + client = client_answering( + 400, {"detail": "PolicyEngine.py 9.9.9 has no deployed v2 worker"}, [] + ) + + with pytest.raises(SimulationServiceRefusedError, match=r"9\.9\.9"): + client.submit(SUBMISSION) + + +@pytest.mark.parametrize("status_code", [500, 503]) +def test_a_service_that_cannot_answer_is_unavailable(status_code: int) -> None: + client = client_answering(status_code, {"detail": "unavailable"}, []) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +def test_an_unreachable_service_is_unavailable() -> None: + def refuse(request: httpx.Request) -> httpx.Response: + message = "refused" + raise httpx.ConnectError(message, request=request) + + client = SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(refuse)} + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +GATEWAY_AUTH_VARIABLES = ( + "GATEWAY_AUTH_ISSUER", + "GATEWAY_AUTH_AUDIENCE", + "GATEWAY_AUTH_CLIENT_ID", + "GATEWAY_AUTH_CLIENT_SECRET", + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE", + "GATEWAY_AUTH_REQUIRED", +) + + +@pytest.mark.parametrize( + "environment", + [ + {}, + {"SIMULATION_ENTRYPOINT_URL": "not a url"}, + { + "SIMULATION_ENTRYPOINT_URL": BASE_URL, + "GATEWAY_AUTH_ISSUER": "https://issuer.example/", + }, + {"SIMULATION_ENTRYPOINT_URL": BASE_URL, "GATEWAY_AUTH_REQUIRED": "true"}, + ], + ids=["no-url", "invalid-url", "partial-auth", "auth-required"], +) +def test_a_misconfigured_service_is_unavailable( + monkeypatch: pytest.MonkeyPatch, environment: dict[str, str] +) -> None: + for name in ("SIMULATION_ENTRYPOINT_URL", *GATEWAY_AUTH_VARIABLES): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + with pytest.raises(SimulationServiceUnavailableError, match="not configured"): + SimulationHouseholdClient.from_environment() + + +def test_the_configured_service_is_called_without_auth_when_none_is_set( + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name in GATEWAY_AUTH_VARIABLES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("SIMULATION_ENTRYPOINT_URL", f"{BASE_URL}/") + + client = SimulationHouseholdClient.from_environment() + + assert client.base_url == BASE_URL + + +STORED_HOUSEHOLD_REFUSAL = ( + "the household 00000000-0000-4000-8000-0000000000a1 cannot be calculated: " + "benunits.benunit-2.members: List should have at least 1 item after validation" +) + + +def test_caller_data_the_service_refuses_is_reported_with_its_reason() -> None: + """The service refuses a stored household or policy value with 422 and text.""" + client = client_answering(422, {"detail": STORED_HOUSEHOLD_REFUSAL}, []) + + with pytest.raises( + SimulationServiceInvalidRequestError, match=r"benunit-2\.members" + ): + client.submit(SUBMISSION) + + +def test_request_fields_the_service_refuses_are_reported_one_by_one() -> None: + client = client_answering( + 422, + { + "detail": [ + { + "loc": ["body", "options", "axes", 0, 0, "count"], + "msg": "Input should be greater than or equal to 1", + "type": "greater_than_equal", + } + ] + }, + [], + ) + + with pytest.raises( + SimulationServiceInvalidRequestError, + match=r"options\.axes\.0\.0\.count: Input should be greater", + ): + client.submit(SUBMISSION) + + +@pytest.mark.parametrize("status_code", [400, 503, 202, 200]) +def test_an_answer_that_is_not_json_is_unavailable(status_code: int) -> None: + def answer(request: httpx.Request) -> httpx.Response: + return httpx.Response( + status_code, + content=b"Service Unavailable", + headers={"Content-Type": "text/html"}, + ) + + client = SimulationHouseholdClient( + BASE_URL, httpx_args={"transport": httpx.MockTransport(answer)} + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +def test_an_answer_of_the_wrong_shape_is_unavailable() -> None: + client = client_answering(202, {"calculation_id": "not a calculation"}, []) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + + +def test_a_gateway_token_that_cannot_be_fetched_is_unavailable() -> None: + class FailingTokenAuth(httpx.Auth): + def auth_flow(self, request: httpx.Request): # httpx's generator protocol + message = "The gateway token request failed" + raise GatewayAuthError(message) + yield request + + client = SimulationHouseholdClient( + BASE_URL, + httpx_args={ + "auth": FailingTokenAuth(), + "transport": httpx.MockTransport( + lambda request: httpx.Response(202, json=simulation_calculation()) + ), + }, + ) + + with pytest.raises(SimulationServiceUnavailableError): + client.submit(SUBMISSION) + with pytest.raises(SimulationServiceUnavailableError): + client.read(CALCULATION_ID) + + +def test_the_household_keeps_its_key_order_as_answered() -> None: + requests: list[httpx.Request] = [] + answer = succeeded_calculation() + client = client_answering(200, answer, requests) + submit_client = client_answering(202, answer, requests) + + submitted = submit_client.submit(SUBMISSION) + read = client.read(CALCULATION_ID) + + assert json.loads(requests[0].content) == SUBMISSION + for calculation in (submitted, read): + assert calculation is not None + returned_household = calculation["result"]["household"] + assert list(returned_household) == list(answer["result"]["household"]) + assert list(returned_household["households"]["household-1"]) == list( + answer["result"]["household"]["households"]["household-1"] + ) + + +@pytest.mark.parametrize( + "body", + [ + {"detail": None}, + {"detail": {"reason": "x"}}, + {"detail": 7}, + {"detail": ["the household is refused"]}, + {"detail": [{"loc": ["body", "options"], "msg": "Field required"}]}, + ], + ids=["null", "object", "number", "list-of-strings", "entry-without-type"], +) +def test_a_refusal_of_another_shape_is_unavailable(body) -> None: + client = client_answering(422, body, []) + + with pytest.raises(SimulationServiceUnavailableError, match="could not be read"): + client.submit(SUBMISSION) diff --git a/tests/unit/v2/test_household_models.py b/tests/unit/v2/test_household_models.py index f0798e35e..bef59c843 100644 --- a/tests/unit/v2/test_household_models.py +++ b/tests/unit/v2/test_household_models.py @@ -18,6 +18,7 @@ User, UserHouseholdAssociation, ) +from tests.fixtures.v2_countries import add_listed_countries def _engine(): @@ -30,6 +31,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 7b753004d..af6913897 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 39 +assert len(V2_METADATA.tables) == 40 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_live_household_parity_households.py b/tests/unit/v2/test_live_household_parity_households.py new file mode 100644 index 000000000..d66ebd6ea --- /dev/null +++ b/tests/unit/v2/test_live_household_parity_households.py @@ -0,0 +1,64 @@ +"""The staging parity set's households calculate the same year on both sides. + +API v1's ``/calculate-full`` adds missing variables for the first age period +of a person named ``you``, or else the current year; the v2 household +function adds them for the stored household's default year. A parity +household that adds missing variables must therefore name its person +``you``, or the two sides differ once the calendar year moves on. +""" + +from __future__ import annotations + +import json +from datetime import date + +import httpx + +from policyengine_api.services import household_calculation_service +from tests.fixtures.integration.live_household_calculations import store_household +from tests.integration.test_live_v2_household_parity import parity_households + + +class DateIn2027(date): + """Today is a day in 2027, a year after the households' year.""" + + @classmethod + def today(cls) -> DateIn2027: + """Return 2027-06-01.""" + return cls(2027, 6, 1) + + +def test_households_that_add_variables_use_one_year_on_both_sides( + monkeypatch, +) -> None: + monkeypatch.setattr(household_calculation_service, "date", DateIn2027) + adding = [ + household + for household in parity_households(run_amount=0) + if household.add_missing_variables + ] + + assert adding + for household in adding: + v1_year = household_calculation_service.get_household_year( + household.situation() + ) + # The v2 side calculates the default year the household is stored with. + assert str(v1_year) == str(_stored_default_year(household)), household.name + + +def _stored_default_year(household) -> int: + """Return the default year the live check stores the household with.""" + sent: list[dict] = [] + + def answer(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response(201, json={"result": {"item": {"id": "stored"}}}) + + with httpx.Client( + base_url="https://public-api.example", + transport=httpx.MockTransport(answer), + ) as client: + store_household(client, household) + (request,) = sent + return request["default_year"] diff --git a/tests/unit/v2/test_metadata_deployment.py b/tests/unit/v2/test_metadata_deployment.py index c752b46c0..1784a108a 100644 --- a/tests/unit/v2/test_metadata_deployment.py +++ b/tests/unit/v2/test_metadata_deployment.py @@ -54,6 +54,12 @@ def test_reusable_seeding_workflow_separates_database_credentials() -> None: assert "V2_MIGRATION_DATABASE_URL" not in publication assert "V2_RUNTIME_DATABASE_URL" not in workflow assert "scripts/publish_v2_metadata_catalog.py" in publication + # Publication records each version's SPM availability from the + # environment's own simulation service. + assert ( + "SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + in publication + ) assert (REPO / "scripts/publish_v2_metadata_catalog.py").is_file() assert not (REPO / "scripts/initialize_v2_metadata.py").exists() @@ -171,3 +177,12 @@ def test_stage_9_deployment_shell_script_is_syntax_valid() -> None: ) assert result.returncode == 0, result.stderr + + +def test_only_catalog_publication_receives_the_hugging_face_read_token() -> None: + workflow = _read(".github/workflows/seed-v2-database.yml") + publication = _step(workflow, "Seed and validate the v2 metadata catalog") + token = "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}" + + assert token in publication + assert workflow.count("HUGGING_FACE_TOKEN") == 1 diff --git a/tests/unit/v2/test_metadata_response_cache.py b/tests/unit/v2/test_metadata_response_cache.py new file mode 100644 index 000000000..a0a788c1c --- /dev/null +++ b/tests/unit/v2/test_metadata_response_cache.py @@ -0,0 +1,268 @@ +"""The process-wide store of whole catalog-version responses.""" + +from __future__ import annotations + +import threading +import time + +import pytest +from starlette.responses import JSONResponse + +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + BuiltResponse, + KeptResponse, + MetadataResponseCache, + body_entity_tag, +) + +RUNNING_VERSION = "6.10.0" + + +def _counting_build( + builds: list[str], *, body: bytes = b'{"a":1}', version: str = RUNNING_VERSION +): + def build() -> BuiltResponse: + builds.append("built") + return BuiltResponse(body=body, policyengine_version=version) + + return build + + +def test_a_response_is_built_once_and_kept_with_its_entity_tag() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + first = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", RUNNING_VERSION, _counting_build(builds, body=b"{}") + ) + + assert first == KeptResponse( + body=b'{"a":1}', entity_tag=body_entity_tag(b'{"a":1}') + ) + assert second is first + assert builds == ["built"] + + +@pytest.mark.parametrize("requested_first", [None, RUNNING_VERSION]) +def test_the_running_version_and_its_number_share_one_kept_response( + requested_first: str | None, +) -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + requested_second = RUNNING_VERSION if requested_first is None else None + + first = cache.get_or_build( + "variables", "us", requested_first, _counting_build(builds) + ) + second = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + third = cache.get_or_build( + "variables", "us", requested_second, _counting_build(builds) + ) + + assert second is first + assert third is first + assert builds == (["built"] if requested_first is None else ["built", "built"]) + + +def test_different_routes_countries_and_versions_are_kept_apart() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + + for route, country_id, version in ( + ("variables", "us", "6.10.0"), + ("variables", "uk", "6.10.0"), + ("variables", "us", "6.9.0"), + ("model-catalog", "us", "6.10.0"), + ): + cache.get_or_build( + route, country_id, version, _counting_build(builds, version=version) + ) + + assert len(builds) == 4 + + +def test_a_failed_read_is_returned_but_not_kept() -> None: + cache = MetadataResponseCache() + failure = JSONResponse({"status": "error"}, status_code=503) + builds: list[str] = [] + + def failing_build() -> JSONResponse: + builds.append("failed") + return failure + + first = cache.get_or_build("variables", "us", None, failing_build) + second = cache.get_or_build("variables", "us", None, _counting_build(builds)) + + assert first is failure + assert isinstance(second, KeptResponse) + assert builds == ["failed", "built"] + + +def test_the_least_recently_used_response_is_dropped_beyond_capacity() -> None: + cache = MetadataResponseCache(capacity=2) + builds: list[str] = [] + + def get(country_id: str) -> None: + cache.get_or_build("variables", country_id, "6.10.0", _counting_build(builds)) + + for country_id in ("a", "b", "a", "c", "a", "b"): + get(country_id) + + assert builds == ["built", "built", "built", "built"] + + +def test_concurrent_requests_for_one_response_build_it_once() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + building = threading.Event() + release = threading.Event() + + def blocked_build() -> BuiltResponse: + builds.append("built") + building.set() + release.wait() + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + def request() -> None: + cache.get_or_build("variables", "us", None, blocked_build) + + first = threading.Thread(target=request) + first.start() + building.wait() + others = [threading.Thread(target=request) for _ in range(3)] + for thread in others: + thread.start() + release.set() + for thread in [first, *others]: + thread.join() + + # Whether the others waited on the build or arrived after it, none read. + assert builds == ["built"] + + +def test_a_request_arriving_while_a_response_is_stored_waits_for_it() -> None: + cache = MetadataResponseCache() + builds: list[str] = [] + second_results: list[object] = [] + second_requests: list[threading.Thread] = [] + original_keep = cache._keep + + def keep_after_another_request_arrives(*keep_arguments: object) -> KeptResponse: + second_request = threading.Thread( + target=lambda: second_results.append( + cache.get_or_build("variables", "us", None, _counting_build(builds)) + ) + ) + second_requests.append(second_request) + second_request.start() + # The second request may not build while this one stores its result: + # it waits on the build, whatever the timing. + second_request.join(timeout=0.2) + return original_keep(*keep_arguments) + + cache._keep = keep_after_another_request_arrives # type: ignore[method-assign] + + first = cache.get_or_build("variables", "us", None, _counting_build(builds)) + second_requests[0].join() + + assert builds == ["built"] + assert second_results == [first] + + +def _wait_for_waiters(cache: MetadataResponseCache, key: tuple, count: int) -> None: + # Polls until every request has registered as waiting on the read in + # progress, so the result does not depend on thread timing. + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + with cache._lock: + in_flight = cache._builds.get(key) + if in_flight is not None and in_flight.waiters == count: + return + time.sleep(0.001) + pytest.fail(f"{count} requests did not wait on the read in progress") + + +@pytest.mark.parametrize("read_outcome", ["failure", "success"]) +def test_waiting_requests_receive_the_outcome_of_the_one_read( + read_outcome: str, +) -> None: + read_fails = read_outcome == "failure" + cache = MetadataResponseCache() + builds: list[str] = [] + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> BuiltResponse | JSONResponse: + builds.append("built") + release.wait() + if read_fails: + return JSONResponse({"status": "error"}, status_code=503) + return BuiltResponse(body=b"{}", policyengine_version=RUNNING_VERSION) + + results: list[object] = [] + + def request() -> None: + results.append(cache.get_or_build(*key, blocked_build)) + + owner = threading.Thread(target=request, daemon=True) + waiters = [threading.Thread(target=request, daemon=True) for _ in range(9)] + try: + owner.start() + while not builds: + time.sleep(0.001) + for thread in waiters: + thread.start() + _wait_for_waiters(cache, key, 9) + finally: + release.set() + for thread in [owner, *waiters]: + thread.join(timeout=10) + + assert builds == ["built"] + assert len(results) == 10 + if read_fails: + assert all(result.status_code == 503 for result in results) + assert {bytes(result.body) for result in results} == {b'{"status":"error"}'} + # Each request gets its own response object, so middleware headers + # added for one request cannot reach another. + assert len({id(result) for result in results}) == 10 + else: + assert all(result is results[0] for result in results) + assert key not in cache._builds + + +def test_a_finished_read_leaves_a_newer_reads_entry_in_place() -> None: + cache = MetadataResponseCache() + building = threading.Event() + release = threading.Event() + key = ("variables", "us", None) + + def blocked_build() -> JSONResponse: + building.set() + release.wait() + return JSONResponse({"status": "error"}, status_code=503) + + owner = threading.Thread( + target=lambda: cache.get_or_build(*key, blocked_build), daemon=True + ) + newer_read = object() + try: + owner.start() + building.wait(timeout=10) + with cache._lock: + cache._builds[key] = newer_read + finally: + release.set() + owner.join(timeout=10) + + assert cache._builds[key] is newer_read + + +def test_capacity_must_be_positive() -> None: + with pytest.raises(ValueError, match="capacity"): + MetadataResponseCache(capacity=0) diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..701e2ff2d 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -20,17 +20,30 @@ from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.fastapi_routes import dependencies as route_dependencies from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, +) from policyengine_api.migration_flags import ( RouteImplementation, RouteImplementationSettings, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.types import ( MetadataCanonicalParameterValue, + MetadataCatalogParameter, + MetadataCatalogParametersResult, + MetadataCatalogVariablesResult, MetadataDataset, MetadataDatasetOption, + MetadataDatedValue, MetadataDetailResult, + MetadataEconomyOptions, MetadataEconomyOptionsResult, + MetadataEntity, MetadataModel, + MetadataModelCatalogResult, MetadataModelSelectionResult, MetadataModelVersionDetail, MetadataPageResult, @@ -38,8 +51,10 @@ MetadataParameterSummary, MetadataRegion, MetadataRegionOption, + MetadataSPMAvailability, MetadataTimePeriodOption, MetadataVariable, + ParameterFilter, ) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, @@ -112,6 +127,14 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: default_value=0, adds=None, subtracts=None, + is_input_variable=True, + hidden_input=False, + module_name=None, + adds_parameter_path=None, + subtracts_parameter_path=None, + enum_values=None, + unit="currency-USD", + definition_period="year", ) parameter = MetadataParameterSummary( id=parameter_id, @@ -120,6 +143,8 @@ def _resource_results() -> tuple[dict[str, object], dict[str, object]]: description=None, data_type="float", unit="/1", + economy=True, + household=False, ) parameter_value = MetadataCanonicalParameterValue( id=parameter_value_id, @@ -193,6 +218,68 @@ def detail(item: object) -> MetadataDetailResult: "list_regions": page([region]), "get_region_by_code": detail(region), "get_region": detail(region), + "get_catalog_parameters": MetadataCatalogParametersResult( + policyengine_version=version, + catalog_format=2, + parameters=[ + MetadataCatalogParameter( + **parameter.model_dump(), + values=[ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=0.1, + ) + ], + ) + ], + node_labels={"gov.example": "Example policy"}, + missing_names=["gov.removed"], + ), + "get_catalog_variables": MetadataCatalogVariablesResult( + policyengine_version=version, + catalog_format=2, + variables=[variable], + part_parameter_values={ + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income"], + ) + ] + }, + ), + "get_model_catalog": MetadataModelCatalogResult( + policyengine_version=version, + catalog_format=2, + model=model, + model_version=model_version, + entities=[ + MetadataEntity(key="person", plural="people", is_person=True) + ], + basic_inputs=["employment_income"], + regions=[region], + datasets=[dataset], + economy_options=MetadataEconomyOptions( + region=[ + MetadataRegionOption( + name="us", label="United States", type="national" + ) + ], + time_period=[MetadataTimePeriodOption(name=2026, label="2026")], + datasets=[ + MetadataDatasetOption( + name="populace_us_2024", label="Microcosm" + ) + ], + ), + spm=MetadataSPMAvailability( + available=True, + settings_schema={"type": "object"}, + defaults={"scenario": "ce_trend"}, + ), + ), "get_economy_options": MetadataEconomyOptionsResult( policyengine_version=version, current_law_id=2, @@ -221,7 +308,7 @@ def detail(item: object) -> MetadataDetailResult: ) -def _client(factory) -> TestClient: +def _client(factory, *, response_cache=None) -> TestClient: flask_app = Flask(__name__) @flask_app.get("//metadata") @@ -240,6 +327,7 @@ def v1_metadata(country_id: str): metadata_reader_factory=lambda: None, specification_provider=dict, v2_metadata_reader_factory=factory, + v2_metadata_response_cache=response_cache, ) settings = RouteImplementationSettings( health=RouteImplementation.FLASK_FALLBACK, @@ -443,12 +531,17 @@ def test_default_reader_uses_the_installed_policyengine_version( captured = {} reader = object() - def query_service(candidate_session, *, running_policyengine_version): + def query_service( + candidate_session, *, running_policyengine_version, selected_catalogs + ): captured["session"] = candidate_session.session captured["version"] = running_policyengine_version + captured["selected_catalogs"] = selected_catalogs return reader - monkeypatch.setattr(database, "get_v2_session_factory", lambda: lambda: session) + monkeypatch.setattr( + database, "get_v2_read_session_factory", lambda: lambda: session + ) monkeypatch.setattr(metadata_services, "V2MetadataService", query_service) monkeypatch.setattr( route_dependencies.importlib_metadata, @@ -462,7 +555,12 @@ def query_service(candidate_session, *, running_policyengine_version): route_dependencies._running_policyengine_version.cache_clear() assert result is reader - assert captured == {"session": session, "version": "5.2.0"} + assert captured["session"] is session + assert captured["version"] == "5.2.0" + assert isinstance(captured["selected_catalogs"], SelectedCatalogStore) + assert ( + route_dependencies._process_selected_catalogs() is captured["selected_catalogs"] + ) def test_default_factory_and_close_failures_preserve_success( @@ -521,6 +619,9 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/datasets", "/v2/datasets/{dataset_id}", "/v2/economy-options", + "/v2/model-catalog", + "/v2/model-catalog/variables", + "/v2/model-catalog/parameters", "/v2/parameters", "/v2/parameters/children", "/v2/parameters/{parameter_id}", @@ -538,8 +639,11 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/variables/{variable_id}", } native_paths = { + "/v2/countries", "/v2/households", "/v2/households/{household_id}", + "/v2/household-calculations", + "/v2/household-calculations/{calculation_id}", "/v2/policies", "/v2/policies/{policy_id}", "/v2/user-households", @@ -565,21 +669,821 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: for path in metadata_paths: operation = schema["paths"][path]["get"] - assert set(operation["responses"]) >= { - "200", - "400", - "404", - "405", - "422", - "500", - "503", - } + error_statuses = ["404", "405", "422", "500", "503"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + error_statuses.append("400") + assert set(operation["responses"]) >= {"200", *error_statuses} success_schema = operation["responses"]["200"]["content"]["application/json"][ "schema" ] assert success_schema["$ref"].startswith("#/components/schemas/Metadata") - for status in ("400", "404", "405", "422", "500", "503"): + for status in error_statuses: error_schema = operation["responses"][status]["content"][ "application/json" ]["schema"] assert error_schema["$ref"] == "#/components/schemas/V2ErrorResponse" + + +def _every_resource_path(ids: dict[str, object]) -> list[str]: + return [ + "/v2/tax-benefit-models?country_id=us", + "/v2/tax-benefit-models/by-country/us", + f"/v2/tax-benefit-models/{ids['model_id']}?country_id=us", + "/v2/tax-benefit-model-versions?country_id=us", + f"/v2/tax-benefit-model-versions/{ids['model_version_id']}?country_id=us", + "/v2/variables?country_id=us", + f"/v2/variables/{ids['variable_id']}?country_id=us", + "/v2/parameters?country_id=us", + "/v2/parameters/children?country_id=us&parent_path=gov.example", + f"/v2/parameters/{ids['parameter_id']}?country_id=us", + "/v2/parameter-values?country_id=us", + f"/v2/parameter-values/{ids['parameter_value_id']}?country_id=us", + "/v2/datasets?country_id=us", + f"/v2/datasets/{ids['dataset_id']}?country_id=us", + "/v2/regions?country_id=us", + "/v2/regions/by-code/us?country_id=us", + f"/v2/regions/{ids['region_id']}?country_id=us", + "/v2/economy-options?country_id=us", + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/parameters?country_id=us&name=gov.example.rate", + ] + + +def test_every_resource_route_serves_an_entity_tag_and_cache_lifetime() -> None: + results, ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + + for path in _every_resource_path(ids): + response = client.get(path) + assert response.status_code == 200, path + assert response.headers["cache-control"] == "public, max-age=300", path + assert response.headers["etag"].startswith('"'), path + assert response.headers["etag"].endswith('"'), path + + +def test_a_named_catalog_version_is_reusable_for_a_day() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + query = {"country_id": "us", "policyengine_version": "4.20.3"} + + first = client.get("/v2/variables", params=query) + second = client.get("/v2/variables", params=query) + + assert first.headers["cache-control"] == "public, max-age=86400" + assert first.headers["etag"] == second.headers["etag"] + + +def test_a_matching_entity_tag_gets_a_not_modified_response_without_a_body() -> None: + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + path = "/v2/variables?country_id=us&policyengine_version=4.20.3" + entity_tag = client.get(path).headers["etag"] + + revalidated = client.get(path, headers={"If-None-Match": entity_tag}) + weakly_revalidated = client.get(path, headers={"If-None-Match": f"W/{entity_tag}"}) + listed = client.get(path, headers={"If-None-Match": f'"other", {entity_tag}'}) + stale = client.get(path, headers={"If-None-Match": '"other"'}) + + for not_modified in (revalidated, weakly_revalidated, listed): + assert not_modified.status_code == 304 + assert not_modified.content == b"" + assert not_modified.headers["etag"] == entity_tag + assert not_modified.headers["cache-control"] == "public, max-age=86400" + assert stale.status_code == 200 + assert stale.json()["status"] == "ok" + + +def test_different_content_gets_a_different_entity_tag() -> None: + results, _ids = _resource_results() + changed_results, _changed_ids = _resource_results() + + first = _client(lambda: ResourceReader(results)).get("/v2/variables?country_id=us") + changed = _client(lambda: ResourceReader(changed_results)).get( + "/v2/variables?country_id=us" + ) + + assert first.headers["etag"] != changed.headers["etag"] + + +def test_failed_reads_carry_no_cache_headers() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get( + "/v2/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert response.status_code == 404 + assert "etag" not in response.headers + assert "cache-control" not in response.headers + + +def test_model_catalog_route_serves_the_small_parts_of_the_selected_version() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", params={"country_id": "US"} + ) + + assert response.status_code == 200 + result = response.json()["result"] + assert result["policyengine_version"] == "4.20.3" + assert result["catalog_format"] == 2 + assert result["entities"] == [ + {"key": "person", "plural": "people", "is_person": True} + ] + assert result["basic_inputs"] == ["employment_income"] + assert [region["code"] for region in result["regions"]] == ["us"] + assert result["economy_options"]["datasets"][0]["default"] is True + assert result["spm"] == { + "available": True, + "settings_schema": {"type": "object"}, + "defaults": {"scenario": "ce_trend"}, + } + assert reader.calls == [("get_model_catalog", ("us", None), {})] + assert reader.closed + + +def test_model_catalog_route_names_an_explicit_version_for_a_day() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog", + params={"country_id": "uk", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + assert reader.calls == [("get_model_catalog", ("uk", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + [ + "", + "country_id=ca", + "country_id=us&country_id=uk", + "country_id=us&limit=10", + "country_id=us&policyengine_version=latest", + ], +) +def test_model_catalog_route_rejects_invalid_queries(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog?{query}") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert calls == [] + + +def test_model_catalog_route_declares_its_query_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version"} + assert parameters["country_id"]["required"] is True + assert parameters["policyengine_version"]["required"] is False + success = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert success == {"$ref": "#/components/schemas/MetadataModelCatalogResponse"} + + +def test_catalog_variables_route_serves_every_variable_and_part_values() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/variables", + params={"country_id": "us", "policyengine_version": "4.20.3"}, + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["catalog_format"] == 2 + assert [variable["name"] for variable in result["variables"]] == [ + "employment_income" + ] + assert result["variables"][0]["module_name"] is None + assert result["part_parameter_values"] == { + "gov.example.parts": [ + { + "start_date": "2025-01-01T00:00:00Z", + "end_date": None, + "value": ["employment_income"], + } + ] + } + assert reader.calls == [("get_catalog_variables", ("us", "4.20.3"), {})] + + +@pytest.mark.parametrize( + "query", + ["", "country_id=us&offset=0", "country_id=us&search=income"], +) +def test_catalog_variables_route_rejects_paging_and_unknown_queries( + query: str, +) -> None: + results, _ids = _resource_results() + response = _client(lambda: ResourceReader(results)).get( + f"/v2/model-catalog/variables?{query}" + ) + + assert response.status_code == 422 + + +def test_catalog_parameters_route_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/model-catalog/parameters" + "?country_id=us&policyengine_version=4.20.3" + "&name=gov.example.rate&name=gov.removed" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=86400" + result = response.json()["result"] + assert result["parameters"][0]["name"] == "gov.example.rate" + assert result["parameters"][0]["values"] == [ + {"start_date": "2025-01-01T00:00:00Z", "end_date": None, "value": 0.1} + ] + assert result["node_labels"] == {"gov.example": "Example policy"} + assert result["missing_names"] == ["gov.removed"] + assert reader.calls == [ + ( + "get_catalog_parameters", + ("us", "4.20.3"), + {"names": ["gov.example.rate", "gov.removed"]}, + ) + ] + + +@pytest.mark.parametrize( + "query", + [ + "country_id=us", + "country_id=us&name=", + "country_id=us&name=gov.example.rate&offset=0", + "country_id=us&" + "&".join(f"name=gov.p{index}" for index in range(101)), + ], +) +def test_catalog_parameters_route_rejects_invalid_name_lists(query: str) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(f"/v2/model-catalog/parameters?{query}") + + assert response.status_code == 422 + assert calls == [] + + +def test_catalog_parameters_route_declares_a_repeated_name_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + operation = schema["paths"]["/v2/model-catalog/parameters"]["get"] + parameters = {parameter["name"]: parameter for parameter in operation["parameters"]} + assert set(parameters) == {"country_id", "policyengine_version", "name"} + assert parameters["name"]["required"] is True + name_schema = parameters["name"]["schema"] + assert name_schema["type"] == "array" + assert name_schema["minItems"] == 1 + assert name_schema["maxItems"] == 100 + + +def test_parameter_collection_fetches_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + client = _client(lambda: reader) + + named = client.get( + "/v2/parameters?country_id=US&name=gov.example.rate&name=gov.other.rate" + ) + paged = client.get("/v2/parameters?country_id=us&offset=5&limit=50&search=rate") + + assert named.status_code == 200 + assert paged.status_code == 200 + assert reader.calls == [ + ( + "list_parameters", + ("us", None), + { + "offset": 0, + "limit": 100, + "parameter_filter": ParameterFilter( + search=None, names=("gov.example.rate", "gov.other.rate") + ), + }, + ), + ( + "list_parameters", + ("us", None), + { + "offset": 5, + "limit": 50, + "parameter_filter": ParameterFilter(search="rate", names=None), + }, + ), + ] + + +def test_parameter_values_fetch_several_parameters_by_name() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3" + "¶meter_name=gov.example.rate¶meter_name=gov.other.rate¤t=true" + ) + + assert response.status_code == 200 + name, args, kwargs = reader.calls[0] + assert (name, args) == ("list_parameter_values", ("us", "4.20.3")) + assert kwargs["value_filter"].parameter_names == ( + "gov.example.rate", + "gov.other.rate", + ) + assert kwargs["value_filter"].parameter_id is None + assert kwargs["value_filter"].current is True + assert (kwargs["offset"], kwargs["limit"]) == (0, 100) + + +@pytest.mark.parametrize( + "path", + [ + "/v2/parameters?country_id=us¶meter_id=1", + "/v2/parameters?country_id=us&limit=5&limit=6", + "/v2/parameters?country_id=us&name=", + "/v2/parameters?country_id=ca", + "/v2/parameters?country_id=us&" + + "&".join(f"name=gov.p{index}" for index in range(101)), + "/v2/parameter-values?country_id=us&name=gov.example.rate", + "/v2/parameter-values?country_id=us¤t=true¤t=false", + "/v2/parameter-values?country_id=us¶meter_id=not-a-uuid", + ], +) +def test_parameter_collections_reject_unknown_repeated_and_invalid_queries( + path: str, +) -> None: + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + response = _client(factory).get(path) + + assert response.status_code == 422 + assert calls == [] + + +def test_parameter_collections_declare_their_name_filters_in_openapi() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + def query_parameters(path: str) -> dict[str, dict]: + return { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + } + + parameters = query_parameters("/v2/parameters") + values = query_parameters("/v2/parameter-values") + assert set(parameters) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "search", + "name", + } + assert parameters["name"]["required"] is False + assert set(values) == { + "country_id", + "policyengine_version", + "offset", + "limit", + "parameter_id", + "parameter_name", + "current", + } + assert values["parameter_name"]["required"] is False + + +def test_whole_version_responses_are_read_once_per_process_and_version() -> None: + results, _ids = _resource_results() + readers: list[ResourceReader] = [] + + def factory() -> ResourceReader: + readers.append(ResourceReader(results)) + return readers[-1] + + client = _client(factory, response_cache=MetadataResponseCache()) + paths = [ + "/v2/model-catalog?country_id=us", + "/v2/model-catalog/variables?country_id=us", + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3", + "/v2/model-catalog/variables?country_id=uk", + ] + + first_responses = [client.get(path) for path in paths] + repeated_responses = [client.get(path) for path in paths] + + # The explicit request names the running version (4.20.3), so it shares + # the response the request without a version read: three reads in all. + assert len(readers) == len(paths) - 1 + assert first_responses[2].content == first_responses[1].content + for first, repeated in zip(first_responses, repeated_responses, strict=True): + assert repeated.status_code == 200 + assert repeated.content == first.content + assert repeated.headers["etag"] == first.headers["etag"] + assert repeated.headers["cache-control"] == first.headers["cache-control"] + assert all(reader.closed for reader in readers) + + +def test_failed_whole_version_reads_are_retried() -> None: + results, _ids = _resource_results() + readers = [ + ResourceReader(results, error=MetadataCatalogUnavailableError("down")), + ResourceReader(results), + ] + client = _client(lambda: readers.pop(0), response_cache=MetadataResponseCache()) + + failed = client.get("/v2/model-catalog/variables?country_id=us") + recovered = client.get("/v2/model-catalog/variables?country_id=us") + + assert failed.status_code == 503 + assert "etag" not in failed.headers + assert recovered.status_code == 200 + assert readers == [] + + +def test_default_dependencies_share_one_response_cache() -> None: + first = route_dependencies.NativeRouteDependencies.defaults() + second = route_dependencies.NativeRouteDependencies.defaults() + + assert isinstance(first.v2_metadata_response_cache, MetadataResponseCache) + assert first.v2_metadata_response_cache is second.v2_metadata_response_cache + + +@pytest.mark.parametrize( + ("now", "query", "expected_cache_control"), + [ + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=true", + "public, max-age=3600", + ), + ( + datetime(2026, 12, 31, 23, 58, tzinfo=UTC), + "¤t=true", + "public, max-age=120", + ), + ( + datetime(2026, 6, 1, 12, 0, tzinfo=UTC), + "¤t=true", + "public, max-age=300", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3¤t=false", + "public, max-age=86400", + ), + ( + datetime(2026, 12, 31, 23, 0, tzinfo=UTC), + "&policyengine_version=4.20.3", + "public, max-age=86400", + ), + ], +) +def test_current_parameter_values_are_reused_only_until_the_next_utc_day( + monkeypatch: pytest.MonkeyPatch, + now: datetime, + query: str, + expected_cache_control: str, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + monkeypatch.setattr(common, "utc_now", lambda: now) + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + f"/v2/parameter-values?country_id=us{query}" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == expected_cache_control + + +def test_a_kept_response_is_served_without_hashing_its_body_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common, response_cache + + hashed_bodies: list[int] = [] + original_entity_tag = response_cache.body_entity_tag + + def counting_entity_tag(body: bytes) -> str: + hashed_bodies.append(len(body)) + return original_entity_tag(body) + + monkeypatch.setattr(response_cache, "body_entity_tag", counting_entity_tag) + monkeypatch.setattr(common, "body_entity_tag", counting_entity_tag) + results, _ids = _resource_results() + client = _client( + lambda: ResourceReader(results), response_cache=MetadataResponseCache() + ) + + first = client.get("/v2/model-catalog/variables?country_id=us") + hashed_after_first = len(hashed_bodies) + second = client.get( + "/v2/model-catalog/variables?country_id=us&policyengine_version=4.20.3" + ) + + assert hashed_after_first == 1 + assert len(hashed_bodies) == 1 + assert second.headers["etag"] == first.headers["etag"] + assert second.content == first.content + + +def _query_rejections(path: str) -> list[str]: + separator = "&" if "?" in path else "?" + rejections = [ + f"{path}{separator}policyengine_version=not%20a%20version", + f"{path}{separator}unexpected=1", + f"{path}{separator}policyengine_version=4.20.3&policyengine_version=4.20.3", + ] + if "country_id=us" in path: + rejections.append(path.replace("country_id=us", "country_id=ca")) + rejections.append( + path.replace("country_id=us&", "").replace("?country_id=us", "") + ) + return rejections + + +def test_every_metadata_route_rejects_invalid_queries_before_reading() -> None: + _results, ids = _resource_results() + calls = [] + + def factory(): + calls.append("called") + results, _ids = _resource_results() + return ResourceReader(results) + + client = _client(factory) + for path in _every_resource_path(ids): + for rejected in _query_rejections(path): + response = client.get(rejected) + assert response.status_code == 422, (rejected, response.text) + assert response.json() == { + "status": "error", + "message": "Invalid API v2 request", + } + assert calls == [] + + +def test_every_metadata_route_declares_the_shared_catalog_query() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + parameters = { + parameter["name"]: parameter + for parameter in schema["paths"][path]["get"]["parameters"] + if parameter["in"] == "query" + } + version = parameters["policyengine_version"] + assert version["required"] is False, path + assert version["schema"]["anyOf"][0]["maxLength"] == 128, path + if path != "/v2/tax-benefit-models/by-country/{country_id}": + country = parameters["country_id"] + assert country["required"] is True, path + assert country["schema"]["enum"] == ["us", "uk"], path + + +def test_browser_scripts_can_read_a_metadata_response_entity_tag() -> None: + results, _ids = _resource_results() + + response = _client(lambda: ResourceReader(results)).get( + "/v2/model-catalog?country_id=us", + headers={"Origin": "https://app.policyengine.org"}, + ) + + assert response.status_code == 200 + assert response.headers["access-control-allow-origin"] == ( + "https://app.policyengine.org" + ) + exposed_headers = { + value.strip().lower() + for value in response.headers["access-control-expose-headers"].split(",") + } + assert "etag" in exposed_headers + + +def test_current_values_read_just_before_midnight_expire_at_that_midnight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The read and the response headers must use the same day: values read for + # 31 December may not be reused into 1 January, even when the headers are + # written after midnight. + from policyengine_api.fastapi_routes.v2.metadata import common + + read_instant = datetime(2026, 12, 31, 23, 59, 59, tzinfo=UTC) + instants = iter([read_instant, datetime(2027, 1, 1, 0, 0, 1, tzinfo=UTC)]) + monkeypatch.setattr(common, "utc_now", lambda: next(instants)) + results, _ids = _resource_results() + reader = ResourceReader(results) + + response = _client(lambda: reader).get( + "/v2/parameter-values?country_id=us&policyengine_version=4.20.3¤t=true" + ) + + assert response.status_code == 200 + assert response.headers["cache-control"] == "public, max-age=0" + _name, _args, kwargs = reader.calls[0] + assert kwargs["value_filter"].now == read_instant + + +def _vary_values(response) -> set[str]: + return { + value.strip().lower() + for value in response.headers.get("vary", "").split(",") + if value.strip() + } + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_metadata_responses_vary_by_origin(origin: str | None) -> None: + # The API reflects any origin, so a shared cache must not reuse a response + # stored for a request without Origin (no Access-Control-Allow-Origin) for + # a cross-origin one, or the reverse. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {} if origin is None else {"Origin": origin} + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + large = client.get("/v2/model-catalog?country_id=us", headers=headers) + revalidated = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": large.headers["etag"]}, + ) + + assert revalidated.status_code == 304 + for response in (small, large, revalidated): + assert "origin" in _vary_values(response) + assert "accept-encoding" in _vary_values(large) + + +def test_failed_metadata_reads_do_not_add_vary_by_origin() -> None: + results, _ids = _resource_results() + reader = ResourceReader(results, error=MetadataResourceNotFoundError("missing")) + + response = _client(lambda: reader).get("/v2/variables?country_id=us") + + assert response.status_code == 404 + assert "origin" not in _vary_values(response) + + +def test_openapi_declares_caching_headers_and_not_modified_responses() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + metadata_paths = [ + path + for path in schema["paths"] + if path.startswith( + ( + "/v2/datasets", + "/v2/economy-options", + "/v2/model-catalog", + "/v2/parameter", + "/v2/regions", + "/v2/tax-benefit-model", + "/v2/variables", + ) + ) + ] + + assert len(metadata_paths) == 21 + for path in metadata_paths: + responses = schema["paths"][path]["get"]["responses"] + assert set(responses["200"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "$ref" in responses["200"]["content"]["application/json"]["schema"] + assert set(responses["304"]["headers"]) == {"ETag", "Cache-Control", "Vary"} + assert "content" not in responses["304"] + if path == "/v2/tax-benefit-models/by-country/{country_id}": + assert "path" in responses["400"]["description"] + else: + # Query parameters are validated first (422), so only a country + # given in the path can return 400. + assert "400" not in responses, path + + +def _vary_value_counts(response) -> dict[str, int]: + counts: dict[str, int] = {} + for value in response.headers.get("vary", "").split(","): + if value.strip(): + counts[value.strip().lower()] = counts.get(value.strip().lower(), 0) + 1 + return counts + + +@pytest.mark.parametrize("origin", [None, "https://app.policyengine.org"]) +def test_cacheable_responses_and_their_304_carry_the_same_vary_once_each( + origin: str | None, +) -> None: + # Through the whole middleware stack (gzip, then CORS): a 304 must carry + # the Vary its 200 would, and each value appears once. + results, _ids = _resource_results() + client = _client(lambda: ResourceReader(results)) + headers = {"Accept-Encoding": "gzip"} + if origin is not None: + headers["Origin"] = origin + + small = client.get("/v2/tax-benefit-models?country_id=us", headers=headers) + compressed = client.get("/v2/model-catalog?country_id=us", headers=headers) + not_modified = client.get( + "/v2/model-catalog?country_id=us", + headers={**headers, "If-None-Match": compressed.headers["etag"]}, + ) + + assert compressed.headers.get("content-encoding") == "gzip" + assert not_modified.status_code == 304 + expected = {"origin": 1, "accept-encoding": 1} + assert _vary_value_counts(small) == expected + assert _vary_value_counts(compressed) == expected + assert _vary_value_counts(not_modified) == expected + + +def test_openapi_declares_each_variables_unit_and_definition_period() -> None: + results, _ids = _resource_results() + schema = _client(lambda: ResourceReader(results)).get("/v2/openapi.json").json() + + variable_properties = schema["components"]["schemas"]["MetadataVariable"][ + "properties" + ] + for field in ("unit", "definition_period"): + assert {"type": "string"} in variable_properties[field]["anyOf"] + assert {"type": "null"} in variable_properties[field]["anyOf"] + + +def test_openapi_requires_every_catalog_format_2_field_as_nullable() -> None: + # Every response includes these fields (null for versions published before + # catalog format 2), so a generated client must not type them as missing. + results, _ids = _resource_results() + schemas = ( + _client(lambda: ResourceReader(results)) + .get("/v2/openapi.json") + .json()["components"]["schemas"] + ) + format_2_fields = { + "MetadataVariable": { + "is_input_variable", + "hidden_input", + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + "enum_values", + "unit", + "definition_period", + }, + "MetadataParameterSummary": {"economy", "household"}, + "MetadataCatalogParameter": {"economy", "household"}, + } + + for schema_name, fields in format_2_fields.items(): + schema = schemas[schema_name] + assert set(schema["properties"]) <= set(schema["required"]), schema_name + for field in fields: + assert {"type": "null"} in schema["properties"][field]["anyOf"], field diff --git a/tests/unit/v2/test_metadata_service.py b/tests/unit/v2/test_metadata_service.py index 1f80d1b3f..6b6a19f32 100644 --- a/tests/unit/v2/test_metadata_service.py +++ b/tests/unit/v2/test_metadata_service.py @@ -33,8 +33,20 @@ from policyengine_api.services.v2.metadata.database_session import ( MetadataDatabaseSession, ) +from policyengine_api.services.v2.metadata.selected_catalogs import ( + SelectedCatalogStore, +) from policyengine_api.services.v2.metadata.services import V2MetadataService -from policyengine_api.services.v2.metadata.types import ParameterValueFilter +from policyengine_api.services.v2.metadata.transformations import ancestor_paths +from policyengine_api.services.v2.metadata.types import ( + CurrentLawValueRow, + MetadataDatedValue, + MetadataEntity, + MetadataEnumValue, + ParameterFilter, + ParameterNodeLabelRow, + ParameterValueFilter, +) from policyengine_api.services.v2.metadata.validators import ( InvalidMetadataPageError, MetadataResourceNotFoundError, @@ -74,6 +86,16 @@ def _insert_country( if time_periods is None else time_periods ), + catalog_format=country.model_version.catalog_format, + entities=[ + { + "key": entity.key, + "plural": entity.plural, + "is_person": entity.is_person, + } + for entity in country.model_version.entities + ], + basic_inputs=list(country.model_version.basic_inputs), ) ) session.add_all( @@ -89,6 +111,21 @@ def _insert_country( default_value=record.default_value, adds=record.adds, subtracts=record.subtracts, + is_input_variable=record.is_input_variable, + hidden_input=record.hidden_input, + module_name=record.module_name, + adds_parameter_path=record.adds_parameter_path, + subtracts_parameter_path=record.subtracts_parameter_path, + enum_values=( + None + if record.enum_values is None + else [ + {"value": enum_value.value, "label": enum_value.label} + for enum_value in record.enum_values + ] + ), + unit=record.unit, + definition_period=record.definition_period, ) for record in country.variables ) @@ -111,6 +148,8 @@ def _insert_country( description=record.description, data_type=record.data_type, unit=record.unit, + economy=record.economy, + household=record.household, ) for record in country.parameters ) @@ -196,28 +235,33 @@ def _add_country_version( session.commit() -def _us_model_version(session: Session) -> TaxBenefitModelVersion: +def _model_version( + session: Session, model_name: str, version: str = POLICYENGINE_VERSION +) -> TaxBenefitModelVersion: return session.exec( select(TaxBenefitModelVersion) .join(TaxBenefitModel, TaxBenefitModel.id == TaxBenefitModelVersion.model_id) .where( - TaxBenefitModel.name == "policyengine-us", - TaxBenefitModelVersion.version == POLICYENGINE_VERSION, + TaxBenefitModel.name == model_name, + TaxBenefitModelVersion.version == version, ) ).one() -def _service(session: Session) -> V2MetadataService: +def _service( + session: Session, *, selected_catalogs: SelectedCatalogStore | None = None +) -> V2MetadataService: return V2MetadataService( MetadataDatabaseSession(session), running_policyengine_version=POLICYENGINE_VERSION, + selected_catalogs=selected_catalogs, ) def test_resource_collection_uses_bounded_pagination_without_counting( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") catalog_session.add( Variable( id=uuid4(), @@ -309,7 +353,7 @@ def record_statement(_connection, _cursor, statement, *_args) -> None: def test_parameter_values_are_separate_canonical_resources( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") parameter = catalog_session.exec( select(Parameter).where( Parameter.tax_benefit_model_version_id == model_version.id, @@ -370,7 +414,7 @@ def test_current_parameter_value_uses_inclusive_effective_dates( def test_parameter_children_are_loaded_one_level_at_a_time( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") nested_node = ParameterNode( id=uuid4(), tax_benefit_model_version_id=model_version.id, @@ -421,7 +465,9 @@ def test_resource_filters_and_details_remain_inside_selected_catalog( ) -> None: service = _service(catalog_session) variables = service.list_variables("us", search="employment") - parameters = service.list_parameters("us", search="example rate") + parameters = service.list_parameters( + "us", parameter_filter=ParameterFilter(search="example rate") + ) states = service.list_regions("us", region_type="state") region = service.get_region_by_code("us", "state/ca") @@ -503,7 +549,7 @@ def test_version_selection_rejects_invalid_absent_and_unsupported_requests( def test_dataset_collection_excludes_outputs_and_storage_references( catalog_session: Session, ) -> None: - model_version = _us_model_version(catalog_session) + model_version = _model_version(catalog_session, "policyengine-us") output_id = uuid4() catalog_session.add( Dataset( @@ -560,6 +606,713 @@ def test_economy_options_require_a_national_region_and_dataset( _service(catalog_session).get_economy_options("us") +def _recorded_statements(session: Session, read): + statements: list[str] = [] + + def record_statement(_connection, _cursor, statement, *_args) -> None: + statements.append(statement.lower()) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record_statement) + try: + result = read() + finally: + event.remove(bind, "before_cursor_execute", record_statement) + return result, statements + + +def test_model_catalog_holds_only_the_small_parts_of_a_catalog_version( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, lambda: _service(catalog_session).get_model_catalog("us") + ) + us_catalog = normalized_catalog().country("us") + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert result.model.name == "policyengine-us" + assert ( + result.model_version.current_law_id == us_catalog.model_version.current_law_id + ) + assert result.entities == [ + MetadataEntity(key="person", plural="people", is_person=True), + MetadataEntity(key="household", plural="households", is_person=False), + ] + assert result.basic_inputs == ["employment_income"] + assert [region.code for region in result.regions] == sorted( + region.code for region in us_catalog.regions + ) + assert [dataset.name for dataset in result.datasets] == sorted( + dataset.name for dataset in us_catalog.datasets + ) + assert [option.name for option in result.economy_options.region] == [ + region.code for region in result.regions + ] + assert [option.name for option in result.economy_options.time_period] == list( + us_catalog.model_version.metadata_time_periods + ) + assert [option.label for option in result.economy_options.datasets] == ["Microcosm"] + assert len(statements) == 3 + for table in ("variables", "parameters", "parameter_values", "parameter_nodes"): + assert all(f"from {table}" not in statement for statement in statements) + + +def test_model_catalog_without_regions_offers_no_economy_calculation( + catalog_session: Session, +) -> None: + # A household-only country publishes no regions or datasets; its catalog + # still serves entities and basic inputs, with empty economy choices. + model_version = _model_version(catalog_session, "policyengine-us") + for region in catalog_session.exec( + select(Region).where(Region.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(region) + catalog_session.flush() + for dataset in catalog_session.exec( + select(Dataset).where(Dataset.tax_benefit_model_version_id == model_version.id) + ).all(): + catalog_session.delete(dataset) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.regions == [] + assert result.datasets == [] + assert result.economy_options.region == [] + assert result.economy_options.time_period == [] + assert result.economy_options.datasets == [] + assert result.basic_inputs == ["employment_income"] + + +def test_model_catalog_published_before_format_2_has_empty_new_fields( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + model_version.catalog_format = None + model_version.entities = None + model_version.basic_inputs = None + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.catalog_format is None + assert result.entities == [] + assert result.basic_inputs == [] + + +SPM_AVAILABLE = { + "available": True, + "settings_schema": {"title": "SPMSelection", "type": "object"}, + "defaults": {"scenario": "ce_trend", "geography_kind": "county"}, +} + + +def test_model_catalog_serves_the_versions_recorded_spm_availability( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + model_version.spm_availability = SPM_AVAILABLE + catalog_session.add(model_version) + catalog_session.commit() + + result = _service(catalog_session).get_model_catalog("us") + + assert result.spm.model_dump(mode="json") == SPM_AVAILABLE + + +def test_model_catalog_without_recorded_spm_availability_offers_none( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_model_catalog("uk") + + assert result.spm.model_dump(mode="json") == { + "available": False, + "settings_schema": None, + "defaults": None, + } + + +def test_model_catalog_with_regions_requires_the_national_region( + catalog_session: Session, +) -> None: + national_region = catalog_session.exec( + select(Region).where(Region.code == "us") + ).one() + catalog_session.delete(national_region) + catalog_session.commit() + + with pytest.raises(MetadataCatalogUnavailableError, match="national v2 region"): + _service(catalog_session).get_model_catalog("us") + + +def _add_variable_with_parameter_named_parts(session: Session) -> None: + model_version = _model_version(session, "policyengine-us") + parts = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.parts", + label="Income parts", + description=None, + data_type="list", + unit=None, + economy=False, + household=True, + ) + session.add(parts) + session.add_all( + [ + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income", "pension_income"], + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["employment_income"], + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + ), + ParameterValue( + id=uuid4(), + parameter_id=parts.id, + value_json=["reform_only_income"], + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ), + ] + ) + session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="total_income", + label="Total income", + entity="person", + description=None, + data_type="float", + possible_values=["LOW", "HIGH"], + default_value=0, + adds=["employment_income"], + subtracts=None, + is_input_variable=False, + hidden_input=True, + module_name="household.income.total_income", + adds_parameter_path="gov.example.parts", + subtracts_parameter_path="gov.example.unpublished_parts", + enum_values=[ + {"value": "LOW", "label": "Low"}, + {"value": "HIGH", "label": 2}, + ], + ) + ) + session.commit() + + +def test_catalog_variables_hold_every_variable_with_its_format_2_fields( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("us"), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [variable.name for variable in result.variables] == [ + "employment_income", + "total_income", + ] + employment_income, total_income = result.variables + assert employment_income.is_input_variable is True + assert employment_income.hidden_input is False + assert employment_income.module_name == "household.income.employment_income" + assert employment_income.adds_parameter_path is None + assert total_income.hidden_input is True + assert total_income.adds_parameter_path == "gov.example.parts" + assert total_income.subtracts_parameter_path == "gov.example.unpublished_parts" + assert total_income.enum_values == [ + MetadataEnumValue(value="LOW", label="Low"), + MetadataEnumValue(value="HIGH", label=2), + ] + assert len(statements) == 3 + + +def test_catalog_variables_carry_the_dated_values_of_parameter_named_parts( + catalog_session: Session, +) -> None: + _add_variable_with_parameter_named_parts(catalog_session) + + result = _service(catalog_session).get_catalog_variables("us") + + assert result.part_parameter_values == { + "gov.example.parts": [ + MetadataDatedValue( + start_date=datetime(2024, 1, 1, tzinfo=UTC), + end_date=datetime(2025, 12, 31, tzinfo=UTC), + value=["employment_income"], + ), + MetadataDatedValue( + start_date=datetime(2026, 1, 1, tzinfo=UTC), + end_date=None, + value=["employment_income", "pension_income"], + ), + ], + "gov.example.unpublished_parts": [], + } + + +def test_catalog_variables_without_parameter_named_parts_read_no_values( + catalog_session: Session, +) -> None: + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_variables("uk"), + ) + + assert [variable.name for variable in result.variables] == ["employment_income"] + assert result.part_parameter_values == {} + assert len(statements) == 2 + assert all("parameter_values" not in statement for statement in statements) + + +def test_variable_pages_carry_the_format_2_fields(catalog_session: Session) -> None: + variable = _service(catalog_session).list_variables("us").items[0] + + assert variable.is_input_variable is True + assert variable.module_name == "household.income.employment_income" + assert variable.enum_values is None + + +def _add_bracket_schedule(session: Session) -> None: + model_version = _model_version(session, "policyengine-us") + session.add_all( + [ + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates", + label="Example rates", + description=None, + ), + ParameterNode( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0]", + label=None, + description=None, + ), + ] + ) + bracket_rate = Parameter( + id=uuid4(), + tax_benefit_model_version_id=model_version.id, + name="gov.example.rates[0].rate", + label="Rate", + description=None, + data_type="float", + unit="/1", + economy=True, + household=True, + ) + session.add(bracket_rate) + session.add( + ParameterValue( + id=uuid4(), + parameter_id=bracket_rate.id, + value_json=0.2, + start_date=datetime(2025, 1, 1, tzinfo=UTC), + end_date=None, + ) + ) + session.commit() + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("gov", []), + ("gov.example.rate", ["gov", "gov.example"]), + ( + "gov.example.rates[0].rate", + ["gov", "gov.example", "gov.example.rates", "gov.example.rates[0]"], + ), + ("gov.example.rates[10]", ["gov", "gov.example", "gov.example.rates"]), + ], +) +def test_ancestor_paths_treat_a_bracket_index_as_a_child( + name: str, expected: list[str] +) -> None: + assert ancestor_paths(name) == expected + + +def test_named_parameters_load_with_values_and_ancestor_labels_in_one_read( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + + result, statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session).get_catalog_parameters( + "us", + names=["gov.example.rates[0].rate", "gov.example.rate", "gov.example.rate"], + ), + ) + + assert result.policyengine_version == POLICYENGINE_VERSION + assert result.catalog_format == 2 + assert [parameter.name for parameter in result.parameters] == [ + "gov.example.rate", + "gov.example.rates[0].rate", + ] + example_rate, bracket_rate = result.parameters + assert (example_rate.economy, example_rate.household) == (True, False) + assert [value.value for value in example_rate.values] == [0.1, 0.2] + assert example_rate.values[0].start_date == datetime(2025, 1, 1, tzinfo=UTC) + assert example_rate.values[0].end_date == datetime(2025, 12, 31, tzinfo=UTC) + assert [value.value for value in bracket_rate.values] == [0.2] + assert result.node_labels == { + "gov.example": "Example policy", + "gov.example.rates": "Example rates", + "gov.example.rates[0]": None, + } + assert result.missing_names == [] + assert len(statements) == 4 + + +def test_named_parameters_list_names_the_version_lacks( + catalog_session: Session, +) -> None: + result = _service(catalog_session).get_catalog_parameters( + "uk", names=["gov.example.rate", "gov.removed.amount"] + ) + + assert [parameter.name for parameter in result.parameters] == ["gov.example.rate"] + assert result.missing_names == ["gov.removed.amount"] + + +def test_named_parameters_exclude_policy_and_dynamic_values( + catalog_session: Session, +) -> None: + model_version = _model_version(catalog_session, "policyengine-us") + parameter = catalog_session.exec( + select(Parameter).where( + Parameter.tax_benefit_model_version_id == model_version.id, + Parameter.name == "gov.example.rate", + ) + ).one() + catalog_session.add( + ParameterValue( + id=uuid4(), + parameter_id=parameter.id, + value_json=0.9, + start_date=datetime(2027, 1, 1, tzinfo=UTC), + end_date=None, + policy_id=uuid4(), + ) + ) + catalog_session.commit() + + result = _service(catalog_session).get_catalog_parameters( + "us", names=["gov.example.rate"] + ) + + assert [value.value for value in result.parameters[0].values] == [0.1, 0.2] + + +def test_parameter_pages_and_values_can_be_narrowed_to_named_parameters( + catalog_session: Session, +) -> None: + _add_bracket_schedule(catalog_session) + service = _service(catalog_session) + + parameters = service.list_parameters( + "us", + parameter_filter=ParameterFilter( + names=("gov.example.rates[0].rate", "gov.removed.amount") + ), + ) + values = service.list_parameter_values( + "us", + value_filter=ParameterValueFilter( + parameter_names=("gov.example.rate", "gov.example.rates[0].rate") + ), + ) + + assert [parameter.name for parameter in parameters.items] == [ + "gov.example.rates[0].rate" + ] + assert parameters.items[0].economy is True + assert sorted(value.value for value in values.items) == [0.1, 0.2, 0.2] + + +def test_a_kept_catalog_selection_saves_its_query_on_later_reads( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + + first, first_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + second, second_statements = _recorded_statements( + catalog_session, + lambda: _service(catalog_session, selected_catalogs=store).list_parameters( + "us" + ), + ) + + assert second == first + assert len(first_statements) == 2 + assert len(second_statements) == 1 + assert "tax_benefit_model_versions" not in second_statements[0] + + +def test_failed_catalog_selections_are_not_kept(catalog_session: Session) -> None: + store = SelectedCatalogStore() + service = _service(catalog_session, selected_catalogs=store) + + for _attempt in range(2): + with pytest.raises(MetadataCatalogVersionNotFoundError): + service.list_parameters("us", "9.9.9") + with pytest.raises(UnsupportedPreviewCountryError): + service.list_parameters("ca") + with pytest.raises(InvalidPolicyEngineVersionError): + service.list_parameters("us", "not a version") + + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + assert service.list_parameters("us", "9.9.9").policyengine_version == "9.9.9" + + +def test_each_country_and_version_keeps_its_own_selection( + catalog_session: Session, +) -> None: + store = SelectedCatalogStore() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + service = _service(catalog_session, selected_catalogs=store) + + running_us = service.get_model_by_country("us") + named_us = service.get_model_by_country("us", "9.9.9") + running_uk = service.get_model_by_country("uk") + + assert (running_us.policyengine_version, named_us.policyengine_version) == ( + POLICYENGINE_VERSION, + "9.9.9", + ) + assert named_us.model_version.current_law_id == 7 + assert running_uk.model.name == "policyengine-uk" + assert service.get_model_by_country("us", "9.9.9") == named_us + + +def test_value_and_node_label_reads_return_typed_rows( + catalog_session: Session, +) -> None: + from policyengine_api.services.v2.metadata.database_connectors import ( + reads_parameter_nodes, + reads_parameters, + ) + + model_version = _model_version(catalog_session, "policyengine-us") + values = reads_parameters.read_current_law_values_by_parameter_name( + catalog_session, + model_version_id=model_version.id, + parameter_names={"gov.example.rate"}, + ) + labels = reads_parameter_nodes.read_parameter_node_labels( + catalog_session, + model_version_id=model_version.id, + node_names={"gov.example", "gov.absent"}, + ) + + assert [type(row) for row in values] == [CurrentLawValueRow, CurrentLawValueRow] + assert [(row.name, row.value_json) for row in values] == [ + ("gov.example.rate", 0.1), + ("gov.example.rate", 0.2), + ] + assert labels == [ParameterNodeLabelRow(name="gov.example", label="Example policy")] + + +def test_parameter_tree_leaves_carry_the_same_summary_as_parameter_pages( + catalog_session: Session, +) -> None: + service = _service(catalog_session) + + leaf = next( + child + for child in service.list_parameter_children( + "us", parent_path="gov.example" + ).items + if child.type == "parameter" + ) + page_item = service.list_parameters( + "us", parameter_filter=ParameterFilter(names=("gov.example.rate",)) + ).items[0] + + assert leaf.parameter == page_item + assert (leaf.parameter.economy, leaf.parameter.household) == (True, False) + + +def test_variables_carry_their_unit_and_definition_period( + catalog_session: Session, +) -> None: + uk_version = _model_version(catalog_session, "policyengine-uk") + catalog_session.add_all( + [ + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ), + # A variable published before catalog format 2 stored neither. + Variable( + id=uuid4(), + tax_benefit_model_version_id=uk_version.id, + name="age", + entity="person", + data_type="int", + default_value=0, + ), + ] + ) + catalog_session.commit() + service = _service(catalog_session) + + us_variables = service.get_catalog_variables("us").variables + uk_variables = { + variable.name: variable + for variable in service.get_catalog_variables("uk").variables + } + us_page = service.list_variables("us").items + + assert [(item.unit, item.definition_period) for item in us_variables] == [ + ("currency-USD", "year") + ] + assert (us_page[0].unit, us_page[0].definition_period) == ( + "currency-USD", + "year", + ) + assert ( + uk_variables["universal_credit"].unit, + uk_variables["universal_credit"].definition_period, + ) == ("currency-GBP", "month") + assert (uk_variables["age"].unit, uk_variables["age"].definition_period) == ( + None, + None, + ) + + +@pytest.mark.parametrize( + ("country_id", "version", "expected_catalog_format", "expected_fields"), + [ + ("us", None, 2, {"employment_income": ("currency-USD", "year")}), + ( + "uk", + None, + 2, + { + "employment_income": ("currency-USD", "year"), + "universal_credit": ("currency-GBP", "month"), + }, + ), + # A version published before catalog format 2 stored neither field. + ("us", "9.9.9", None, {"employment_income": (None, None)}), + ], +) +def test_both_variables_routes_serve_unit_and_definition_period( + catalog_session: Session, + country_id: str, + version: str | None, + expected_catalog_format: int | None, + expected_fields: dict[str, tuple[str | None, str | None]], +) -> None: + from policyengine_api.fastapi_routes.v2.metadata.response_cache import ( + MetadataResponseCache, + ) + + from .test_metadata_routes import _client + + catalog_session.add( + Variable( + id=uuid4(), + tax_benefit_model_version_id=_model_version( + catalog_session, "policyengine-uk" + ).id, + name="universal_credit", + entity="benunit", + data_type="float", + default_value=0, + unit="currency-GBP", + definition_period="month", + ) + ) + catalog_session.commit() + _add_country_version( + catalog_session, + policyengine_version="9.9.9", + current_law_id=7, + time_periods=[2027], + ) + earlier_version = _model_version(catalog_session, "policyengine-us", "9.9.9") + earlier_version.catalog_format = None + for variable in catalog_session.exec( + select(Variable).where( + Variable.tax_benefit_model_version_id == earlier_version.id + ) + ).all(): + variable.unit = None + variable.definition_period = None + catalog_session.commit() + client = _client( + lambda: _service(catalog_session), response_cache=MetadataResponseCache() + ) + query = f"country_id={country_id}" + ( + "" if version is None else f"&policyengine_version={version}" + ) + + whole_version_bodies = [ + client.get(f"/v2/model-catalog/variables?{query}").json()["result"] + for _attempt in ("read", "stored") + ] + page = client.get(f"/v2/variables?{query}").json()["result"] + + for result in whole_version_bodies: + assert result["catalog_format"] == expected_catalog_format + served = { + variable["name"]: (variable["unit"], variable["definition_period"]) + for variable in result["variables"] + } + assert served == expected_fields + paged = { + item["name"]: (item["unit"], item["definition_period"]) + for item in page["items"] + } + assert paged == expected_fields + + def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package = Path(__file__).parents[3] / "policyengine_api" connector_directory = ( @@ -569,6 +1322,7 @@ def test_read_modules_import_no_policyengine_or_v1_metadata_source() -> None: project_package / "data" / "v2" / "catalog" / "catalog_selection.py", connector_directory / "reads.py", connector_directory / "reads_datasets.py", + connector_directory / "reads_parameter_nodes.py", connector_directory / "reads_parameter_tree.py", connector_directory / "reads_parameters.py", connector_directory / "reads_regions.py", @@ -624,6 +1378,9 @@ def test_resource_entrypoints_are_defined_in_the_service_module() -> None: "get_region", "get_region_by_code", "get_economy_options", + "get_model_catalog", + "get_catalog_variables", + "get_catalog_parameters", } for method_name in method_names: diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index be7879424..732fa7d98 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -10,7 +10,9 @@ from policyengine_api.data.v2.models import ( V2_METADATA, + Country, Dynamic, + Household, LegacyPolicyMapping, LegacyUserMapping, LegacyUserPolicyMapping, @@ -20,9 +22,11 @@ TaxBenefitModel, TaxBenefitModelVersion, User, + UserHouseholdAssociation, UserPolicy, ) from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS +from tests.fixtures.v2_countries import add_listed_countries def _relational_sqlite_engine(): @@ -35,6 +39,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine @@ -99,9 +105,8 @@ def test_direct_sqlalchemy_categories_are_complete_and_documented() -> None: assert all(DIRECT_SQLALCHEMY_EXCEPTIONS.values()) -def test_user_primary_country_can_change_between_us_and_uk_only() -> None: - engine = create_engine("sqlite://") - V2_METADATA.create_all(engine) +def test_user_primary_country_must_be_a_listed_country() -> None: + engine = _relational_sqlite_engine() with Session(engine) as session: user = User( @@ -126,6 +131,73 @@ def test_user_primary_country_can_change_between_us_and_uk_only() -> None: engine.dispose() +def test_a_country_row_lets_records_use_a_household_only_country() -> None: + engine = _relational_sqlite_engine() + household_only_country = Country( + id="be", + household_enabled=True, + society_wide_enabled=False, + ) + + with Session(engine) as session: + session.add(User(primary_country="be")) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.add(household_only_country) + session.commit() + user = User(primary_country="be") + household = Household( + country_id="be", + household_data={"people": {}}, + canonicalization_version=1, + content_hash="b" * 64, + ) + session.add_all([user, household]) + session.flush() + session.add( + UserHouseholdAssociation( + user_id=user.id, + household_id=household.id, + country_id="be", + ) + ) + session.commit() + + stored_country = session.exec(select(UserHouseholdAssociation.country_id)).one() + assert stored_country == "be" + + engine.dispose() + + +def test_a_listed_country_cannot_be_deleted_while_records_use_it() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add(User(primary_country="uk")) + session.commit() + + session.delete(session.get(Country, "uk")) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_a_society_wide_country_must_also_be_household_capable() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add( + Country(id="be", household_enabled=False, society_wide_enabled=True) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + def test_only_unauthenticated_users_may_omit_a_primary_country() -> None: engine = create_engine("sqlite://") V2_METADATA.create_all(engine) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 20947ffcb..88ed2db5e 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -4,10 +4,12 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.orm import configure_mappers from sqlalchemy.schema import CreateTable +from sqlmodel.sql.sqltypes import AutoString from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.models import ( V2_METADATA, + Country, DatasetVersion, Dynamic, Household, @@ -29,6 +31,18 @@ UserType, ) +# Every column that names the country a record belongs to. Each references the +# countries table instead of checking a fixed list of country IDs. +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policy_associations", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) + RUN_OUTPUT_TABLES = frozenset( { "aggregates", @@ -48,6 +62,7 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: expected_modules = { + Country: "countries", User: "users", UserIdentity: "users", Policy: "policies", @@ -130,7 +145,7 @@ def test_user_owned_associations_have_relational_integrity() -> None: "user_simulation_associations", "simulation_associations", ), - (UserPolicy, "user_policies", "policy_associations"), + (UserPolicy, "user_policy_associations", "policy_associations"), (UserReportAssociation, "user_report_associations", "report_associations"), ) @@ -195,7 +210,7 @@ def test_policy_parameter_values_use_jsonb_and_enforce_period_identity() -> None def test_user_policy_is_an_independent_country_scoped_association() -> None: - associations = V2_METADATA.tables["user_policies"] + associations = V2_METADATA.tables["user_policy_associations"] assert isinstance(associations.c.user_id.type, sa.Uuid) user_foreign_key = next(iter(associations.c.user_id.foreign_keys)) @@ -215,7 +230,7 @@ def test_user_policy_is_an_independent_country_scoped_association() -> None: policy_country = next( constraint for constraint in associations.foreign_key_constraints - if constraint.name == "fk_user_policies_policy_country" + if constraint.name == "fk_user_policy_associations_policy_country" ) assert [column.name for column in policy_country.columns] == [ "policy_id", @@ -354,8 +369,8 @@ def test_standard_users_require_a_supported_primary_country() -> None: for constraint in users.constraints if isinstance(constraint, sa.CheckConstraint) } + # The countries table, not a check, lists the countries a user may choose. assert checks == { - "ck_users_primary_country": "primary_country IN ('us', 'uk')", "ck_users_user_type": "user_type IN ('unauthenticated', 'standard')", "ck_users_standard_primary_country": ( "user_type = 'unauthenticated' OR primary_country IS NOT NULL" @@ -474,7 +489,8 @@ def test_named_checks_and_required_indexes_cover_core_invariants() -> None: assert { "ck_regions_required_filter_values", "ck_simulations_type_input", - "ck_users_primary_country", + "ck_users_standard_primary_country", + "ck_countries_society_wide_requires_household", "ck_report_runs_terminal_completion", "ck_parameter_values_single_owner", }.issubset(constraint_names) @@ -571,7 +587,7 @@ def test_user_identity_maps_one_issuer_subject_to_one_user() -> None: def test_associations_are_v1_anchored_without_client_refs() -> None: for table_name, reference in ( - ("user_policies", "policy"), + ("user_policy_associations", "policy"), ("user_household_associations", "household"), ("user_simulation_associations", "simulation"), ("user_report_associations", "report"), @@ -607,10 +623,11 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None assert table.c.label.nullable assert { foreign_key.target_fullname for foreign_key in simulations.foreign_keys - } == {"users.id"} + } == {"users.id", "countries.id"} assert {foreign_key.target_fullname for foreign_key in reports.foreign_keys} == { "users.id", - "user_policies.id", + "user_policy_associations.id", + "countries.id", } user_policy_key = next(iter(reports.c.user_policy_id.foreign_keys)) assert user_policy_key.ondelete == "SET NULL" @@ -623,15 +640,208 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None def test_user_policy_records_v3_reform_fields() -> None: - policies = V2_METADATA.tables["user_policies"] + policies = V2_METADATA.tables["user_policy_associations"] assert policies.c.baseline.type.length == 16 baseline_check = next( constraint for constraint in policies.constraints - if constraint.name == "ck_user_policies_baseline" + if constraint.name == "ck_user_policy_associations_baseline" ) assert "current-law" in str(baseline_check.sqltext) assert "current-policy" in str(baseline_check.sqltext) postgres_type = policies.c.provenance.type.dialect_impl(postgresql.dialect()) assert isinstance(postgres_type, postgresql.JSONB) + + +def test_saved_policy_table_is_named_like_the_other_association_tables() -> None: + """The saved-reform table and its constraints follow the association pattern. + + Composite indexes keep their short ``ix_user_policies_*`` names, as the + household, simulation and report associations keep ``ix_user_households_*`` + and the like; constraints and convention-named indexes take the table name. + """ + assert UserPolicy.__tablename__ == "user_policy_associations" + assert "user_policies" not in V2_METADATA.tables + table = V2_METADATA.tables["user_policy_associations"] + + assert {constraint.name for constraint in table.constraints} == { + "pk_user_policy_associations", + "uq_user_policy_associations_id_country", + "fk_user_policy_associations_country_id_countries", + "ck_user_policy_associations_policy_ref", + "ck_user_policy_associations_baseline", + "fk_user_policy_associations_policy_country", + "fk_user_policy_associations_user_id_users", + } + assert {index.name for index in table.indexes} == { + "ix_user_policy_associations_user_id", + "ix_user_policy_associations_policy_id", + "ix_user_policies_country_user_created_id", + "ix_user_policies_country_policy", + "ix_user_policies_country_legacy_policy", + } + referring_keys = { + (foreign_key.parent.table.name, foreign_key.parent.name) + for referring_table in V2_METADATA.tables.values() + for foreign_key in referring_table.foreign_keys + if foreign_key.column.table is table + } + assert referring_keys == { + ("legacy_user_policy_mappings", "user_policy_id"), + ("legacy_user_policy_mappings", "country_id"), + ("user_report_associations", "user_policy_id"), + ("user_report_associations", "baseline_user_policy_id"), + } + + +def test_spm_availability_is_nullable_for_versions_not_yet_recorded() -> None: + """Publication records it after the catalog, so a version may lack it briefly.""" + column = V2_METADATA.tables["tax_benefit_model_versions"].c["spm_availability"] + + assert column.nullable + assert column.server_default is None + assert isinstance(column.type, sa.JSON) + + +def test_catalog_format_2_columns_are_nullable_for_earlier_versions() -> None: + """Catalog format 2 stores v3's metadata; versions published before it keep null. + + The model version records its catalog format, entities and basic inputs; each + variable its input flags, module, parameter-named parts and enum labels; each + parameter its economy and household flags. + """ + expected_columns = { + "tax_benefit_model_versions": { + "catalog_format": sa.Integer, + "entities": sa.JSON, + "basic_inputs": sa.JSON, + }, + "variables": { + "is_input_variable": sa.Boolean, + "hidden_input": sa.Boolean, + "module_name": AutoString, + "adds_parameter_path": AutoString, + "subtracts_parameter_path": AutoString, + "enum_values": sa.JSON, + "unit": AutoString, + "definition_period": AutoString, + }, + "parameters": { + "economy": sa.Boolean, + "household": sa.Boolean, + }, + } + + for table_name, columns in expected_columns.items(): + table = V2_METADATA.tables[table_name] + for column_name, column_type in columns.items(): + column = table.c[column_name] + assert column.nullable, f"{table_name}.{column_name}" + assert column.server_default is None, f"{table_name}.{column_name}" + assert isinstance(column.type, column_type), f"{table_name}.{column_name}" + + variables = V2_METADATA.tables["variables"] + for column_name in ( + "module_name", + "adds_parameter_path", + "subtracts_parameter_path", + ): + assert variables.c[column_name].type.length == 512 + assert variables.c["unit"].type.length == 128 + assert variables.c["definition_period"].type.length == 32 + + +def test_catalog_format_2_json_columns_have_typed_shapes() -> None: + """The entity and enum-value columns name typed shapes the catalog records match. + + Readers of these columns get the keys and value types without re-checking. + """ + from typing import get_type_hints + + from policyengine_api.data.v2.catalog.records import EntityRecord, EnumValueRecord + from policyengine_api.data.v2.models import ( + CatalogEntity, + CatalogEnumValue, + Variable, + ) + + assert TaxBenefitModelVersion.model_fields["entities"].annotation == ( + list[CatalogEntity] | None + ) + assert Variable.model_fields["enum_values"].annotation == ( + list[CatalogEnumValue] | None + ) + for typed_shape, record in ( + (CatalogEntity, EntityRecord), + (CatalogEnumValue, EnumValueRecord), + ): + assert get_type_hints(typed_shape) == get_type_hints(record) + + +def test_countries_record_household_and_society_wide_capability() -> None: + countries = V2_METADATA.tables["countries"] + + assert countries.c.keys() == ["id", "household_enabled", "society_wide_enabled"] + assert countries.primary_key.columns.keys() == ["id"] + assert countries.primary_key.name == "pk_countries" + assert countries.c.id.type.length == 2 + for flag_name in ("household_enabled", "society_wide_enabled"): + assert isinstance(countries.c[flag_name].type, sa.Boolean) + assert not countries.c[flag_name].nullable + assert countries.c[flag_name].server_default is None + checks = { + constraint.name: str(constraint.sqltext) + for constraint in countries.constraints + if isinstance(constraint, sa.CheckConstraint) + } + # Society-wide reports need household calculations, so a society-wide + # country is always household capable too. + assert checks == { + "ck_countries_society_wide_requires_household": ( + "household_enabled OR NOT society_wide_enabled" + ), + } + + +def test_country_columns_reference_the_countries_table_instead_of_a_fixed_list() -> ( + None +): + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + table = V2_METADATA.tables[table_name] + country_keys = [ + foreign_key + for foreign_key in table.c[column_name].foreign_keys + if foreign_key.column.table.name == "countries" + ] + assert len(country_keys) == 1, table_name + assert country_keys[0].target_fullname == "countries.id" + assert country_keys[0].ondelete == "RESTRICT" + assert country_keys[0].constraint.name == ( + f"fk_{table_name}_{column_name}_countries" + ) + fixed_country_checks = [ + constraint.name + for constraint in table.constraints + if isinstance(constraint, sa.CheckConstraint) + and "'us', 'uk'" in str(constraint.sqltext) + ] + assert fixed_country_checks == [], table_name + + +def test_v1_copy_mappings_keep_their_us_and_uk_checks() -> None: + # These tables pair v1 records with their v2 copies. Each already + # references a v2 record of the same country, and they are removed with + # dual-write, so they keep their fixed checks. + for table_name in ( + "legacy_household_mappings", + "legacy_policy_mappings", + "legacy_user_policy_mappings", + ): + table = V2_METADATA.tables[table_name] + assert f"ck_{table_name}_country" in { + constraint.name for constraint in table.constraints + } + assert "countries.id" not in { + foreign_key.target_fullname for foreign_key in table.foreign_keys + } diff --git a/tests/unit/v2/test_parameter_value_listing_route.py b/tests/unit/v2/test_parameter_value_listing_route.py index 39ac27b72..a07aa4ca5 100644 --- a/tests/unit/v2/test_parameter_value_listing_route.py +++ b/tests/unit/v2/test_parameter_value_listing_route.py @@ -2,6 +2,8 @@ from __future__ import annotations +from datetime import UTC, datetime + import pytest from policyengine_api.services.v2.metadata.types import ParameterValueFilter @@ -9,7 +11,13 @@ from .test_metadata_routes import ResourceReader, _client, _resource_results -def test_listing_forwards_every_query_parameter() -> None: +def test_listing_forwards_every_query_parameter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_api.fastapi_routes.v2.metadata import common + + request_instant = datetime(2026, 6, 1, 12, 0, tzinfo=UTC) + monkeypatch.setattr(common, "utc_now", lambda: request_instant) results, ids = _resource_results() reader = ResourceReader(results) @@ -32,7 +40,7 @@ def test_listing_forwards_every_query_parameter() -> None: ("uk", "4.19.0"), { "value_filter": ParameterValueFilter( - parameter_id=ids["parameter_id"], current=True + parameter_id=ids["parameter_id"], current=True, now=request_instant ), "offset": 5, "limit": 7, diff --git a/tests/unit/v2/test_policy_migration_qualification.py b/tests/unit/v2/test_policy_migration_qualification.py index 54a711ef4..dc28306a5 100644 --- a/tests/unit/v2/test_policy_migration_qualification.py +++ b/tests/unit/v2/test_policy_migration_qualification.py @@ -175,3 +175,29 @@ def fail() -> None: assert "qualification failed unexpectedly" in captured.err assert "secret" not in captured.err assert "private-host" not in captured.err + + +class RecordingExecutor: + def __init__(self) -> None: + self.statements: list[str] = [] + + def scalar(self, statement: object) -> int: + self.statements.append(str(statement)) + return 0 + + +def test_saved_policies_are_counted_under_the_name_of_the_pending_revision() -> None: + """The policy revision is pending only before the rename of ``user_policies``. + + Counting through the current model would query ``user_policy_associations``, + which a database still waiting for the policy revision does not have yet. + """ + executor = RecordingExecutor() + + qualification.read_policy_data_counts( + executor, existing_tables={"policies", "parameter_values", "user_policies"} + ) + + saved_policy_count = executor.statements[2] + assert "FROM user_policies" in saved_policy_count + assert "user_policy_associations" not in saved_policy_count diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 7a3a76b44..380d9a81e 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -22,6 +22,7 @@ READ_WRITE, RUNTIME_ROLE, TABLE_ACCESS, + TEMPORARY_VIEW_TABLES, InvalidDeploymentEnvironmentError, TablePrivilege, declared_privileges, @@ -72,7 +73,28 @@ def test_every_v2_table_has_an_entry_for_every_managed_role() -> None: assert set(TABLE_ACCESS) == metadata_tables | {"alembic_version"} for role, privileges_by_table in declared_privileges(TEST_ENVIRONMENT).items(): - assert set(privileges_by_table) == set(TABLE_ACCESS), role + assert set(privileges_by_table) == set(TABLE_ACCESS) | set( + TEMPORARY_VIEW_TABLES + ), role + + +def test_each_temporary_view_grants_every_role_what_its_table_grants() -> None: + """A view under a renamed table's earlier name serves the earlier release. + + That release reads and writes the table through the view, so each role + needs on the view exactly what it holds on the table. + """ + metadata_tables = {table.name for table in V2_METADATA.tables.values()} + + assert TEMPORARY_VIEW_TABLES == {"user_policies": "user_policy_associations"} + for view, table in TEMPORARY_VIEW_TABLES.items(): + assert view not in metadata_tables + assert table in metadata_tables + for role, privileges_by_table in declared_privileges(TEST_ENVIRONMENT).items(): + assert privileges_by_table[view] == privileges_by_table[table], role + assert declared_privileges(TEST_ENVIRONMENT)[RUNTIME_ROLE]["user_policies"] == ( + READ_WRITE + ) def test_v2_schema_has_no_sequences_the_manifest_would_need_to_cover() -> None: @@ -133,7 +155,7 @@ def test_rule_proof_runtime_privileges_declared() -> None: reads ``legacy_policy_mappings``, which the runtime role had no grant on. """ runtime_before_3977 = dict.fromkeys(TABLE_ACCESS, NO_ACCESS) | { - "user_policies": READ_WRITE + "user_policy_associations": READ_WRITE } table_by_model = _table_by_model() diff --git a/tests/unit/v2/test_privilege_reconciliation.py b/tests/unit/v2/test_privilege_reconciliation.py index 73ca30e19..a6d9590dc 100644 --- a/tests/unit/v2/test_privilege_reconciliation.py +++ b/tests/unit/v2/test_privilege_reconciliation.py @@ -11,6 +11,7 @@ from policyengine_api.data.v2.privilege_reconciliation import ( DefaultPrivilege, PrivilegeEntry, + PublicRelations, ReconciliationPlan, grant_statements, plan_reconciliation, @@ -48,12 +49,16 @@ def _declared() -> dict[str, dict[str, frozenset[TablePrivilege]]]: } +def _tables(names: set[str]) -> PublicRelations: + return PublicRelations(tables=frozenset(names), views=frozenset()) + + def test_missing_declared_privileges_become_grants() -> None: plan = plan_reconciliation( _declared(), granted={RUNTIME: {"users": {"SELECT", "INSERT"}}}, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports"}, + existing_relations=_tables({"users", "legacy_policy_mappings", "reports"}), ) assert plan.grants == ( @@ -78,7 +83,7 @@ def test_privileges_beyond_the_manifest_are_reported_not_granted_or_revoked() -> STAGE12: {"users": {"SELECT"}}, }, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports"}, + existing_relations=_tables({"users", "legacy_policy_mappings", "reports"}), ) assert plan.grants == () @@ -96,7 +101,9 @@ def test_privileges_on_tables_outside_the_manifest_are_reported() -> None: _declared(), granted={RUNTIME: {"scratch_table": {"SELECT"}}}, existing_roles={RUNTIME, STAGE12}, - existing_tables={"users", "legacy_policy_mappings", "reports", "scratch_table"}, + existing_relations=_tables( + {"users", "legacy_policy_mappings", "reports", "scratch_table"} + ), ) assert PrivilegeEntry(RUNTIME, "scratch_table", "SELECT") in plan.undeclared @@ -107,7 +114,7 @@ def test_missing_roles_and_tables_are_recorded_and_block_the_plan() -> None: _declared(), granted={}, existing_roles={RUNTIME}, - existing_tables={"users", "reports"}, + existing_relations=_tables({"users", "reports"}), ) assert plan.missing_roles == (STAGE12,) @@ -120,6 +127,71 @@ def test_missing_roles_and_tables_are_recorded_and_block_the_plan() -> None: ) +def test_a_relation_of_the_other_kind_does_not_satisfy_a_declaration() -> None: + """A declared table must be a table, and a declared view must be a view. + + Before the rename, ``user_policies`` is a table, which must not satisfy the + temporary view of that name; a view named like a declared table must not + satisfy the table either. + """ + declared = { + RUNTIME: {"users": READ_WRITE, "user_policies": READ_WRITE, "reports": READ} + } + + plan = plan_reconciliation( + declared, + granted={}, + existing_roles={RUNTIME}, + existing_relations=PublicRelations( + tables=frozenset({"user_policies", "reports"}), views=frozenset({"users"}) + ), + declared_views={"user_policies"}, + ) + + assert plan.missing_tables == ("users",) + assert plan.missing_views == ("user_policies",) + assert not plan.can_apply + + +def test_declared_views_that_exist_as_views_are_granted() -> None: + declared = {RUNTIME: {"users": READ, "user_policies": READ_WRITE}} + + plan = plan_reconciliation( + declared, + granted={}, + existing_roles={RUNTIME}, + existing_relations=PublicRelations( + tables=frozenset({"users"}), views=frozenset({"user_policies"}) + ), + declared_views={"user_policies"}, + ) + + assert plan.can_apply + assert plan.missing_tables == plan.missing_views == () + assert {entry.table for entry in plan.grants} == {"users", "user_policies"} + + +def test_summary_names_missing_views_separately_from_missing_tables() -> None: + plan = ReconciliationPlan( + grants=(), + undeclared=(), + missing_roles=(), + missing_tables=("users",), + missing_views=("user_policies",), + ) + + summary = render_summary(plan, (), deployment_environment="staging", applied=False) + + assert ( + "Declared tables missing from the database (nothing was granted): `users`" + in summary + ) + assert ( + "Declared views missing from the database (nothing was granted): " + "`user_policies`" in summary + ) + + def test_grant_statements_group_privileges_per_role_and_table() -> None: plan = ReconciliationPlan( grants=( diff --git a/tests/unit/v2/test_privilege_requirements.py b/tests/unit/v2/test_privilege_requirements.py index 76ba0b5eb..314a85aff 100644 --- a/tests/unit/v2/test_privilege_requirements.py +++ b/tests/unit/v2/test_privilege_requirements.py @@ -20,7 +20,7 @@ TABLE_BY_MODEL = { "User": "users", - "UserPolicy": "user_policies", + "UserPolicy": "user_policy_associations", "LegacyPolicyMapping": "legacy_policy_mappings", } @@ -72,8 +72,8 @@ def create(session, values): ) assert _needed(path) == { - ("user_policies", TablePrivilege.INSERT), - ("user_policies", TablePrivilege.SELECT), + ("user_policy_associations", TablePrivilege.INSERT), + ("user_policy_associations", TablePrivilege.SELECT), } @@ -169,5 +169,5 @@ def read_with_owner(session): """, ) - assert ("user_policies", TablePrivilege.UPDATE) in _needed(path) + assert ("user_policy_associations", TablePrivilege.UPDATE) in _needed(path) assert ("users", TablePrivilege.UPDATE) not in _needed(path) diff --git a/tests/unit/v2/test_region_data_verification.py b/tests/unit/v2/test_region_data_verification.py new file mode 100644 index 000000000..11c719a41 --- /dev/null +++ b/tests/unit/v2/test_region_data_verification.py @@ -0,0 +1,186 @@ +"""Publication refuses regions the certified data cannot select.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from policyengine_api.data.v2.catalog.errors import CertifiedDataReadError +from policyengine_api.data.v2.catalog.region_data_verification import ( + RegionDataVerificationError, + verify_region_data, +) +from tests.fixtures.v2_catalog import normalized_catalog + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.records import CertifiedArtifact + +ALL_COLUMNS = { + "populace_us_2024_acs_local": frozenset( + {"state_fips", "congressional_district_geoid", "household_weight"} + ), + "microcosm_uk_2024_25": frozenset({"region", "household_weight"}), + "microcosm_uk_2024_25_dense": frozenset( + {"constituency_code_oa", "la_code_oa", "household_weight"} + ), +} + + +class FakeInspector: + """Answers from fixed columns, and records each artifact read.""" + + def __init__( + self, + *, + columns: dict[str, frozenset[str]] | None = None, + failure: Exception | None = None, + ) -> None: + self.columns = ALL_COLUMNS if columns is None else columns + self.failure = failure + self.reads: list[str] = [] + + def household_columns(self, artifact: CertifiedArtifact) -> frozenset[str]: + self.reads.append(artifact.name) + if self.failure is not None: + raise self.failure + return self.columns[artifact.name] + + +def _verify( + inspector: FakeInspector, + *, + countries: tuple[str, ...] = ("us", "uk"), +) -> None: + verify_region_data(normalized_catalog(), inspector, countries=countries) + + +def test_catalog_records_only_the_artifacts_its_filters_read() -> None: + catalog = normalized_catalog() + + assert [ + (artifact.name, artifact.repo_id, artifact.repo_type, artifact.revision) + for artifact in catalog.country("us").certified_artifacts + ] == [ + ( + "populace_us_2024_acs_local", + "policyengine/populace-us", + "dataset", + "fixture", + ) + ] + assert [ + (artifact.name, artifact.repo_id, artifact.path, artifact.repo_type) + for artifact in catalog.country("uk").certified_artifacts + ] == [ + ( + "microcosm_uk_2024_25", + "example-org/example-uk-data", + "microcosm_uk_2024_25.h5", + "model", + ), + ( + "microcosm_uk_2024_25_dense", + "example-org/example-uk-data", + "microcosm_uk_2024_25_dense.h5", + "model", + ), + ] + + +def test_accepts_regions_whose_filter_columns_exist() -> None: + inspector = FakeInspector() + + _verify(inspector) + + assert sorted(inspector.reads) == [ + "microcosm_uk_2024_25", + "microcosm_uk_2024_25_dense", + "populace_us_2024_acs_local", + ] + + +def test_reads_only_the_artifacts_of_the_countries_it_verifies() -> None: + inspector = FakeInspector() + + _verify(inspector, countries=("us",)) + + assert inspector.reads == ["populace_us_2024_acs_local"] + + +def test_verifies_nothing_when_every_version_is_already_published() -> None: + inspector = FakeInspector(failure=AssertionError("must not read")) + + _verify(inspector, countries=()) + + assert inspector.reads == [] + + +def test_rejects_a_filter_column_missing_from_the_dataset() -> None: + inspector = FakeInspector( + columns={ + **ALL_COLUMNS, + "populace_us_2024_acs_local": frozenset({"state_fips"}), + } + ) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the certified dataset populace_us_2024_acs_local has no household " + r"column congressional_district_geoid, which 1 us region filters on " + r"\(congressional_district/CA-01\)$", + ): + _verify(inspector) + + +def test_rejects_a_local_area_dataset_without_the_area_code_column() -> None: + inspector = FakeInspector( + columns={ + **ALL_COLUMNS, + "microcosm_uk_2024_25_dense": frozenset({"la_code_oa"}), + } + ) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the certified dataset microcosm_uk_2024_25_dense has no household " + r"column constituency_code_oa, which 1 uk region filters on " + r"\(constituency/E14001063\)$", + ): + _verify(inspector, countries=("uk",)) + + +def test_reports_an_unreadable_artifact_without_its_error_text() -> None: + inspector = FakeInspector( + failure=PermissionError("403 for https://example.test/?token=do-not-print") + ) + + with pytest.raises(RegionDataVerificationError) as raised: + _verify(inspector, countries=("us",)) + + assert str(raised.value) == ( + "could not read the certified artifact populace_us_2024_acs_local: " + "PermissionError" + ) + assert "do-not-print" not in str(raised.value) + assert "fixture" not in str(raised.value) + + +def test_reports_an_artifact_structure_error_with_its_own_message() -> None: + inspector = FakeInspector( + failure=CertifiedDataReadError("household group holds no data frame") + ) + + with pytest.raises( + RegionDataVerificationError, + match=r"^the certified artifact populace_us_2024_acs_local cannot be " + r"checked: household group holds no data frame$", + ): + _verify(inspector, countries=("us",)) + + +def test_passes_on_a_verification_error_the_inspector_raises() -> None: + inspector = FakeInspector(failure=RegionDataVerificationError("already explained")) + + with pytest.raises(RegionDataVerificationError, match=r"^already explained$"): + _verify(inspector, countries=("us",)) diff --git a/tests/unit/v2/test_region_references.py b/tests/unit/v2/test_region_references.py new file mode 100644 index 000000000..e2493a2d5 --- /dev/null +++ b/tests/unit/v2/test_region_references.py @@ -0,0 +1,353 @@ +"""The checked-in region reference files and the rules their loader enforces.""" + +from __future__ import annotations + +import csv +import json +from collections import Counter +from pathlib import Path + +import pytest + +from policyengine_api.data.congressional_districts import ( + AT_LARGE_STATES, + CONGRESSIONAL_DISTRICTS, + STATE_CODE_TO_NAME, + build_congressional_district_metadata, +) +from policyengine_api.data.v2.catalog.errors import CatalogExtractionError +from policyengine_api.data.v2.catalog.region_references import ( + REGION_REFERENCE_FIELDS, + RegionReference, + load_region_references, + parse_region_references, +) + +V1_DATA_DIRECTORY = Path(__file__).parents[3] / "policyengine_api" / "data" +NATION_BY_ONS_PREFIX = { + "E": "country/england", + "S": "country/scotland", + "W": "country/wales", + "N": "country/northern_ireland", +} +ENGLISH_ITL1_REGIONS = [ + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", +] + + +def _by_code(references: tuple[RegionReference, ...]) -> dict[str, RegionReference]: + return {reference.code: reference for reference in references} + + +def _v1_lookup_rows(filename: str) -> list[dict[str, str]]: + with (V1_DATA_DIRECTORY / filename).open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +def _row(code: str, region_type: str, **fields: str) -> dict[str, str]: + """Return one reference-file row; unspecified columns are empty.""" + row = dict.fromkeys(REGION_REFERENCE_FIELDS, "") + row.update(code=code, label=code.title(), region_type=region_type, v1_code=code) + row.update(fields) + return row + + +def _small_us_rows() -> list[dict[str, str]]: + return [ + _row("us", "national"), + _row("state/ca", "state", parent_code="us", filter_value="6"), + _row( + "congressional_district/CA-01", + "congressional_district", + parent_code="state/ca", + ), + ] + + +def test_each_country_lists_every_region_once_with_the_expected_counts() -> None: + us = load_region_references("us") + uk = load_region_references("uk") + + assert Counter(reference.region_type for reference in us) == { + "national": 1, + "state": 51, + "congressional_district": 436, + "place": 333, + } + assert Counter(reference.region_type for reference in uk) == { + "national": 1, + "country": 4, + "constituency": 650, + "local_authority": 361, + } + assert len(uk) == 1_016 + for references in (us, uk): + codes = [reference.code for reference in references] + v1_codes = [reference.v1_code for reference in references] + assert len(set(codes)) == len(codes) + assert len(set(v1_codes)) == len(v1_codes) + + +def test_national_regions_and_the_v1_identifiers_they_replace() -> None: + us = _by_code(load_region_references("us")) + uk = _by_code(load_region_references("uk")) + + assert us["us"].label == "United States" + assert uk["uk"].label == "United Kingdom" + assert us["us"].parent_code is None + assert uk["uk"].parent_code is None + assert uk["country/northern_ireland"].v1_code == "country/ni" + assert uk["country/northern_ireland"].label == "Northern Ireland" + assert "country/ni" not in uk + assert uk["constituency/E14001063"].label == "Aldershot" + assert uk["constituency/E14001063"].v1_code == "constituency/Aldershot" + assert uk["local_authority/E07000110"].label == "Maidstone" + assert uk["local_authority/E07000110"].v1_code == "local_authority/Maidstone" + assert uk["local_authority/N09000011"].label == "Ards and North Down" + assert uk["local_authority/N09000011"].parent_code == "country/northern_ireland" + assert uk["local_authority/N09000011"].v1_code == ( + "local_authority/Ards and North Down" + ) + assert us["state/ca"].v1_code == "state/ca" + assert us["congressional_district/CA-01"].v1_code == "congressional_district/CA-01" + + +def test_uk_constituencies_and_local_authorities_use_ons_codes_from_v1_lookups() -> ( + None +): + uk = load_region_references("uk") + for region_type, filename in ( + ("constituency", "constituencies_2024.csv"), + ("local_authority", "local_authorities_2021.csv"), + ): + expected = { + f"{region_type}/{row['code']}": ( + row["name"], + f"{region_type}/{row['name']}", + row["code"], + NATION_BY_ONS_PREFIX[row["code"][0]], + ) + for row in _v1_lookup_rows(filename) + } + actual = { + reference.code: ( + reference.label, + reference.v1_code, + reference.filter_value, + reference.parent_code, + ) + for reference in uk + if reference.region_type == region_type + } + assert actual == expected + + +def test_uk_nations_select_their_stored_regions() -> None: + uk = _by_code(load_region_references("uk")) + + assert json.loads(uk["country/england"].filter_value or "") == ( + ENGLISH_ITL1_REGIONS + ) + assert uk["country/scotland"].filter_value == "SCOTLAND" + assert uk["country/wales"].filter_value == "WALES" + assert uk["country/northern_ireland"].filter_value == "NORTHERN_IRELAND" + assert all(uk[code].parent_code == "uk" for code in NATION_BY_ONS_PREFIX.values()) + + +def test_us_states_and_districts_match_v1_and_select_by_fips_and_geoid() -> None: + us = load_region_references("us") + states = { + reference.state_code: reference + for reference in us + if reference.region_type == "state" + } + districts = { + reference.code: reference + for reference in us + if reference.region_type == "congressional_district" + } + + assert {code: state.label for code, state in states.items()} == (STATE_CODE_TO_NAME) + assert all(state.code == f"state/{code.lower()}" for code, state in states.items()) + assert all(state.parent_code == "us" for state in states.values()) + assert states["CA"].filter_value == "6" + assert states["DC"].filter_value == "11" + + v1_districts = { + entry["name"]: entry for entry in build_congressional_district_metadata() + } + assert set(districts) == set(v1_districts) + for district in CONGRESSIONAL_DISTRICTS: + code = f"congressional_district/{district.state_code}-{district.number:02d}" + reference = districts[code] + number = 0 if district.state_code in AT_LARGE_STATES else district.number + state_fips = int(states[district.state_code].filter_value or "") + assert reference.label == v1_districts[code]["label"] + assert reference.parent_code == f"state/{district.state_code.lower()}" + assert reference.state_code == district.state_code + assert reference.state_name == STATE_CODE_TO_NAME[district.state_code] + assert reference.filter_value == str(state_fips * 100 + number) + assert districts["congressional_district/CA-01"].filter_value == "601" + assert districts["congressional_district/AK-01"].filter_value == "200" + assert districts["congressional_district/DC-01"].filter_value == "1100" + + +def test_us_places_are_listed_under_their_state_without_a_filter() -> None: + places = [ + reference + for reference in load_region_references("us") + if reference.region_type == "place" + ] + + assert places + assert all(place.filter_value is None for place in places) + assert all( + place.parent_code == f"state/{(place.state_code or '').lower()}" + for place in places + ) + assert all(place.v1_code == place.code for place in places) + + +def test_parses_a_valid_minimal_file() -> None: + references = parse_region_references("us", _small_us_rows()) + + assert [reference.code for reference in references] == [ + "us", + "state/ca", + "congressional_district/CA-01", + ] + assert references[1] == RegionReference( + code="state/ca", + label="State/Ca", + region_type="state", + parent_code="us", + state_code=None, + state_name=None, + filter_value="6", + v1_code="state/ca", + ) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ( + lambda rows: rows.append(_row("state/ca", "state", parent_code="us")), + r"^us region reference file repeats region codes: \['state/ca'\]$", + ), + ( + lambda rows: rows[2].update(v1_code="state/ca"), + r"^us region reference file repeats v1 identifiers: \['state/ca'\]$", + ), + ( + lambda rows: rows[2].update(parent_code="state/ny"), + r"^region 'congressional_district/CA-01' names an unknown parent " + r"'state/ny'$", + ), + ( + lambda rows: rows.pop(0), + r"^us region reference file has no national region 'us'$", + ), + ( + lambda rows: rows[1].update(region_type="place"), + r"^region 'state/ca' does not start with its type 'place/'$", + ), + ( + lambda rows: rows.append(_row("county/06037", "county", parent_code="us")), + r"^region 'county/06037' has unsupported type 'county'$", + ), + ( + lambda rows: rows[1].update(parent_code="congressional_district/CA-01"), + r"^region 'state/ca' has a parent cycle$", + ), + ( + lambda rows: rows[0].update(parent_code="state/ca"), + r"^national region 'us' must have no parent$", + ), + ( + lambda rows: rows[1].update(parent_code=""), + r"^region 'state/ca' has no parent$", + ), + ( + lambda rows: rows[1].update(filter_value="x" * 256), + r"^region 'state/ca' filter_value exceeds 255 characters$", + ), + ( + lambda rows: rows[1].update(label=" California"), + r"^region 'state/ca' label has surrounding whitespace$", + ), + ( + lambda rows: rows[1].update(label=""), + r"^region 'state/ca' label must be a non-empty string$", + ), + ( + lambda rows: rows[1].update(code=""), + r"^us region reference file has an invalid code ''$", + ), + ( + lambda rows: rows[1].update(code="state/ca "), + r"^us region reference file has an invalid code 'state/ca '$", + ), + ( + lambda rows: rows[1].update(code="state/" + "c" * 250), + r"^us region reference file has an invalid code 'state/c+'$", + ), + ( + lambda rows: rows.append(_row("usa", "national")), + r"^national region 'usa' must be named 'us'$", + ), + ], +) +def test_rejects_an_inconsistent_reference_file(change, message: str) -> None: + rows = _small_us_rows() + change(rows) + + with pytest.raises(CatalogExtractionError, match=message): + parse_region_references("us", rows) + + +def test_names_the_missing_ancestor_further_up_a_regions_chain() -> None: + rows = [ + _row("us", "national"), + _row( + "congressional_district/CA-01", + "congressional_district", + parent_code="state/ca", + ), + _row("state/ca", "state", parent_code="state/zz"), + ] + + with pytest.raises( + CatalogExtractionError, + match=r"^region 'congressional_district/CA-01' descends from 'state/ca', " + r"whose parent 'state/zz' is unknown$", + ): + parse_region_references("us", rows) + + +def test_rejects_a_file_whose_columns_differ(tmp_path: Path) -> None: + (tmp_path / "us.csv").write_text( + "code,label,region_type\nus,United States,national\n", encoding="utf-8" + ) + + with pytest.raises( + CatalogExtractionError, + match=r"^us region reference file columns must be ", + ): + load_region_references("us", directory=tmp_path) + + +def test_rejects_a_country_without_a_reference_file(tmp_path: Path) -> None: + with pytest.raises( + CatalogExtractionError, + match=r"^ca has no region reference file$", + ): + load_region_references("ca", directory=tmp_path) diff --git a/tests/unit/v2/test_user_association_migration_qualification.py b/tests/unit/v2/test_user_association_migration_qualification.py index 463a073f9..ac3997cac 100644 --- a/tests/unit/v2/test_user_association_migration_qualification.py +++ b/tests/unit/v2/test_user_association_migration_qualification.py @@ -143,3 +143,48 @@ def fail(): payload = json.loads(capsys.readouterr().err) assert payload["outcome"] == "error" assert "secret" not in json.dumps(payload) + + +def test_saved_policies_are_counted_under_the_name_of_the_pending_revision() -> None: + """The reshape is pending only before the later rename of ``user_policies``. + + Counting through the current model would query ``user_policy_associations``, + which a database still waiting for the reshape does not have yet. + """ + _evidence, connection, _engine = _qualify((0, 0, 0, 0), pending=True) + + saved_policy_count = connection.statements[0] + assert "FROM user_policies" in saved_policy_count + assert "user_policy_associations" not in saved_policy_count + + +class RecordingExecutor: + def __init__(self) -> None: + self.statements: list[str] = [] + + def scalar(self, statement: object) -> int: + self.statements.append(str(statement)) + return 3 + + +def test_saved_policies_are_counted_under_their_name_after_the_rename() -> None: + """Once ``user_policies`` is renamed, its rows are counted under the new name. + + The command runs before every upgrade, so it meets the table under either + name; reading only the old name would report no saved policies. + """ + executor = RecordingExecutor() + + counts = qualification.read_user_association_counts( + executor, + existing_tables={ + "user_policy_associations", + "user_household_associations", + "user_simulation_associations", + "user_report_associations", + }, + ) + + assert counts.user_policies == 3 + saved_policy_count = executor.statements[0] + assert "FROM user_policy_associations" in saved_policy_count diff --git a/tests/unit/v2/test_user_household_routes.py b/tests/unit/v2/test_user_household_routes.py index b9bf0df75..3f92084e4 100644 --- a/tests/unit/v2/test_user_household_routes.py +++ b/tests/unit/v2/test_user_household_routes.py @@ -9,6 +9,9 @@ from sqlalchemy.exc import IntegrityError, SQLAlchemyError from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.associations.validators import ( + AssociationReferenceConflictError, +) from policyengine_api.services.v2.legacy_refs.validators import ( LegacyReferenceNotFoundError, ) @@ -141,17 +144,26 @@ def test_list_and_patch_use_the_session_user_and_legacy_fields() -> None: f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", json={"legacy_household_id": 124}, ) - reassigned = client.patch( - f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", - json={"household_id": str(HOUSEHOLD_ID)}, - ) user_filter = client.get(f"/v2/user-households?country_id=us&user_id={OWNER_ID}") + invalid_patches = [ + client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json=body, + ) + for body in ( + {"household_id": None}, + {"household_id": "not-a-uuid"}, + {"user_id": str(OWNER_ID)}, + {"legacy_household_id": None}, + {}, + ) + ] assert page.status_code == 200 assert page.json()["result"]["has_more"] is True assert patched.status_code == 200 - assert reassigned.status_code == 422 assert user_filter.status_code == 422 + assert [response.status_code for response in invalid_patches] == [422] * 5 assert service.calls[0] == ( "list_user_households", { @@ -168,6 +180,22 @@ def test_list_and_patch_use_the_session_user_and_legacy_fields() -> None: assert len(service.calls) == 2 +def test_patch_repoints_to_a_v2_household_reference() -> None: + client, service, flask_calls = _client() + + repointed = client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json={"household_id": str(HOUSEHOLD_ID)}, + ) + + assert repointed.status_code == 200 + assert service.calls[0][0] == "patch_user_household" + association_input = service.calls[0][1]["association_input"] + assert association_input.household_id == HOUSEHOLD_ID + assert association_input.model_fields_set == {"household_id"} + assert flask_calls["count"] == 0 + + @pytest.mark.parametrize( ("error", "status"), [ @@ -196,6 +224,37 @@ def test_failures_map_to_secret_safe_typed_errors( assert "credential" not in response.text +@pytest.mark.parametrize( + ("error", "status", "message"), + [ + (AssociationCountryConflictError("different country"), 400, "country"), + ( + AssociationReferenceConflictError("household_id mismatch"), + 400, + "mismatch", + ), + (AssociationHouseholdNotFoundError("household was not found"), 404, "not"), + (UserHouseholdNotFoundError("association was not found"), 404, "not"), + (LegacyReferenceNotFoundError("v1 household 9 was not found"), 422, "v1"), + ], +) +def test_repoint_failures_map_to_typed_errors( + error: Exception, status: int, message: str +) -> None: + service = RecordingService(_result) + service.error = error + client, _service, _flask_calls = _client(service) + + response = client.patch( + f"/v2/user-households/{ASSOCIATION_ID}?country_id=us", + json={"household_id": str(HOUSEHOLD_ID), "legacy_household_id": 9}, + ) + + assert response.status_code == status + assert response.json()["status"] == "error" + assert message in response.json()["message"].lower() + + def test_patch_preflight_and_native_routes_avoid_cloud_sql_and_flask( monkeypatch, ) -> None: @@ -241,7 +300,9 @@ def test_openapi_publishes_signed_in_household_contract() -> None: patch = schema["paths"]["/v2/user-households/{association_id}"]["patch"] patch_ref = patch["requestBody"]["content"]["application/json"]["schema"]["$ref"] patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] + assert patch_schema["additionalProperties"] is False assert set(patch_schema["properties"]) == { + "household_id", "legacy_household_id", "name", "description", diff --git a/tests/unit/v2/test_user_household_service.py b/tests/unit/v2/test_user_household_service.py index 944018c53..e084c4df6 100644 --- a/tests/unit/v2/test_user_household_service.py +++ b/tests/unit/v2/test_user_household_service.py @@ -2,10 +2,15 @@ from __future__ import annotations +from uuid import UUID + import pytest from sqlmodel import select from policyengine_api.data.v2.models import Household, UserHouseholdAssociation +from policyengine_api.services.v2.associations.validators import ( + AssociationReferenceConflictError, +) from policyengine_api.services.v2.legacy_refs.validators import ( LegacyReferenceNotFoundError, ) @@ -20,6 +25,7 @@ UserHouseholdUpdateInput, ) from policyengine_api.services.v2.user_households.validators import ( + AssociationCountryConflictError, AssociationHouseholdNotFoundError, UserHouseholdNotFoundError, ) @@ -101,6 +107,15 @@ def test_ownership_repointing_and_deletion(store) -> None: service.delete_user_household( principal=BOB, country_id="us", association_id=created.id ) + with pytest.raises(UserHouseholdNotFoundError): + service.patch_user_household( + principal=BOB, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id + ), + ) assert service.list_user_households(principal=BOB, country_id="us").items == () repointed = service.patch_user_household( @@ -132,14 +147,167 @@ def test_ownership_repointing_and_deletion(store) -> None: assert session.get(Household, selected.household_id) is not None +def test_repointing_to_a_v2_household_keeps_the_saved_row_and_clears_the_v1_reference( + store, +) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields(name="Family", description="Note") + ) + + repointed = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput(household_id=selected.household_id), + ) + + assert (repointed.household_id, repointed.legacy_household_id) == ( + selected.household_id, + None, + ) + assert (repointed.id, repointed.name, repointed.description) == ( + created.id, + "Family", + "Note", + ) + assert repointed.created_at == created.created_at + assert repointed.updated_at >= created.updated_at + with selected.sessions() as session: + stored = session.get(UserHouseholdAssociation, created.id) + assert stored is not None + assert (stored.household_id, stored.legacy_household_id) == ( + selected.household_id, + None, + ) + + +def test_repointing_to_both_household_references_stores_both(store) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields() + ) + + copied = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id, + legacy_household_id=MAPPED_LEGACY_HOUSEHOLD_ID, + ), + ) + assert (copied.household_id, copied.legacy_household_id) == ( + selected.household_id, + MAPPED_LEGACY_HOUSEHOLD_ID, + ) + + # v1 household 11 has no v2 copy, so nothing can contradict the v2 + # reference: both are stored, as creation stores them. + uncopied = service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserHouseholdUpdateInput( + household_id=selected.household_id, legacy_household_id=11 + ), + ) + assert (uncopied.household_id, uncopied.legacy_household_id) == ( + selected.household_id, + 11, + ) + assert uncopied.id == created.id + + +def test_repointing_to_a_missing_foreign_or_conflicting_household_changes_nothing( + store, +) -> None: + service, selected = store + created = service.create_user_household( + principal=ALICE, association_input=_fields(name="Family") + ) + with selected.sessions.begin() as session: + other_us_household = Household( + country_id="us", + default_year=2026, + household_data={"people": [], "household": []}, + canonicalization_version=1, + content_hash="e" * 64, + ) + session.add(other_us_household) + session.flush() + other_us_household_id = other_us_household.id + + rejected_updates = ( + ( + AssociationHouseholdNotFoundError, + UserHouseholdUpdateInput( + household_id=UUID("00000000-0000-0000-0000-000000000099"), + name="Lost", + ), + ), + ( + AssociationCountryConflictError, + UserHouseholdUpdateInput( + household_id=selected.uk_household_id, name="Lost" + ), + ), + ( + AssociationReferenceConflictError, + UserHouseholdUpdateInput( + household_id=other_us_household_id, + legacy_household_id=MAPPED_LEGACY_HOUSEHOLD_ID, + name="Lost", + ), + ), + ( + LegacyReferenceNotFoundError, + UserHouseholdUpdateInput( + household_id=selected.household_id, + legacy_household_id=404, + name="Lost", + ), + ), + ) + for expected_error, rejected_update in rejected_updates: + with pytest.raises(expected_error): + service.patch_user_household( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=rejected_update, + ) + + unchanged = service.get_user_household( + principal=ALICE, country_id="us", association_id=created.id + ) + assert ( + unchanged.household_id, + unchanged.legacy_household_id, + unchanged.name, + ) == (None, 10, "Family") + assert unchanged.updated_at == created.updated_at + + def test_patch_input_rejects_reassignment_and_empty_changes() -> None: - with pytest.raises(ValueError, match="At least one of legacy_household_id"): - UserHouseholdUpdateInput() with pytest.raises( - ValueError, match=r"household_id\n Extra inputs are not permitted" + ValueError, match="At least one of household_id, legacy_household_id" ): + UserHouseholdUpdateInput() + with pytest.raises(ValueError, match=r"user_id\n Extra inputs are not permitted"): UserHouseholdUpdateInput.model_validate( - {"household_id": "00000000-0000-0000-0000-000000000001"} + {"user_id": "00000000-0000-0000-0000-0000000000a1"} ) with pytest.raises(ValueError, match="legacy_household_id must not be null"): UserHouseholdUpdateInput(legacy_household_id=None) + with pytest.raises(ValueError, match="household_id must not be null"): + UserHouseholdUpdateInput(household_id=None) + + +def test_patch_input_accepts_a_v2_household_reference() -> None: + repoint = UserHouseholdUpdateInput.model_validate( + {"household_id": "00000000-0000-0000-0000-000000000001"} + ) + + assert repoint.household_id == UUID("00000000-0000-0000-0000-000000000001") + assert repoint.model_fields_set == {"household_id"} diff --git a/tests/unit/v2/test_user_policy_legacy_persistence.py b/tests/unit/v2/test_user_policy_legacy_persistence.py index e52134e67..a6911a0db 100644 --- a/tests/unit/v2/test_user_policy_legacy_persistence.py +++ b/tests/unit/v2/test_user_policy_legacy_persistence.py @@ -44,13 +44,15 @@ SELECT_MAPPING_FOR_UPDATE = f"{SELECT_MAPPING} FOR UPDATE" MAPPING_LOOKUP = {"country_id_1": "us", "legacy_user_policy_id_1": 10} SELECT_MAPPED_ASSOCIATION = ( - "SELECT user_policies.id, user_policies.created_at, user_policies.updated_at, " - "user_policies.user_id, user_policies.policy_id, " - "user_policies.legacy_policy_id, user_policies.country_id, " - "user_policies.name, user_policies.description, user_policies.baseline, " - "user_policies.provenance \nFROM user_policies \n" - "WHERE user_policies.id = %(id_1)s::UUID AND " - "user_policies.country_id = %(country_id_1)s::VARCHAR" + "SELECT user_policy_associations.id, user_policy_associations.created_at, " + "user_policy_associations.updated_at, user_policy_associations.user_id, " + "user_policy_associations.policy_id, " + "user_policy_associations.legacy_policy_id, " + "user_policy_associations.country_id, user_policy_associations.name, " + "user_policy_associations.description, user_policy_associations.baseline, " + "user_policy_associations.provenance \nFROM user_policy_associations \n" + "WHERE user_policy_associations.id = %(id_1)s::UUID AND " + "user_policy_associations.country_id = %(country_id_1)s::VARCHAR" ) INSERT_MAPPING = ( "INSERT INTO legacy_user_policy_mappings (id, created_at, updated_at, " diff --git a/tests/unit/v2/test_user_policy_routes.py b/tests/unit/v2/test_user_policy_routes.py index 10d837038..37c969add 100644 --- a/tests/unit/v2/test_user_policy_routes.py +++ b/tests/unit/v2/test_user_policy_routes.py @@ -250,7 +250,8 @@ def test_patch_repoints_legacy_reference_and_rejects_invalid_changes() -> None: json=body, ) for body in ( - {"policy_id": str(POLICY_ID)}, + {"policy_id": None}, + {"policy_id": "not-a-uuid"}, {"user_id": str(OWNER_ID)}, {"client_ref": "rf-2"}, {"legacy_policy_id": None}, @@ -262,11 +263,27 @@ def test_patch_repoints_legacy_reference_and_rejects_invalid_changes() -> None: association_input = service.calls[0][1]["association_input"] assert association_input.legacy_policy_id == 101 assert association_input.model_fields_set == {"legacy_policy_id", "name"} - assert [response.status_code for response in invalid] == [422] * 5 + assert [response.status_code for response in invalid] == [422] * 6 assert [call[0] for call in service.calls] == ["patch_user_policy"] assert flask_calls["count"] == 0 +def test_patch_repoints_to_a_v2_policy_reference() -> None: + client, service, flask_calls = _client() + + repointed = client.patch( + f"/v2/user-policies/{ASSOCIATION_ID}?country_id=us", + json={"policy_id": str(POLICY_ID)}, + ) + + assert repointed.status_code == 200 + assert service.calls[0][0] == "patch_user_policy" + association_input = service.calls[0][1]["association_input"] + assert association_input.policy_id == POLICY_ID + assert association_input.model_fields_set == {"policy_id"} + assert flask_calls["count"] == 0 + + def test_delete_returns_no_content_for_the_session_user() -> None: client, service, _flask_calls = _client() @@ -383,6 +400,7 @@ def test_openapi_publishes_signed_in_association_contracts() -> None: patch_schema = schema["components"]["schemas"][patch_ref.rsplit("/", 1)[-1]] assert patch_schema["additionalProperties"] is False assert set(patch_schema["properties"]) == { + "policy_id", "legacy_policy_id", "name", "description", diff --git a/tests/unit/v2/test_user_policy_service.py b/tests/unit/v2/test_user_policy_service.py index 70352bb34..f1a69c048 100644 --- a/tests/unit/v2/test_user_policy_service.py +++ b/tests/unit/v2/test_user_policy_service.py @@ -164,7 +164,7 @@ def test_create_rejects_bad_references_before_writing(store) -> None: def test_other_users_rows_are_not_found_and_lists_are_owner_scoped(store) -> None: - service, _selected, _legacy_references = store + service, selected, _legacy_references = store alice_row = service.create_user_policy( principal=ALICE, association_input=_fields(name="Alice") ) @@ -183,6 +183,12 @@ def test_other_users_rows_are_not_found_and_lists_are_owner_scoped(store) -> Non association_id=alice_row.id, association_input=UserPolicyUpdateInput(name="stolen"), ), + lambda: service.patch_user_policy( + principal=BOB, + country_id="us", + association_id=alice_row.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ), lambda: service.delete_user_policy( principal=BOB, country_id="us", association_id=alice_row.id ), @@ -253,6 +259,145 @@ def test_repointing_re_resolves_or_clears_the_v2_reference(store) -> None: assert legacy_references.calls[-1] == ("us", {"policy": [404]}) +def test_repointing_to_a_v2_policy_keeps_the_saved_row_and_clears_the_v1_reference( + store, +) -> None: + service, selected, legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + v1_checks_before_repoint = list(legacy_references.calls) + + repointed = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ) + + assert (repointed.policy_id, repointed.legacy_policy_id) == ( + selected.policy_id, + None, + ) + assert (repointed.id, repointed.name, repointed.description) == ( + created.id, + "Saved reform", + "Personal note", + ) + assert repointed.created_at == created.created_at + assert repointed.updated_at >= created.updated_at + assert legacy_references.calls == v1_checks_before_repoint + with selected.sessions() as session: + stored = session.get(UserPolicy, created.id) + assert stored is not None + assert (stored.policy_id, stored.legacy_policy_id) == ( + selected.policy_id, + None, + ) + + +def test_repointing_to_both_references_stores_both(store) -> None: + service, selected, legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + + mirrored = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput( + policy_id=selected.policy_id, + legacy_policy_id=MAPPED_LEGACY_POLICY_ID, + ), + ) + assert (mirrored.policy_id, mirrored.legacy_policy_id) == ( + selected.policy_id, + MAPPED_LEGACY_POLICY_ID, + ) + assert legacy_references.calls[-1] == ( + "us", + {"policy": [MAPPED_LEGACY_POLICY_ID]}, + ) + + # v1 policy 3 has no v2 copy, so nothing can contradict the v2 reference: + # both are stored, as creation stores them. + unmirrored = service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput( + policy_id=selected.policy_id, legacy_policy_id=3 + ), + ) + assert (unmirrored.policy_id, unmirrored.legacy_policy_id) == ( + selected.policy_id, + 3, + ) + assert unmirrored.id == created.id + + +def test_repointing_to_a_missing_foreign_or_conflicting_policy_changes_nothing( + store, +) -> None: + service, selected, _legacy_references = store + created = service.create_user_policy(principal=ALICE, association_input=_fields()) + with selected.sessions.begin() as session: + mirrored_policy = session.get(Policy, selected.policy_id) + assert mirrored_policy is not None + other_us_policy = Policy( + country_id="us", + tax_benefit_model_id=mirrored_policy.tax_benefit_model_id, + tax_benefit_model_version_id=mirrored_policy.tax_benefit_model_version_id, + canonicalization_version=1, + content_hash="e" * 64, + ) + session.add(other_us_policy) + session.flush() + other_us_policy_id = other_us_policy.id + + rejected_updates = ( + ( + AssociationPolicyNotFoundError, + UserPolicyUpdateInput( + policy_id=UUID("00000000-0000-0000-0000-000000000099"), name="Lost" + ), + ), + ( + AssociationCountryConflictError, + UserPolicyUpdateInput(policy_id=selected.uk_policy_id, name="Lost"), + ), + ( + AssociationReferenceConflictError, + UserPolicyUpdateInput( + policy_id=other_us_policy_id, + legacy_policy_id=MAPPED_LEGACY_POLICY_ID, + name="Lost", + ), + ), + ( + LegacyReferenceNotFoundError, + UserPolicyUpdateInput( + policy_id=selected.policy_id, legacy_policy_id=404, name="Lost" + ), + ), + ) + for expected_error, rejected_update in rejected_updates: + with pytest.raises(expected_error): + service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=rejected_update, + ) + + unchanged = service.get_user_policy( + principal=ALICE, country_id="us", association_id=created.id + ) + assert (unchanged.policy_id, unchanged.legacy_policy_id, unchanged.name) == ( + None, + 1, + "Saved reform", + ) + assert unchanged.updated_at == created.updated_at + + def test_rows_owned_by_the_stage_10_mirror_conflict(store) -> None: service, selected, _legacy_references = store created = service.create_user_policy(principal=ALICE, association_input=_fields()) @@ -274,6 +419,13 @@ def test_rows_owned_by_the_stage_10_mirror_conflict(store) -> None: association_id=created.id, association_input=UserPolicyUpdateInput(name="Renamed"), ) + with pytest.raises(MirrorOwnedAssociationError): + service.patch_user_policy( + principal=ALICE, + country_id="us", + association_id=created.id, + association_input=UserPolicyUpdateInput(policy_id=selected.policy_id), + ) with pytest.raises(MirrorOwnedAssociationError): service.delete_user_policy( principal=ALICE, country_id="us", association_id=created.id @@ -331,13 +483,22 @@ def test_stage_10_projection_records_the_v1_reform_id() -> None: def test_patch_input_requires_a_supported_field() -> None: - with pytest.raises(ValueError, match="At least one of legacy_policy_id"): + with pytest.raises(ValueError, match="At least one of policy_id, legacy_policy_id"): UserPolicyUpdateInput() - with pytest.raises( - ValueError, match=r"policy_id\n Extra inputs are not permitted" - ): + with pytest.raises(ValueError, match=r"user_id\n Extra inputs are not permitted"): UserPolicyUpdateInput.model_validate( - {"policy_id": "00000000-0000-0000-0000-000000000001"} + {"user_id": "00000000-0000-0000-0000-0000000000a1"} ) with pytest.raises(ValueError, match="legacy_policy_id must not be null"): UserPolicyUpdateInput(legacy_policy_id=None) + with pytest.raises(ValueError, match="policy_id must not be null"): + UserPolicyUpdateInput(policy_id=None) + + +def test_patch_input_accepts_a_v2_policy_reference() -> None: + repoint = UserPolicyUpdateInput.model_validate( + {"policy_id": "00000000-0000-0000-0000-000000000001"} + ) + + assert repoint.policy_id == UUID("00000000-0000-0000-0000-000000000001") + assert repoint.model_fields_set == {"policy_id"} diff --git a/tests/unit/v2/user_service_support.py b/tests/unit/v2/user_service_support.py index ce54237e3..6018df644 100644 --- a/tests/unit/v2/user_service_support.py +++ b/tests/unit/v2/user_service_support.py @@ -31,6 +31,7 @@ from policyengine_api.services.v2.legacy_refs.validators import ( require_legacy_references, ) +from tests.fixtures.v2_countries import add_listed_countries if TYPE_CHECKING: from collections.abc import Collection, Iterator, Mapping @@ -106,6 +107,7 @@ class V2Store: policy_id: UUID uk_policy_id: UUID household_id: UUID + uk_household_id: UUID @contextmanager @@ -128,6 +130,8 @@ def begin_transaction(connection) -> None: V2_METADATA.create_all(engine) sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + with sessions() as session: + add_listed_countries(session) with sessions.begin() as session: model = TaxBenefitModel(name="policyengine-us") version = TaxBenefitModelVersion( @@ -166,7 +170,14 @@ def begin_transaction(connection) -> None: canonicalization_version=1, content_hash="c" * 64, ) - session.add_all([value, uk_policy, household]) + uk_household = Household( + country_id="uk", + default_year=2026, + household_data={"people": [], "household": []}, + canonicalization_version=1, + content_hash="f" * 64, + ) + session.add_all([value, uk_policy, household, uk_household]) session.flush() session.add_all( [ @@ -192,6 +203,7 @@ def begin_transaction(connection) -> None: policy_id=policy.id, uk_policy_id=uk_policy.id, household_id=household.id, + uk_household_id=uk_household.id, ) try: yield store diff --git a/uv.lock b/uv.lock index 094aec459..f9bdfaf54 100644 --- a/uv.lock +++ b/uv.lock @@ -2786,12 +2786,15 @@ dependencies = [ { name = "google-cloud-logging" }, { name = "google-cloud-secret-manager" }, { name = "gunicorn" }, + { name = "h5py" }, { name = "httpx" }, + { name = "huggingface-hub" }, { name = "markupsafe" }, { name = "microdf-python" }, { name = "openai" }, { name = "packaging" }, { name = "policyengine", extra = ["models"] }, + { name = "policyengine-api-simulation-client" }, { name = "policyengine-canada" }, { name = "policyengine-il" }, { name = "policyengine-ng" }, @@ -2840,13 +2843,16 @@ requires-dist = [ { name = "google-cloud-logging", specifier = ">=3,<4" }, { name = "google-cloud-secret-manager", specifier = ">=2,<3" }, { name = "gunicorn" }, + { name = "h5py", specifier = ">=3.16,<4" }, { name = "httpx", specifier = ">=0.27.0" }, + { name = "huggingface-hub", specifier = ">=1.12,<2" }, { name = "markupsafe", specifier = ">=3,<4" }, { name = "microdf-python", specifier = ">=1.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.15,<2" }, { name = "openai" }, { name = "packaging", specifier = ">=24,<27" }, { name = "policyengine", extras = ["models"], specifier = "==6.10.0" }, + { name = "policyengine-api-simulation-client", specifier = ">=0.20261027.288" }, { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, @@ -2872,6 +2878,19 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "policyengine-api-simulation-client" +version = "0.20261027.288" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "httpx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/21/145e238dec20e79a1ce7fb4818271e08603fab88656ce4d7e5c7307ab90e/policyengine_api_simulation_client-0.20261027.288.tar.gz", hash = "sha256:5129724a3e5ba4b0dc257d421c32dd1be7f5c4a1ae3c1a9b60b20d7fab58f9f5", size = 44777, upload-time = "2026-10-27T17:02:39.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/27/85e9da0f0561d0548cbe23ec46e4bf63a4b814d7adbe1f0932c514b52191/policyengine_api_simulation_client-0.20261027.288-py3-none-any.whl", hash = "sha256:82b4aacb954de0b8c747b313c1b92ea1d6a028402618f3d819c8298b87977486", size = 99950, upload-time = "2026-10-27T17:02:37.398Z" }, +] + [[package]] name = "policyengine-canada" version = "0.96.3"