From bdbb58ce1fa268d0c2f1d47ed18d45e09c8ae19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:29:30 +0200 Subject: [PATCH 01/14] Rename the local-areas line to uk-local and ratify its Logbook scope The rowwise drivers' pipeline names move from uk-locals-* to uk-local-* so the derived chain scope is uk/local (Maria's naming ruling on #761), and the scope is ratified for the local-areas line's first archived run: the scope_declared migration, the DECLARED_SCOPES mirror, and the README vocabulary move together, per the closed-world convention. No archived rows existed under the old names (the scope was never ratified), so the rename is history-free. Co-Authored-By: Claude Fable 5 --- logbook/README.md | 17 +++++----- .../tests/test_logbook_adoption.py | 10 +++--- .../tests/test_logbook_chain_scopes_pg.py | 19 +++++------ .../microcosm-build/tests/test_logbook_cli.py | 32 +++++++++++++++---- .../tests/test_uk_rowwise_build_driver.py | 2 +- .../tests/test_uk_rowwise_candidate.py | 2 +- .../20260829000000_logbook_uk_local_scope.sql | 13 ++++++++ tools/build_uk_rowwise_candidate.py | 4 +-- tools/build_uk_rowwise_dataset.py | 4 +-- tools/logbook.py | 5 +-- 10 files changed, 71 insertions(+), 37 deletions(-) create mode 100644 supabase/migrations/20260829000000_logbook_uk_local_scope.sql diff --git a/logbook/README.md b/logbook/README.md index ff96975853..a148dd142a 100644 --- a/logbook/README.md +++ b/logbook/README.md @@ -22,7 +22,7 @@ the digest and destroy the evidence the chain exists to preserve. Every new archive uses `logbook//.jsonl`. The dataset token names the line's base data, not an epic and not a build mechanism: `frs` is the FRS-derived survey-microdata line (persons, benunits, and households — -its spine, staging, and every imputation stage share one chain), `locals` is +its spine, staging, and every imputation stage share one chain), `local` is the local-areas product line, and a future firm line will carry its own base-data name. Different base data need different scopes, and scope is also where builds serialize: if two lines must append concurrently, they need @@ -30,13 +30,12 @@ separate chains. ## The vocabulary is closed-world -The ratified scopes are exactly `us` and `uk/frs` — deliberately minimal: a -scope is ratified when its line starts archiving, not speculatively. The -`uk-locals-*` drivers already follow the naming convention, and their rows -spool locally regardless (recording never consults this list); `uk/locals` -gets ratified when the local-areas line first archives. A pipeline whose -name derives any unratified scope is refused — even at genesis. Opening a -scope is a reviewed decision made in three places together: the +The ratified scopes are exactly `us`, `uk/frs`, and `uk/local` — deliberately +minimal: a scope is ratified when its line starts archiving, not speculatively. +The `uk/local` scope was ratified for the local-areas line's first archived run +in #761. A pipeline whose name derives any unratified scope is refused — even +at genesis. Opening a scope is a reviewed decision made in three places +together: the `logbook.scope_declared` migration, the `DECLARED_SCOPES` mirror in `tools/logbook.py`, and this list. It is never a side effect of a well-formed pipeline name: without the list, a typo'd `uk-huseholds-...` @@ -102,7 +101,7 @@ The database keeps `builds_unique_predecessor` global because two rows claiming one predecessor is a fork wherever it happens. Genesis, tail discovery, and advisory locking are scoped: `logbook.chain_scope(pipeline)` maps the three legacy US pipeline names to `us`, and maps new pipelines like -`uk-frs-staging` or `uk-locals-rowwise` to `uk/frs` and `uk/locals`; +`uk-frs-staging` or `uk-local-rowwise` to `uk/frs` and `uk/local`; `logbook.scope_declared` then refuses any scope outside the ratified list. The per-scope advisory lock lets independent scopes append concurrently while appends within one scope still serialize. diff --git a/packages/microcosm-build/tests/test_logbook_adoption.py b/packages/microcosm-build/tests/test_logbook_adoption.py index 0432d5879e..e0bafe7db5 100644 --- a/packages/microcosm-build/tests/test_logbook_adoption.py +++ b/packages/microcosm-build/tests/test_logbook_adoption.py @@ -189,7 +189,7 @@ def test_error_receipt_and_placeholder_drop_only_when_sole_key(tmp_path: Path) - path = write_error_receipt( error_receipt_path(tmp_path, build_id=state.build_id), state=state, - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", error=RuntimeError("boom"), ) apply_error_verdict( @@ -205,7 +205,7 @@ def test_error_receipt_and_placeholder_drop_only_when_sole_key(tmp_path: Path) - } assert state.phases_reached == ["attempt_started", "error"] payload = json.loads(path.read_text(encoding="utf-8")) - assert payload["pipeline"] == "uk-locals-rowwise" + assert payload["pipeline"] == "uk-local-rowwise" assert payload["error_type"] == "builtins.RuntimeError" assert payload["message"] == "boom" @@ -240,7 +240,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec started_at=1.0, now=lambda: 3.5, started_ts=datetime(2026, 8, 13, 12, 0, tzinfo=UTC), - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", rung="f100", seed=42, code_pin="a" * 40, @@ -253,7 +253,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec row = load_logbook_row(spool_path) assert frozenset(row.to_mapping()) == LOGBOOK_ROW_FIELDS assert row.build_id == "uk-fixture-attempt" - assert row.pipeline == "uk-locals-rowwise" + assert row.pipeline == "uk-local-rowwise" assert row.phases_reached == ("attempt_started", "configured") assert row.wall_seconds == 2.5 @@ -263,7 +263,7 @@ def test_record_terminal_attempt_writes_one_validated_row_and_refuses_double_rec started_at=1.0, now=lambda: 4.0, started_ts=datetime(2026, 8, 13, 12, 0, tzinfo=UTC), - pipeline="uk-locals-rowwise", + pipeline="uk-local-rowwise", rung="f100", seed=42, code_pin="a" * 40, diff --git a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py index 32bddc801c..1230c84cd9 100644 --- a/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py +++ b/packages/microcosm-build/tests/test_logbook_chain_scopes_pg.py @@ -30,7 +30,10 @@ ROOT = Path(__file__).resolve().parents[3] MIGRATIONS = ROOT / "supabase/migrations" -NEW_MIGRATION = MIGRATIONS / "20260818000000_logbook_chain_scopes.sql" +CHAIN_SCOPES_MIGRATION = MIGRATIONS / "20260818000000_logbook_chain_scopes.sql" +UK_LOCAL_SCOPE_MIGRATION = ( + MIGRATIONS / "20260829000000_logbook_uk_local_scope.sql" +) ROWS = ROOT / "logbook/us.jsonl" BASE_MIGRATIONS = [ "20260805000000_logbook.sql", @@ -172,7 +175,8 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None _insert(connection, row) assert _count_builds(connection) == 28 - _apply_sql(connection, NEW_MIGRATION.read_text(encoding="utf-8")) + _apply_sql(connection, CHAIN_SCOPES_MIGRATION.read_text(encoding="utf-8")) + _apply_sql(connection, UK_LOCAL_SCOPE_MIGRATION.read_text(encoding="utf-8")) us_tail = _digest_of(connection, archived[-1]["build_id"]) _insert( @@ -198,17 +202,14 @@ def test_logbook_chain_scopes_migration_preserves_and_scopes_live_rows() -> None predecessor=None, ), ) - # Vocabulary is closed-world and minimal: uk/locals and uk/firms both - # derive cleanly but are not ratified yet, so even their geneses are - # refused until a migration adds them (locals rows keep spooling locally - # in the meantime — recording never consults the allowlist). Cross-scope + # Vocabulary is closed-world and minimal: uk/local is ratified for #761, + # while uk/firms still derives cleanly but remains unratified. Cross-scope # independence is already proven above: uk/frs opened while us had rows. - _refuses( + _insert( connection, _build_row( - "uk-locals-genesis", pipeline="uk-locals-rowwise", predecessor=None + "uk-local-genesis", pipeline="uk-local-rowwise", predecessor=None ), - "not in the ratified scope list", ) _refuses( connection, diff --git a/packages/microcosm-build/tests/test_logbook_cli.py b/packages/microcosm-build/tests/test_logbook_cli.py index 15c3a56f2d..de3e0b752f 100644 --- a/packages/microcosm-build/tests/test_logbook_cli.py +++ b/packages/microcosm-build/tests/test_logbook_cli.py @@ -246,14 +246,14 @@ def test_cli_local_export_refuses_wrong_scope_rows( capsys: pytest.CaptureFixture[str], ) -> None: # The adversarial finding on this PR: the chain verifier authenticates - # payloads, never filenames, so without this gate a uk-locals spool + # payloads, never filenames, so without this gate a uk-local spool # exported into the uk/frs archive would chain validly and # permanently mis-scope lineage. Local exports get the same scope # discipline as the remote branch. archive = tmp_path / "logbook" / "uk" / "frs.jsonl" archive.parent.mkdir(parents=True) source = tmp_path / "source.jsonl" - _write_jsonl(source, _chain(pipeline="uk-locals-rowwise")) + _write_jsonl(source, _chain(pipeline="uk-local-rowwise")) cli = _load_cli() exit_code = cli.main( @@ -263,7 +263,7 @@ def test_cli_local_export_refuses_wrong_scope_rows( assert exit_code == 1 err = capsys.readouterr().err assert "outside scope uk/frs" in err - assert "uk-locals-rowwise" in err + assert "uk-local-rowwise" in err assert not archive.exists() @@ -289,6 +289,26 @@ def test_cli_export_refuses_an_unratified_scope_archive( assert not archive.exists() +def test_cli_export_accepts_uk_local_scope_archive( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + archive = tmp_path / "logbook" / "uk" / "local.jsonl" + archive.parent.mkdir(parents=True) + source = tmp_path / "source.jsonl" + rows = _chain(pipeline="uk-local-rowwise") + _write_jsonl(source, rows) + cli = _load_cli() + + exit_code = cli.main( + ["export", "--archive", str(archive), "--source", str(source)] + ) + + assert exit_code == 0 + assert "exported 3 new Logbook rows" in capsys.readouterr().out + assert load_logbook_file(archive) == rows + + def test_cli_export_chain_orders_a_spool_directory( tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -557,7 +577,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: - rows = _chain(pipeline="uk-locals-rowwise") + rows = _chain(pipeline="uk-local-rowwise") archive = tmp_path / "logbook" / "uk" / "frs.jsonl" cli = _load_cli() monkeypatch.setenv("POPULACE_LEDGER_URL", "https://fixture.supabase.co") @@ -574,7 +594,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( assert result == 1 err = capsys.readouterr().err assert "outside scope uk/frs" in err - assert "uk-locals-rowwise" in err + assert "uk-local-rowwise" in err @pytest.mark.parametrize( @@ -584,7 +604,7 @@ def test_cli_remote_export_refuses_wrong_scope_rows( ("us-pool-inc2", "us"), ("us-stacked-pool", "us"), ("uk-frs-staging", "uk/frs"), - ("uk-locals-rowwise", "uk/locals"), + ("uk-local-rowwise", "uk/local"), ("us-pool-inc3", "us/pool"), ("mystery-pipeline", None), ], diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index 2ece175f64..3216ebb273 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -202,7 +202,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( rows = _spool_rows(output_dir) assert len(rows) == 1 first_row = rows[0] - assert first_row.pipeline == "uk-locals-rowwise" + assert first_row.pipeline == "uk-local-rowwise" assert first_row.rung == "f100" assert first_row.seed == 42 assert first_row.disposition == "iterating" diff --git a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py index b7965cadd4..f98c54f1ca 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_candidate.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_candidate.py @@ -349,7 +349,7 @@ def test_candidate_build_writes_calibrated_h5_and_evidence(tmp_path) -> None: rows = _spool_rows(output_dir) assert len(rows) == 1 row = rows[0] - assert row.pipeline == "uk-locals-candidate" + assert row.pipeline == "uk-local-candidate" assert row.rung == "f100" assert row.seed == 7 assert row.disposition == "iterating" diff --git a/supabase/migrations/20260829000000_logbook_uk_local_scope.sql b/supabase/migrations/20260829000000_logbook_uk_local_scope.sql new file mode 100644 index 0000000000..ed9aa23342 --- /dev/null +++ b/supabase/migrations/20260829000000_logbook_uk_local_scope.sql @@ -0,0 +1,13 @@ +-- Ratify uk/local for the local-areas line's first archived run (#761). +CREATE OR REPLACE FUNCTION logbook.scope_declared(p_scope text) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +SET search_path = pg_catalog +AS $function$ + SELECT p_scope IN ('us', 'uk/frs', 'uk/local'); +$function$; + +GRANT EXECUTE ON FUNCTION logbook.scope_declared(text) + TO logbook_writer, logbook_exporter, logbook_break_glass_admin; diff --git a/tools/build_uk_rowwise_candidate.py b/tools/build_uk_rowwise_candidate.py index 34b18da459..cc39600e09 100644 --- a/tools/build_uk_rowwise_candidate.py +++ b/tools/build_uk_rowwise_candidate.py @@ -86,7 +86,7 @@ _TARGET_RECORDS: int | None = None _L0_LAMBDA = 0.0 _BUDGET_ITERS = 10 -_UK_CANDIDATE_PIPELINE = "uk-locals-candidate" +_UK_CANDIDATE_PIPELINE = "uk-local-candidate" _REPOSITORY = Path(__file__).resolve().parents[1] _PAST_CAP_COUNT_KEYS = ( "n_targets", @@ -113,7 +113,7 @@ def __init__( def _new_candidate_build_id(*, seed: int, timestamp: datetime) -> str: instant = timestamp.astimezone(UTC) return ( - f"uk-locals-candidate-f100-s{seed}-" + f"uk-local-candidate-f100-s{seed}-" f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" ) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 0b6fb75412..6d47e28381 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -71,7 +71,7 @@ COVERAGE_FILENAME = "geography_coverage_summary.csv" DRY_RUN_PLAN_FILENAME = "rowwise_dry_run_plan.json" EXPECTED_SUPPORT_BOTTOM_AREAS = 15 -_UK_ROWWISE_PIPELINE = "uk-locals-rowwise" +_UK_ROWWISE_PIPELINE = "uk-local-rowwise" _REPOSITORY = Path(__file__).resolve().parents[1] @@ -203,7 +203,7 @@ def _new_rowwise_build_id( ) -> str: instant = timestamp.astimezone(UTC) return ( - f"uk-locals-rowwise-{route}-f100-s{seed}-" + f"uk-local-rowwise-{route}-f100-s{seed}-" f"{instant.strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex[:8]}" ) diff --git a/tools/logbook.py b/tools/logbook.py index 4956799fde..78aaee9d40 100644 --- a/tools/logbook.py +++ b/tools/logbook.py @@ -60,11 +60,12 @@ _PIPELINE_SCOPE_PATTERN = re.compile( r"^(?P[a-z]{2})-(?P[a-z0-9_]+)(?:-[a-z0-9_-]+)?$" ) -#: Mirror of logbook.scope_declared() in the same migration: the ratified +#: Mirror of logbook.scope_declared() in +#: supabase/migrations/20260829000000_logbook_uk_local_scope.sql: the ratified #: scope vocabulary, closed-world. Opening a scope is a reviewed diff here, #: in the migration, and in logbook/README.md -- never a side effect of a #: well-formed pipeline name. -DECLARED_SCOPES = frozenset({"us", "uk/frs"}) +DECLARED_SCOPES = frozenset({"us", "uk/frs", "uk/local"}) def _parser() -> argparse.ArgumentParser: From 48e355d0933f0c2b93e5f466a8a8007305606e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:38:10 +0200 Subject: [PATCH 02/14] Restate the source-lineage rule for the spine's explicit lineage columns The household_id-mod-10^8 rule was written for the certified pool's tier-offset id scheme. The microcosm spine's ids stack flag-driven offsets (sernum + 1{spi}.10^d + 1{cgt_clone}.10^(d+1) + 1{band_donor}.10^(d+2)) and carry explicit lineage columns, so apply_uk_source_lineage_modulus now refuses any input carrying them -- the columns are the source of truth, and a derived modulus layer would mint a false second lineage. The dry-run plan and build manifest gain an additive explicit lineage block (distinct source households, per-channel splits, flag counts, computed at pre-clone grain), null for pool inputs. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/rowwise_dataset.py | 40 +++++++--- .../tests/test_uk_ladder_rowwise_clone.py | 74 ++++++++++++++++++- .../tests/test_uk_rowwise_dry_run.py | 3 + .../tests/test_uk_rowwise_weight_metadata.py | 13 ++++ tools/build_uk_rowwise_dataset.py | 53 ++++++++++--- 5 files changed, 163 insertions(+), 20 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index 86fb5544d0..f98d3aceb5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -51,6 +51,17 @@ #: staging H5 already carries. POOL_SOURCE_LINEAGE_COLUMN = "pool_source_household_id" +#: Explicit lineage carried by accepted UK spine inputs. These columns are +#: authoritative for the spine id scheme and make the pool-only modulus rule +#: inapplicable. +UK_SPINE_LINEAGE_COLUMNS = ( + "household_support_channel", + "household_support_clone_index", + "household_is_spi_synthetic", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", +) + PERSON_ID_COLUMNS = ( "person_id", "person_household_id", @@ -892,15 +903,15 @@ def apply_uk_source_lineage_modulus( ) -> pd.DataFrame: """Derive pool-grain lineage as ``pool_source_household_id``. - The certified UK pool encodes its 10x clone tiers as - ``household_id = tier * 10**8 + base``, so a modulus of ``10**8`` recovers - the enhanced-FRS pool source. The derived column is a *distinct lineage - layer*: any immediate-layer ``source_household_id``/``source_household_key`` - the input carries (the national staging H5 does) is left untouched. The - modulus is only meaningful for rows whose ids follow the pool's tier - scheme — on a seam output, channel-rebuilt households (e.g. the rebuilt - SPI channel) carry ids outside that scheme, so pool-lineage diagnostics - must be read per support channel. + The pool scheme is ``household_id = tier * 10**8 + base``; a modulus of + ``10**8`` recovers the enhanced-FRS pool source as a *distinct lineage + layer*, leaving any immediate-layer ``source_household_id`` or + ``source_household_key`` untouched. + + Spine inputs use a different scheme: ``household_id = sernum + + 1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * 10**(d+2)``. + Their explicit spine lineage columns are the source of truth, so applying + the pool modulus would mint a false second lineage layer and is refused. The mapping is refused when it would be ambiguous (the pool column already exists) or vacuous (no id reaches the modulus, making it the @@ -915,6 +926,17 @@ def apply_uk_source_lineage_modulus( f"household already carries {POOL_SOURCE_LINEAGE_COLUMN!r}; " "applying source_lineage_modulus would be ambiguous." ) + spine_lineage_columns = [ + column for column in UK_SPINE_LINEAGE_COLUMNS if column in household.columns + ] + if spine_lineage_columns: + raise ValueError( + "household carries explicit spine lineage column(s) " + f"{spine_lineage_columns!r}; explicit spine lineage columns are " + "the source of truth. The source_lineage_modulus rule was written " + "for the pool's tier·10^8+base scheme and would mint a false " + "second lineage layer." + ) numeric = pd.to_numeric(household[household_id_column], errors="raise").to_numpy( dtype=np.float64 ) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 1144019452..d95de66b33 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -97,6 +97,17 @@ def _household_frame() -> pd.DataFrame: ) +def _spine_household_frame() -> pd.DataFrame: + return _household_frame().assign( + source_household_id=[10, 20, 10, 30], + household_support_channel=["frs", "spi", "frs", "spi"], + household_support_clone_index=[0, 1, 0, 0], + household_is_spi_synthetic=[False, True, False, True], + household_is_capital_gains_clone=[False, False, True, False], + household_is_cgt_band_donor=[False, False, False, True], + ) + + def _person_frame() -> pd.DataFrame: return pd.DataFrame( { @@ -348,14 +359,14 @@ def _load_builder_module(): return module -def _write_seam_h5(path) -> None: +def _write_seam_h5(path, *, household: pd.DataFrame | None = None) -> None: from microcosm.build.uk_runtime import write_uk_national_frame from microcosm.build.uk_runtime.national_frame import uk_national_frame dataset = uk_national_frame( person=_person_frame(), benunit=_benunit_frame(), - household=_household_frame(), + household=_household_frame() if household is None else household, time_period="2023", weight_kind=WeightKind.IMPORTANCE, mass_log=( @@ -410,6 +421,7 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) assert summary["assigned_constituencies"] >= 4 assert summary["weights"]["household_weight_kind"] == "importance" assert summary["weights"]["mass_conservation"]["passed"] is True + assert summary["source_lineage"]["explicit"] is None assert (output_dir / "staging_rowwise.h5").exists() @@ -462,6 +474,64 @@ def test_driver_ladder_dry_run_matches_real_assignment( plan["realized_support"]["constituency"]["n_areas"] >= manifest["rowwise_dataset"]["assigned_constituencies"] ) + assert plan["source_lineage"]["explicit"] is None + assert manifest["rowwise_dataset"]["source_lineage"]["explicit"] is None + + +def test_driver_ladder_spine_lineage_plan_manifest_parity( + monkeypatch, toy_ladder, tmp_path +): + pytest.importorskip("tables") + pytest.importorskip("h5py") + import json + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "spine.h5" + _write_seam_h5(input_h5, household=_spine_household_frame()) + plan_dir = tmp_path / "plan" + build_dir = tmp_path / "build" + base_argv = [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--n-clones", + "2", + "--seed", + "7", + ] + + monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(plan_dir), "--dry-run"]) + assert builder.main() == 0 + plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) + + monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) + assert builder.main() == 0 + manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) + + explicit = plan["source_lineage"]["explicit"] + assert explicit == manifest["rowwise_dataset"]["source_lineage"]["explicit"] + assert explicit == { + "basis": "explicit_lineage_columns", + "columns_present": [ + "source_household_id", + "household_support_channel", + "household_support_clone_index", + "household_is_spi_synthetic", + "household_is_capital_gains_clone", + "household_is_cgt_band_donor", + ], + "distinct_source_households": 3, + "distinct_by_support_channel": {"frs": 1, "spi": 2}, + "flag_counts": { + "household_is_spi_synthetic": 2, + "household_is_capital_gains_clone": 1, + "household_is_cgt_band_donor": 1, + }, + } def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path): diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index 9d1b2a2fbf..cc2583fc42 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -244,6 +244,7 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["source_lineage"]["pool_modulus"] is None assert plan["source_lineage"]["pool"] is None + assert plan["source_lineage"]["explicit"] is None def test_driver_dry_run_reports_weight_chain_and_pool_lineage( @@ -316,6 +317,7 @@ def test_driver_dry_run_reports_weight_chain_and_pool_lineage( assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 # The staging input's immediate layer is reported untouched alongside. assert lineage["immediate"]["distinct_source_households"] == 4 + assert lineage["explicit"] is None def test_driver_dry_run_supports_fixed_format_stores(monkeypatch, tmp_path) -> None: @@ -466,6 +468,7 @@ def test_driver_full_build_records_weight_chain_and_lineage( assert lineage["pool"]["pool_copies_per_source"]["min"] == 2 assert lineage["pool"]["pool_copies_per_source"]["max"] == 2 assert lineage["immediate"] is None + assert lineage["explicit"] is None assert manifest["base_dataset"]["distinct_source_households"] is None output_h5 = output_dir / "pool_rowwise.h5" diff --git a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py index b9f6ba84ce..5db2c3aefd 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_weight_metadata.py @@ -32,6 +32,7 @@ from microcosm.build.uk_runtime.national_frame import ( UK_HOUSEHOLD_WEIGHT_KIND_ATTR, ) +from microcosm.build.uk_runtime.rowwise_dataset import UK_SPINE_LINEAGE_COLUMNS from microcosm.frame import Frame, MassChangeRecord, WeightKind @@ -415,6 +416,18 @@ def test_source_lineage_modulus_rejects_bad_inputs() -> None: apply_uk_source_lineage_modulus(fractional, modulus=100) +@pytest.mark.parametrize("column", UK_SPINE_LINEAGE_COLUMNS) +def test_source_lineage_modulus_refuses_spine_lineage(column: str) -> None: + household = household_frame() + household[column] = 1 + + with pytest.raises( + ValueError, + match="explicit spine lineage columns are the source of truth", + ): + apply_uk_source_lineage_modulus(household, modulus=100) + + def test_clone_entity_frame_refuses_int64_overflow() -> None: frame = pd.DataFrame({"benunit_id": [np.iinfo(np.int64).max - 5]}) with pytest.raises(ValueError, match="overflow"): diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 6d47e28381..e860b1b14c 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -63,6 +63,7 @@ validate_geography_coverage, write_geography_crosswalk, ) +from microcosm.build.uk_runtime.rowwise_dataset import UK_SPINE_LINEAGE_COLUMNS from microcosm.frame import engine_tables CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" @@ -163,13 +164,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "--source-lineage-modulus", type=int, help=( - "Derive pool_source_household_id as household_id mod this value " - "before cloning, recovering pool-grain lineage as a distinct " - "layer (the certified UK pool encodes its 10x clone tiers with a " - "10**8 id offset; the staging H5's immediate-layer " - "source_household_id is left untouched). Refused when the pool " - "column already exists or the modulus would be an identity " - "mapping." + "Pool inputs only: derive pool_source_household_id before cloning " + "from household_id = tier * 10**8 + base, leaving the immediate " + "source_household_id untouched. Spine inputs instead use sernum + " + "1{spi} * 10**d + 1{cgt_clone} * 10**(d+1) + 1{band_donor} * " + "10**(d+2) and carry authoritative explicit lineage columns, so " + "the modulus is refused for them. Also refused when the pool " + "column exists or the mapping would be an identity." ), ) parser.add_argument( @@ -1278,13 +1279,45 @@ def _pool_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: return block +def _explicit_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: + spine_columns = [ + column for column in UK_SPINE_LINEAGE_COLUMNS if column in household.columns + ] + if not spine_columns: + return None + + columns_present = [ + column + for column in ("source_household_id", *UK_SPINE_LINEAGE_COLUMNS) + if column in household.columns + ] + block: dict[str, Any] = { + "basis": "explicit_lineage_columns", + "columns_present": columns_present, + "flag_counts": { + column: int(household[column].fillna(False).astype(bool).sum()) + for column in UK_SPINE_LINEAGE_COLUMNS + if column.startswith("household_is_") and column in household.columns + }, + } + if "source_household_id" in household.columns: + block["distinct_source_households"] = int( + household["source_household_id"].nunique() + ) + if "household_support_channel" in household.columns: + block["distinct_by_support_channel"] = { + str(channel): int(group["source_household_id"].nunique()) + for channel, group in household.groupby("household_support_channel") + } + return block + + def _source_lineage_report( household: pd.DataFrame, *, modulus: int | None, ) -> dict[str, Any]: - """Report both lineage layers honestly: the derived pool layer (when a - modulus was applied) and any immediate layer the input already carries.""" + """Report pool, immediate, and explicit spine lineage when supported.""" pool = _pool_lineage_block(household) immediate = None @@ -1298,6 +1331,7 @@ def _source_lineage_report( "pool_modulus": modulus, "pool": pool, "immediate": immediate, + "explicit": _explicit_lineage_block(household), } @@ -1368,6 +1402,7 @@ def _rowwise_summary( if base_summary.get("distinct_source_households") is not None else None ), + "explicit": _explicit_lineage_block(clone0), } return { "weights": { From 3a9ef625b7b77b24ab542b3a6d8dfa3c032a94d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:06:59 +0200 Subject: [PATCH 03/14] Refuse pre-assigned geography columns at the ladder clone seam assign_uk_geography_ladder silently overwrites oa/lsoa/msoa codes and would leave stale *_oa columns riding through beside its own outputs. The ladder clone path and its dry-run plan now refuse a household input that already carries any ladder geography column or legacy *_oa name (region, the required enum input, excepted) -- two disagreeing constituency columns on one artifact is exactly the laundering the fences exist to prevent. Co-Authored-By: Claude Fable 5 --- .../build/uk_runtime/rowwise_dataset.py | 24 +++++++++ .../tests/test_uk_ladder_rowwise_clone.py | 49 +++++++++++++++++++ tools/build_uk_rowwise_dataset.py | 6 ++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py index f98d3aceb5..86f67c7026 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_dataset.py @@ -87,6 +87,29 @@ def ladder_clone_index_column(entity: str) -> str: return f"{entity}_{ARTIFACT_CLONE_INDEX_COLUMN}" +def _refuse_preassigned_geography( + table: pd.DataFrame, + *, + label: str, +) -> None: + preassigned_columns = [ + column + for column in ( + *UK_GEOGRAPHY_LADDER_COLUMNS, + "la_code_oa", + "constituency_code_oa", + "region_code_oa", + ) + if column in table.columns + ] + if preassigned_columns: + raise ValueError( + f"{label} carries pre-assigned geography column(s) " + f"{preassigned_columns!r}; pre-assigned geography cannot be " + "silently overwritten and stale *_oa columns cannot ride through." + ) + + @dataclass(frozen=True) class UKRowwiseDatasetResult: """Cloned UK single-year tables and row-wise geography metadata. @@ -307,6 +330,7 @@ def clone_uk_dataset_tables_with_ladder_geography( household_id_column="household_id", household_weight_column="household_weight", ) + _refuse_preassigned_geography(household_frame, label="household") # The artifact's clone_index names the rowwise clone dimension. A # candidate-tier clone_index inherited from the staging input (SPI/pool # lineage) is replaced, exactly as the pre-Frame clone overwrote the diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index d95de66b33..7edcb85db7 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -261,6 +261,19 @@ def test_ladder_clone_refuses_vintage_mismatch(toy_ladder) -> None: ) +def test_ladder_clone_refuses_preassigned_geography(toy_ladder) -> None: + ladder, _ = toy_ladder + preassigned = _seam_frame( + household=_household_frame().assign(oa_code="stale-oa"), + ) + + with pytest.raises( + ValueError, + match="pre-assigned geography cannot be silently overwritten", + ): + clone_uk_dataset_with_ladder_geography(preassigned, ladder, n_clones=1) + + def test_ladder_clone_refuses_uncovered_region(tmp_path) -> None: frame = _ladder_frame() england_only = frame[frame["region_code"].str.startswith("E")] @@ -561,6 +574,42 @@ def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path builder.main() +def test_driver_ladder_dry_run_refuses_legacy_preassigned_geography( + monkeypatch, toy_ladder, tmp_path +): + pytest.importorskip("tables") + pytest.importorskip("h5py") + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "preassigned.h5" + household = _household_frame().assign( + constituency_code_oa="stale-constituency" + ) + _write_seam_h5(input_h5, household=household) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--out", + str(tmp_path / "plan"), + "--dry-run", + ], + ) + + with pytest.raises( + ValueError, + match=r"stale \*_oa columns cannot ride through", + ): + builder.main() + + def test_ladder_clone_pins_per_copy_weights_and_fk_alignment(toy_ladder) -> None: ladder, _ = toy_ladder result = clone_uk_dataset_with_ladder_geography(_seam_frame(), ladder, n_clones=2) diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index e860b1b14c..ce7e5a3ffd 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -63,7 +63,10 @@ validate_geography_coverage, write_geography_crosswalk, ) -from microcosm.build.uk_runtime.rowwise_dataset import UK_SPINE_LINEAGE_COLUMNS +from microcosm.build.uk_runtime.rowwise_dataset import ( + UK_SPINE_LINEAGE_COLUMNS, + _refuse_preassigned_geography, +) from microcosm.frame import engine_tables CROSSWALK_FILENAME = "uk_official_geography_crosswalk.csv.gz" @@ -1005,6 +1008,7 @@ def _ladder_dry_run_plan( person_ids=person_ids, benunit_ids=benunit_ids, ) + _refuse_preassigned_geography(household, label="household") id_multiplier = id_multiplier_for_values( household["household_id"], person_ids["person_id"], From 3e784a5c12cdf8e9b1a53ddbfef6fca00747cabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:14:32 +0200 Subject: [PATCH 04/14] Pin the rowwise inputs fail-closed: --input-sha256 and --ladder-sha256 The pilot's contract is a sha-pinned spine and a sha-pinned ladder. Both pins follow the calibration seam's convention (measured-vs-pinned refusal before any bytes are parsed or output side effect happens), and each artifact block records pin_verified. The ladder block additionally cross-checks the measured sha against the committed local_area_crosswalk.json ladder_artifact_sha256 pin -- informational, never a refusal, so synthetic test ladders keep working while a real run records whether it matched the ratified artifact. Co-Authored-By: Claude Fable 5 --- .../tests/test_uk_ladder_rowwise_clone.py | 80 +++++++++++++++++++ .../tests/test_uk_rowwise_build_driver.py | 40 ++++++++++ .../tests/test_uk_rowwise_dry_run.py | 1 + tools/build_uk_rowwise_dataset.py | 57 ++++++++++++- 4 files changed, 176 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 7edcb85db7..093f860f96 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -428,6 +428,9 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) manifest = json.loads((output_dir / builder.MANIFEST_FILENAME).read_text()) assert manifest["parameters"]["assignment_route"] == "ladder" assert manifest["inputs"]["ladder"]["sha256"] + assert manifest["inputs"]["dataset"]["pin_verified"] is False + assert manifest["inputs"]["ladder"]["pin_verified"] is False + assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False summary = manifest["rowwise_dataset"] assert summary["gate"]["passed"] is True assert summary["missing_geography_rows"] == 0 @@ -452,13 +455,19 @@ def test_driver_ladder_dry_run_matches_real_assignment( _write_seam_h5(input_h5) plan_dir = tmp_path / "plan" build_dir = tmp_path / "build" + input_sha256 = builder._sha256(input_h5) + ladder_sha256 = builder._sha256(ladder_path) base_argv = [ "build_uk_rowwise_dataset.py", "--input-h5", str(input_h5), + "--input-sha256", + input_sha256, "--ladder", str(ladder_path), + "--ladder-sha256", + ladder_sha256, "--n-clones", "2", "--seed", @@ -472,6 +481,12 @@ def test_driver_ladder_dry_run_matches_real_assignment( monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) assert builder.main() == 0 manifest = json.loads((build_dir / builder.MANIFEST_FILENAME).read_text()) + assert plan["input"]["dataset"]["pin_verified"] is True + assert plan["input"]["ladder"]["pin_verified"] is True + assert plan["input"]["ladder"]["matches_local_area_crosswalk_pin"] is False + assert manifest["inputs"]["dataset"]["pin_verified"] is True + assert manifest["inputs"]["ladder"]["pin_verified"] is True + assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False # The dry-run's realized support is exact: identical draws to the build. realized = { @@ -574,6 +589,71 @@ def test_driver_ladder_refuses_crosswalk_combo(monkeypatch, toy_ladder, tmp_path builder.main() +def test_driver_ladder_sha256_refuses_crosswalk(monkeypatch, tmp_path) -> None: + pytest.importorskip("tables") + import sys + + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--crosswalk", + str(tmp_path / "crosswalk.csv"), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(ValueError, match="ladder-sha256.*crosswalk"): + builder.main() + + +def test_driver_ladder_pin_mismatch_refuses_before_parse( + monkeypatch, toy_ladder, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + + def unexpected_parse(_path): + raise AssertionError("ladder pin mismatch must refuse before NPZ parsing") + + monkeypatch.setattr(builder, "load_uk_oa_ladder", unexpected_parse) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(SystemExit, match=r"--ladder sha mismatch: measured"): + builder.main() + + def test_driver_ladder_dry_run_refuses_legacy_preassigned_geography( monkeypatch, toy_ladder, tmp_path ): diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index 3216ebb273..b389da04d8 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -132,6 +132,42 @@ def _crosswalk_frame() -> pd.DataFrame: ) +@pytest.mark.parametrize("route_option", ["--crosswalk", "--ladder"]) +def test_input_pin_mismatch_refuses_before_side_effects( + monkeypatch, + tmp_path, + route_option, +) -> None: + pytest.importorskip("tables") + builder = _load_builder_module() + input_h5 = tmp_path / "input.h5" + route_artifact = tmp_path / "route-artifact" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5) + route_artifact.write_bytes(b"must not be read") + + def unexpected_h5_read(_path): + raise AssertionError("input pin mismatch must refuse before H5 parsing") + + monkeypatch.setattr(builder, "_h5_summary", unexpected_h5_read) + + with pytest.raises(SystemExit, match=r"--input-h5 sha mismatch: measured"): + builder.main( + [ + "--input-h5", + str(input_h5), + "--input-sha256", + "0" * 64, + "--out", + str(output_dir), + route_option, + str(route_artifact), + ] + ) + + assert not output_dir.exists() + + def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( monkeypatch, tmp_path, capsys ): @@ -143,6 +179,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( la_codes = tmp_path / "local_authorities.csv" output_dir = tmp_path / "out" _write_toy_h5(input_h5) + input_sha256 = builder._sha256(input_h5) _crosswalk_frame().to_csv(crosswalk_path, index=False) pd.DataFrame({"code": ["E14000001", "W07000041", "S14000001", "N05000001"]}).to_csv( constituency_codes, index=False @@ -157,6 +194,8 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( "build_uk_rowwise_dataset.py", "--input-h5", str(input_h5), + "--input-sha256", + input_sha256, "--out", str(output_dir), "--crosswalk", @@ -187,6 +226,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( assert manifest["parameters"]["n_clones"] == 2 assert manifest["parameters"]["source_year"] == 2023 assert manifest["parameters"]["require_all_countries"] is False + assert manifest["inputs"]["dataset"]["pin_verified"] is True assert manifest["base_dataset"]["household_weight_sum"] == pytest.approx(30.0) assert manifest["rowwise_dataset"]["household_weight_sum"] == pytest.approx(30.0) assert manifest["rowwise_dataset"]["household_weight_delta"] == pytest.approx(0.0) diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index cc2583fc42..153cd8ca43 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -214,6 +214,7 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["build_kind"] == "uk_rowwise_local_geography_dry_run" assert plan["parameters"]["n_clones"] == 2 assert plan["input"]["dataset"]["sha256"] + assert plan["input"]["dataset"]["pin_verified"] is False assert plan["input"]["household_weight_kind"] == WeightKind.DESIGN.value assert plan["input"]["mass_log_records"] == 0 assert plan["plan"]["rows"] == { diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index ce7e5a3ffd..6864a844cc 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -55,6 +55,7 @@ geography_coverage_summary, id_multiplier_for_values, ladder_clone_index_column, + load_uk_local_area_crosswalk, load_uk_oa_ladder, read_uk_single_year_weight_metadata, uk_geography_ladder_gate, @@ -87,6 +88,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: required=True, help="Compact Microcosm UK single-year H5 to clone.", ) + parser.add_argument( + "--input-sha256", + type=sha256_argument, + help="Optional SHA-256 pin for --input-h5; mismatches fail before H5 parsing.", + ) parser.add_argument( "--out", type=Path, @@ -111,6 +117,11 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: "Mutually exclusive with --crosswalk and the coverage-code checks." ), ) + parser.add_argument( + "--ladder-sha256", + type=sha256_argument, + help="Optional SHA-256 pin for --ladder; mismatches fail before NPZ parsing.", + ) parser.add_argument( "--expected-constituency-vintage", default="2024_pcon", @@ -352,10 +363,14 @@ def _main_impl( *, attempt: dict[str, object] | None, ) -> int: - args.out.mkdir(parents=True, exist_ok=True) - input_h5 = args.input_h5.resolve() input_artifact = _artifact_info(input_h5) + _verify_artifact_pin( + input_artifact, + pinned_sha256=args.input_sha256, + option="--input-h5", + ) + args.out.mkdir(parents=True, exist_ok=True) base_summary = _h5_summary(input_h5) source_year = _source_year(args.source_year, base_summary=base_summary) output_h5 = _dataset_output_path( @@ -365,6 +380,8 @@ def _main_impl( source_year=source_year, ) _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) + if args.crosswalk is not None and args.ladder_sha256 is not None: + raise ValueError("--ladder-sha256 and --crosswalk are mutually exclusive.") if args.ladder is not None: if args.crosswalk is not None: raise ValueError("--ladder and --crosswalk are mutually exclusive.") @@ -865,6 +882,12 @@ def _run_ladder_route( ladder_path = args.ladder.resolve() ladder_artifact = _artifact_info(ladder_path) + _verify_artifact_pin( + ladder_artifact, + pinned_sha256=args.ladder_sha256, + option="--ladder", + ) + _annotate_ladder_crosswalk_pin(ladder_artifact) ladder = load_uk_oa_ladder(ladder_path) if args.dry_run: @@ -1478,6 +1501,36 @@ def _artifact_info(path: Path) -> dict[str, Any]: } +def _verify_artifact_pin( + artifact: dict[str, Any], + *, + pinned_sha256: str | None, + option: str, +) -> None: + measured_sha256 = str(artifact["sha256"]) + if pinned_sha256 is not None and measured_sha256 != pinned_sha256: + raise SystemExit( + f"error: {option} sha mismatch: " + f"measured {measured_sha256}, pinned {pinned_sha256}" + ) + artifact["pin_verified"] = pinned_sha256 is not None + + +def _annotate_ladder_crosswalk_pin(ladder_artifact: dict[str, Any]) -> None: + try: + crosswalk = load_uk_local_area_crosswalk() + expected_sha256 = str(crosswalk["ladder_artifact_sha256"]) + except Exception as error: + ladder_artifact["matches_local_area_crosswalk_pin"] = None + ladder_artifact["local_area_crosswalk_pin_error"] = ( + f"{type(error).__name__}: {error}" + ) + return + ladder_artifact["matches_local_area_crosswalk_pin"] = ( + str(ladder_artifact["sha256"]) == expected_sha256 + ) + + def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: From dda3702b432b29d27460af5551b3bc23021131e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:25:21 +0200 Subject: [PATCH 05/14] Add the analytic expected-support surface for the ladder route expected_uk_ladder_area_support computes exact expected rows per constituency (household-count share within region x input region mix x n_clones) and per local authority (OA population shares within constituency), sharing the sampler's own stage-weight helpers so the expectation cannot drift from the draw. The ladder dry-run plan gains an expected_support block beside realized_support, zero-support areas included. The stage-two zero-mass refusal now names the constituency instead of surfacing as an rng.choice probability error. Co-Authored-By: Claude Fable 5 --- .../microcosm/build/uk_runtime/__init__.py | 2 + .../build/uk_runtime/geography_ladder.py | 210 +++++++++++++++--- .../tests/test_uk_geography_ladder.py | 92 ++++++++ .../tests/test_uk_ladder_rowwise_clone.py | 13 ++ tools/build_uk_rowwise_dataset.py | 17 ++ 5 files changed, 302 insertions(+), 32 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 4d0ba469e7..22fd3d36a9 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -144,6 +144,7 @@ UK_OA_LADDER_SCHEMA_VERSION, UkOaLadder, assign_uk_geography_ladder, + expected_uk_ladder_area_support, load_uk_oa_ladder, uk_geography_ladder_assignment_summary, uk_geography_ladder_gate, @@ -723,6 +724,7 @@ "build_uk_rowwise_local_matrix", "clone_entity_frame", "clone_uk_dataset_tables_with_ladder_geography", + "expected_uk_ladder_area_support", "clone_uk_dataset_tables_with_rowwise_geography", "clone_uk_dataset_with_ladder_geography", "clone_uk_dataset_with_rowwise_geography", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index 024d6b4eaf..0515812f1d 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -365,18 +365,11 @@ def assign_uk_geography_ladder( f"({expected_constituency_vintage!r})." ) - region_codes = _household_region_codes( - household[region_column], label=region_column + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, ) - ladder_regions = set(np.unique(ladder.region_code).tolist()) - missing_regions = sorted(set(region_codes.tolist()) - ladder_regions) - if missing_regions: - raise ValueError( - "UK OA ladder has no output areas for household region(s): " - f"{missing_regions}. The household regions and the ladder artifact " - "must share coverage (Scotland and Northern Ireland rungs are not " - "in an England-&-Wales ladder)." - ) assigned_index = _sample_oa_indices( region_codes.to_numpy(), @@ -419,6 +412,89 @@ def assign_uk_geography_ladder( return assigned +def expected_uk_ladder_area_support( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + n_clones: int = 1, + region_column: str = "region", +) -> pd.DataFrame: + """Return analytic expected rows per constituency and local authority. + + The expectation follows the assignment sampler's two stages exactly: + constituency household-count shares within each input region, followed by + OA population shares within each chosen constituency. Every constituency + and local authority in the ladder is returned, including zero-support + areas. + """ + + if isinstance(n_clones, bool) or not isinstance(n_clones, int) or n_clones <= 0: + raise ValueError("n_clones must be a positive integer.") + + region_codes = _validated_household_ladder_region_codes( + household, + ladder, + region_column=region_column, + ) + constituency_expected = { + str(code): 0.0 for code in np.unique(ladder.constituency_code).tolist() + } + local_authority_expected = { + str(code): 0.0 for code in np.unique(ladder.local_authority_code).tolist() + } + frame = _ladder_sampling_frame(ladder) + region_counts = region_codes.value_counts(sort=False).sort_index() + + for region_code, n_region in region_counts.items(): + region_rows, constituency_weight = _constituency_household_weights( + frame, + region_code=str(region_code), + ) + region_households = float(constituency_weight.sum()) + for constituency_code, household_count in constituency_weight.items(): + expected_rows = ( + n_clones + * int(n_region) + * float(household_count) + / region_households + ) + constituency_key = str(constituency_code) + constituency_expected[constituency_key] += expected_rows + if expected_rows == 0.0: + continue + + oa_rows, oa_weights = _constituency_oa_population_weights( + region_rows, + region_code=str(region_code), + constituency_code=constituency_key, + ) + constituency_population = float(oa_weights.sum()) + local_authority_population = oa_rows.groupby( + "local_authority_code", sort=True + )["population"].sum() + for local_authority_code, population in local_authority_population.items(): + local_authority_expected[str(local_authority_code)] += ( + expected_rows * float(population) / constituency_population + ) + + rows = [ + { + "area_type": area_type, + "area_code": area_code, + "expected_rows": expected_rows, + } + for area_type, expected_by_area in ( + ("constituency", constituency_expected), + ("la", local_authority_expected), + ) + for area_code, expected_rows in expected_by_area.items() + ] + return pd.DataFrame( + rows, + columns=("area_type", "area_code", "expected_rows"), + ) + + def uk_geography_ladder_assignment_summary( household: pd.DataFrame, ladder: UkOaLadder, @@ -659,32 +735,18 @@ def _sample_oa_indices( n = len(region_codes) assigned_index = np.full(n, -1, dtype=np.int64) - frame = pd.DataFrame( - { - "constituency_code": ladder.constituency_code, - "region_code": ladder.region_code, - "population": ladder.population, - "households": ladder.households, - "ladder_index": np.arange(len(ladder), dtype=np.int64), - } - ) + frame = _ladder_sampling_frame(ladder) household_positions = pd.Series(np.arange(n, dtype=np.int64)) for region_code, region_positions in household_positions.groupby( pd.Series(region_codes), sort=True ): - region_rows = frame[frame["region_code"] == region_code] - # Stage one: constituency household-count weights within the region. - constituency_weight = region_rows.groupby("constituency_code", sort=True)[ - "households" - ].sum() + region_rows, constituency_weight = _constituency_household_weights( + frame, + region_code=str(region_code), + ) constituencies = constituency_weight.index.to_numpy() weights = constituency_weight.to_numpy(dtype=np.float64) - if weights.sum() <= 0: - raise ValueError( - f"region {region_code!r} has zero household weight for the " - "constituency draw." - ) positions = region_positions.to_numpy() chosen_constituency = rng.choice( constituencies, @@ -697,9 +759,12 @@ def _sample_oa_indices( for constituency_code, local_positions in pd.Series(positions).groupby( chosen_series, sort=True ): - oa_rows = region_rows[region_rows["constituency_code"] == constituency_code] + oa_rows, oa_weights = _constituency_oa_population_weights( + region_rows, + region_code=str(region_code), + constituency_code=str(constituency_code), + ) oa_indices = oa_rows["ladder_index"].to_numpy() - oa_weights = oa_rows["population"].to_numpy(dtype=np.float64) drawn = rng.choice( oa_indices, size=len(local_positions), @@ -713,6 +778,58 @@ def _sample_oa_indices( return assigned_index +def _ladder_sampling_frame(ladder: UkOaLadder) -> pd.DataFrame: + """Aligned ladder columns used by both sampling and its expectation.""" + + return pd.DataFrame( + { + "constituency_code": ladder.constituency_code, + "region_code": ladder.region_code, + "local_authority_code": ladder.local_authority_code, + "population": ladder.population, + "households": ladder.households, + "ladder_index": np.arange(len(ladder), dtype=np.int64), + } + ) + + +def _constituency_household_weights( + frame: pd.DataFrame, + *, + region_code: str, +) -> tuple[pd.DataFrame, pd.Series]: + """Stage-one constituency weights for one household region.""" + + region_rows = frame[frame["region_code"] == region_code] + constituency_weight = region_rows.groupby("constituency_code", sort=True)[ + "households" + ].sum() + if constituency_weight.sum() <= 0: + raise ValueError( + f"region {region_code!r} has zero household weight for the " + "constituency draw." + ) + return region_rows, constituency_weight + + +def _constituency_oa_population_weights( + region_rows: pd.DataFrame, + *, + region_code: str, + constituency_code: str, +) -> tuple[pd.DataFrame, np.ndarray]: + """Stage-two OA weights for one region-constituency pair.""" + + oa_rows = region_rows[region_rows["constituency_code"] == constituency_code] + oa_weights = oa_rows["population"].to_numpy(dtype=np.float64) + if oa_weights.sum() <= 0: + raise ValueError( + f"constituency {constituency_code!r} in region {region_code!r} has " + "zero population weight for the OA draw." + ) + return oa_rows, oa_weights + + def _metadata_from_scalar(value: np.ndarray) -> dict[str, Any]: if value.shape != (): raise ValueError("metadata_json must be a scalar JSON string.") @@ -841,6 +958,34 @@ def _household_region_codes(values: Any, *, label: str) -> pd.Series: return mapped +def _validated_household_ladder_region_codes( + household: pd.DataFrame, + ladder: UkOaLadder, + *, + region_column: str, +) -> pd.Series: + """Normalize household regions and require matching ladder coverage.""" + + if region_column not in household.columns: + raise ValueError( + f"household table must contain {region_column!r} before geography-" + "ladder assignment (assign regions first)." + ) + region_codes = _household_region_codes( + household[region_column], label=region_column + ) + ladder_regions = set(np.unique(ladder.region_code).tolist()) + missing_regions = sorted(set(region_codes.tolist()) - ladder_regions) + if missing_regions: + raise ValueError( + "UK OA ladder has no output areas for household region(s): " + f"{missing_regions}. The household regions and the ladder artifact " + "must share coverage (Scotland and Northern Ireland rungs are not " + "in an England-&-Wales ladder)." + ) + return region_codes + + def _region_code_for_value(value: Any) -> str | None: text = str(value).strip() if not text: @@ -874,6 +1019,7 @@ def _region_code_for_value(value: Any) -> str | None: "UK_OA_LADDER_SCHEMA_VERSION", "UkOaLadder", "assign_uk_geography_ladder", + "expected_uk_ladder_area_support", "load_uk_oa_ladder", "uk_geography_ladder_assignment_summary", "uk_geography_ladder_gate", diff --git a/packages/microcosm-build/tests/test_uk_geography_ladder.py b/packages/microcosm-build/tests/test_uk_geography_ladder.py index 759d5cfb0c..586285cab7 100644 --- a/packages/microcosm-build/tests/test_uk_geography_ladder.py +++ b/packages/microcosm-build/tests/test_uk_geography_ladder.py @@ -21,6 +21,7 @@ UK_GEOGRAPHY_LADDER_COLUMNS, assemble_uk_oa_ladder, assign_uk_geography_ladder, + expected_uk_ladder_area_support, join_uk_oa_ladder_layers, load_uk_oa_ladder, uk_geography_ladder_assignment_summary, @@ -269,6 +270,97 @@ def test_assignment_is_deterministic_and_two_stage_weighted(tmp_path) -> None: assert dominant > 0.95 +def test_expected_support_matches_hand_computed_two_stage_result(tmp_path) -> None: + ladder = load_uk_oa_ladder( + _write_ladder( + tmp_path / "ladder.npz", + local_authority_code=np.asarray( + ["E09000001", "E09000002", "E09000002", "W06000001"] + ), + ) + ) + household = pd.DataFrame( + {"household_id": [1, 2], "region": ["LONDON", "London"]} + ) + + support = expected_uk_ladder_area_support(household, ladder, n_clones=2) + + assert support.columns.tolist() == ["area_type", "area_code", "expected_rows"] + assert set(support["area_type"]) == {"constituency", "la"} + expected = { + (row.area_type, row.area_code): row.expected_rows + for row in support.itertuples(index=False) + } + assert expected == pytest.approx( + { + ("constituency", "E14000001"): 1.0, + ("constituency", "E14000002"): 3.0, + ("constituency", "W07000041"): 0.0, + ("la", "E09000001"): 0.99, + ("la", "E09000002"): 3.01, + ("la", "W06000001"): 0.0, + } + ) + + +@pytest.mark.parametrize("n_clones", [0, -1, True, 1.5, "2"]) +def test_expected_support_requires_positive_integer_n_clones( + tmp_path, n_clones +) -> None: + ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) + + with pytest.raises(ValueError, match="n_clones must be a positive integer"): + expected_uk_ladder_area_support( + _household(), + ladder, + n_clones=n_clones, + ) + + +def test_expected_support_matches_assignment_uncovered_region_refusal(tmp_path) -> None: + ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) + household = pd.DataFrame({"household_id": [1], "region": ["SCOTLAND"]}) + + with pytest.raises(ValueError) as assignment_error: + assign_uk_geography_ladder(household, ladder) + with pytest.raises(ValueError) as expectation_error: + expected_uk_ladder_area_support(household, ladder) + + assert str(expectation_error.value) == str(assignment_error.value) + + +def test_expected_support_agrees_with_large_realized_assignment(tmp_path) -> None: + ladder = load_uk_oa_ladder( + _write_ladder( + tmp_path / "ladder.npz", + population=np.asarray([300.0, 100.0, 300.0, 400.0]), + local_authority_code=np.asarray( + ["E09000001", "E09000002", "E09000002", "W06000001"] + ), + ) + ) + household = pd.DataFrame( + {"household_id": range(20_000), "region": ["LONDON"] * 20_000} + ) + + expected = expected_uk_ladder_area_support(household, ladder) + assigned = assign_uk_geography_ladder(household, ladder, seed=91) + actual = { + (area_type, code): float(count) + for area_type, column in ( + ("constituency", "constituency_code"), + ("la", "local_authority_code"), + ) + for code, count in assigned[column].value_counts().items() + } + + for row in expected[expected["expected_rows"] > 0].itertuples(index=False): + assert actual[(row.area_type, row.area_code)] == pytest.approx( + row.expected_rows, + rel=0.03, + ) + + def test_assignment_requires_region(tmp_path) -> None: ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) household = pd.DataFrame({"household_id": [1]}) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 093f860f96..0bdc9b171e 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -477,6 +477,19 @@ def test_driver_ladder_dry_run_matches_real_assignment( assert builder.main() == 0 plan = json.loads((plan_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) assert not (plan_dir / "staging_rowwise.h5").exists() + expected_support = plan["expected_support"] + assert expected_support["basis"] == ( + "analytic expectation: constituency household-count share within region x " + "the input's region mix x n_clones; OA population shares within " + "constituency for LA support" + ) + for area_type in ("constituency", "la"): + area_support = expected_support[area_type] + assert area_support["n_areas"] <= builder.EXPECTED_SUPPORT_BOTTOM_AREAS + assert len(area_support["bottom"]) == area_support["n_areas"] + assert sum(row["rows"] for row in area_support["bottom"]) == pytest.approx( + plan["plan"]["rows"]["household"] + ) monkeypatch.setattr(sys, "argv", [*base_argv, "--out", str(build_dir)]) assert builder.main() == 0 diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 6864a844cc..287b4b82a0 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -51,6 +51,7 @@ clone_entity_frame, clone_uk_dataset_with_ladder_geography, clone_uk_dataset_with_rowwise_geography, + expected_uk_ladder_area_support, expected_uk_rowwise_area_support, geography_coverage_summary, id_multiplier_for_values, @@ -1085,6 +1086,11 @@ def _ladder_dry_run_plan( + "; ".join(gate.failures) ) realized = _ladder_realized_support(assigned, ladder) + expected = expected_uk_ladder_area_support( + household, + ladder, + n_clones=args.n_clones, + ) table_rows = { name: base_summary["tables"][name][0] for name in ("person", "benunit", "household") @@ -1128,6 +1134,17 @@ def _ladder_dry_run_plan( for area_type in ("constituency", "la") }, }, + "expected_support": { + "basis": ( + "analytic expectation: constituency household-count share " + "within region x the input's region mix x n_clones; OA " + "population shares within constituency for LA support" + ), + **{ + area_type: _support_summary(expected, area_type) + for area_type in ("constituency", "la") + }, + }, "source_lineage": _source_lineage_report( household, modulus=args.source_lineage_modulus, From 7d2f42299ac90d3b14da7ff916059505c6d8f290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:43:58 +0200 Subject: [PATCH 06/14] Lift the per-area support diagnostics off the solve matrix uk_area_support_summary is the frame-agnostic core (one groupby instead of the per-area scan, numerically identical); the matrix-coupled rowwise_area_support_summary becomes a thin wrapper with its signature untouched. The new uk_ladder_area_support_summary computes the same per-area ESS / distinct-source table for a plain ladder-assigned frame on the full ladder roster, and uk_region_mix reports the input's region composition as shares. The ladder route's manifest and dry-run plan gain area_support (min/median rows, ESS, distinct sources, bottom-15 by rows and by ESS, with the source_basis recorded) and region_mix blocks, and the real build writes the full table as area_support_summary.csv. Co-Authored-By: Claude Fable 5 --- .../microcosm/build/uk_runtime/__init__.py | 6 + .../build/uk_runtime/geography_ladder.py | 40 ++++ .../build/uk_runtime/local_rowwise.py | 174 ++++++++++++++---- .../tests/test_uk_geography_ladder.py | 25 +++ .../tests/test_uk_ladder_rowwise_clone.py | 53 +++++- .../tests/test_uk_local_rowwise.py | 75 ++++++++ .../tests/test_uk_rowwise_build_driver.py | 29 ++- tools/build_uk_rowwise_dataset.py | 106 ++++++++++- 8 files changed, 470 insertions(+), 38 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py index 22fd3d36a9..a43624a88a 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/__init__.py @@ -148,6 +148,7 @@ load_uk_oa_ladder, uk_geography_ladder_assignment_summary, uk_geography_ladder_gate, + uk_region_mix, ) from microcosm.build.uk_runtime.geography_sources import ( ENGLAND_LAD_REGION_URL, @@ -276,6 +277,8 @@ rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, + uk_ladder_area_support_summary, ) from microcosm.build.uk_runtime.local_target_census import ( CENSUS_KIND, @@ -819,6 +822,9 @@ "uk_legacy_jsa_policy", "uk_geography_ladder_assignment_summary", "uk_geography_ladder_gate", + "uk_region_mix", + "uk_area_support_summary", + "uk_ladder_area_support_summary", "uk_release_input_coverage_gate", "uk_release_input_coverage_required_columns", "uk_release_input_coverage_reviewed_exclusions", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py index 0515812f1d..b5015c4b17 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/geography_ladder.py @@ -495,6 +495,45 @@ def expected_uk_ladder_area_support( ) +def uk_region_mix( + household: pd.DataFrame, + *, + region_column: str = "region", + weight_column: str = "household_weight", +) -> pd.DataFrame: + """Summarize household row and weight shares by normalized UK region.""" + + if region_column not in household.columns: + raise ValueError(f"household table must contain region column {region_column!r}.") + if weight_column not in household.columns: + raise ValueError(f"household table must contain weight column {weight_column!r}.") + + region_codes = _household_region_codes( + household[region_column], + label=region_column, + ) + weights = pd.to_numeric(household[weight_column], errors="raise").to_numpy( + dtype=np.float64 + ) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise ValueError(f"{weight_column} must be finite and non-negative.") + if len(weights) == 0: + raise ValueError("household table must contain at least one row for region mix.") + total_weight = float(weights.sum()) + if total_weight <= 0: + raise ValueError(f"{weight_column} must carry positive total weight.") + + frame = pd.DataFrame({"region_code": region_codes, "weight": weights}) + mix = ( + frame.groupby("region_code", sort=True) + .agg(rows=("weight", "size"), weight_sum=("weight", "sum")) + .reset_index() + ) + mix["row_share"] = mix["rows"] / len(frame) + mix["weight_share"] = mix["weight_sum"] / total_weight + return mix[["region_code", "rows", "row_share", "weight_share"]] + + def uk_geography_ladder_assignment_summary( household: pd.DataFrame, ladder: UkOaLadder, @@ -1021,6 +1060,7 @@ def _region_code_for_value(value: Any) -> str | None: "assign_uk_geography_ladder", "expected_uk_ladder_area_support", "load_uk_oa_ladder", + "uk_region_mix", "uk_geography_ladder_assignment_summary", "uk_geography_ladder_gate", ] diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py index c25e285243..fa0b5a52e5 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/local_rowwise.py @@ -62,6 +62,8 @@ "build_uk_rowwise_local_matrix", "past_cap_census", "require_adjudicated_uk_local_binding", + "uk_area_support_summary", + "uk_ladder_area_support_summary", "rowwise_calibration_mass_reason", "rowwise_area_support_summary", "solve_uk_rowwise_weights_under_doctrine", @@ -940,50 +942,154 @@ def rowwise_area_support_summary( ) -> pd.DataFrame: """Per-area support of a rowwise weight vector, all target areas included.""" + return uk_area_support_summary( + problem.assigned_areas, + weights, + area_codes=problem.area_codes, + source_household_ids=( + problem.household_ids + if source_household_ids is None + else source_household_ids + ), + ) + + +def uk_area_support_summary( + assigned_areas: Sequence[Any], + weights: Sequence[float], + *, + area_codes: Sequence[Any], + source_household_ids: Sequence[Any], +) -> pd.DataFrame: + """Summarize row and weighted support for a declared roster of areas.""" + values = np.asarray(weights, dtype=np.float64) if values.ndim != 1: raise ValueError(f"weights must be one-dimensional, got shape {values.shape}.") - if len(values) != problem.n_households: + assigned = np.asarray(assigned_areas, dtype=object) + if assigned.ndim != 1: + raise ValueError( + f"assigned_areas must be one-dimensional, got shape {assigned.shape}." + ) + if len(values) != len(assigned): raise ValueError( "weights must align with households, got " - f"{len(values)} weights for {problem.n_households} households." + f"{len(values)} weights for {len(assigned)} households." ) if not np.isfinite(values).all() or (values < 0).any(): raise ValueError("weights must be finite and non-negative.") - sources_list = ( - list(problem.household_ids) - if source_household_ids is None - else list(source_household_ids) - ) - if len(sources_list) != problem.n_households: + sources_list = list(source_household_ids) + if len(sources_list) != len(assigned): raise ValueError( "source_household_ids must align with households, got " - f"{len(sources_list)} for {problem.n_households}." + f"{len(sources_list)} for {len(assigned)}." ) - sources = np.empty(problem.n_households, dtype=object) + sources = np.empty(len(assigned), dtype=object) for index, source in enumerate(sources_list): sources[index] = source - assigned = np.asarray(problem.assigned_areas, dtype=object) - rows: list[dict[str, Any]] = [] - for area_code in problem.area_codes: - members = np.flatnonzero(assigned == area_code) - member_weights = values[members] - positive = member_weights > 0 - weight_sum = float(member_weights.sum()) - square_sum = float(np.square(member_weights).sum()) - rows.append( - { - "area_code": area_code, - "assigned_households": int(len(members)), - "nonzero_households": int(positive.sum()), - "nonzero_source_households": int( - len(set(sources[members][positive].tolist())) - ), - "weight_sum": weight_sum, - "max_weight": (float(member_weights.max()) if len(members) else 0.0), - "effective_sample_size": ( - weight_sum**2 / square_sum if square_sum > 0 else 0.0 - ), - } - ) - return pd.DataFrame(rows) + area_roster = np.asarray(area_codes, dtype=object) + if area_roster.ndim != 1: + raise ValueError( + f"area_codes must be one-dimensional, got shape {area_roster.shape}." + ) + + source_codes, _ = pd.factorize( + pd.Series(sources, dtype=object), + sort=False, + use_na_sentinel=False, + ) + positive = values > 0 + rows = pd.DataFrame( + { + "area_code": assigned, + "weight": values, + "weight_squared": np.square(values), + "positive": positive, + "positive_source_code": np.where(positive, source_codes, -1), + } + ) + grouped = rows.groupby("area_code", sort=False, dropna=False).agg( + assigned_households=("weight", "size"), + nonzero_households=("positive", "sum"), + nonzero_source_households=( + "positive_source_code", + lambda codes: int(codes[codes >= 0].nunique()), + ), + weight_sum=("weight", "sum"), + max_weight=("weight", "max"), + weight_square_sum=("weight_squared", "sum"), + ) + support = pd.DataFrame({"area_code": area_roster}).merge( + grouped, + how="left", + left_on="area_code", + right_index=True, + sort=False, + ) + count_columns = ( + "assigned_households", + "nonzero_households", + "nonzero_source_households", + ) + support[list(count_columns)] = ( + support[list(count_columns)].fillna(0).astype(np.int64) + ) + support[["weight_sum", "max_weight", "weight_square_sum"]] = support[ + ["weight_sum", "max_weight", "weight_square_sum"] + ].fillna(0.0) + effective_sample_size = np.zeros(len(support), dtype=np.float64) + np.divide( + np.square(support["weight_sum"].to_numpy(dtype=np.float64)), + support["weight_square_sum"].to_numpy(dtype=np.float64), + out=effective_sample_size, + where=support["weight_square_sum"].to_numpy(dtype=np.float64) > 0, + ) + support["effective_sample_size"] = effective_sample_size + return support[ + [ + "area_code", + "assigned_households", + "nonzero_households", + "nonzero_source_households", + "weight_sum", + "max_weight", + "effective_sample_size", + ] + ] + + +def uk_ladder_area_support_summary( + household: pd.DataFrame, + ladder: Any, + *, + weight_column: str = "household_weight", + source_column: str = "source_household_id", +) -> dict[str, pd.DataFrame]: + """Return matrix-free constituency and LA support on the ladder roster.""" + + if source_column not in household.columns: + raise ValueError( + f"household table must contain source column {source_column!r}; " + "distinct-source honesty requires it (pass source_column=" + "'household_id' explicitly for row-grain sources)." + ) + if weight_column not in household.columns: + raise ValueError(f"household table must contain weight column {weight_column!r}.") + + summaries: dict[str, pd.DataFrame] = {} + for area_type, assigned_column, ladder_codes in ( + ("constituency", "constituency_code", ladder.constituency_code), + ("la", "local_authority_code", ladder.local_authority_code), + ): + if assigned_column not in household.columns: + raise ValueError( + f"household table must contain assigned area column " + f"{assigned_column!r}." + ) + summaries[area_type] = uk_area_support_summary( + household[assigned_column], + household[weight_column], + area_codes=np.unique(ladder_codes), + source_household_ids=household[source_column], + ) + return summaries diff --git a/packages/microcosm-build/tests/test_uk_geography_ladder.py b/packages/microcosm-build/tests/test_uk_geography_ladder.py index 586285cab7..5131be258b 100644 --- a/packages/microcosm-build/tests/test_uk_geography_ladder.py +++ b/packages/microcosm-build/tests/test_uk_geography_ladder.py @@ -26,6 +26,7 @@ load_uk_oa_ladder, uk_geography_ladder_assignment_summary, uk_geography_ladder_gate, + uk_region_mix, ) @@ -361,6 +362,30 @@ def test_expected_support_agrees_with_large_realized_assignment(tmp_path) -> Non ) +def test_region_mix_row_and_weight_shares_sum_to_one() -> None: + household = pd.DataFrame( + { + "region": ["LONDON", "London", "WALES"], + "household_weight": [1.0, 2.0, 7.0], + } + ) + + mix = uk_region_mix(household) + + assert mix.columns.tolist() == [ + "region_code", + "rows", + "row_share", + "weight_share", + ] + assert mix["row_share"].sum() == pytest.approx(1.0) + assert mix["weight_share"].sum() == pytest.approx(1.0) + london = mix.set_index("region_code").loc["E12000007"] + assert london["rows"] == 2 + assert london["row_share"] == pytest.approx(2 / 3) + assert london["weight_share"] == pytest.approx(0.3) + + def test_assignment_requires_region(tmp_path) -> None: ladder = load_uk_oa_ladder(_write_ladder(tmp_path / "ladder.npz")) household = pd.DataFrame({"household_id": [1]}) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 0bdc9b171e..831a2d63d3 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -401,7 +401,7 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) import json import sys - _, ladder_path = toy_ladder + ladder, ladder_path = toy_ladder builder = _load_builder_module() input_h5 = tmp_path / "staging.h5" _write_seam_h5(input_h5) @@ -438,6 +438,50 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) assert summary["weights"]["household_weight_kind"] == "importance" assert summary["weights"]["mass_conservation"]["passed"] is True assert summary["source_lineage"]["explicit"] is None + assert summary["area_support"]["source_basis"] == "source_household_id" + assert summary["area_support"]["constituency"]["n_areas"] == len( + np.unique(ladder.constituency_code) + ) + assert summary["area_support"]["la"]["n_areas"] == len( + np.unique(ladder.local_authority_code) + ) + assert set(summary["area_support"]["constituency"]) == { + "n_areas", + "min_rows", + "median_rows", + "min_ess", + "median_ess", + "min_distinct_sources", + "median_distinct_sources", + "bottom_by_rows", + "bottom_by_ess", + } + assert sum(row["row_share"] for row in summary["region_mix"]) == pytest.approx( + 1.0 + ) + assert sum( + row["weight_share"] for row in summary["region_mix"] + ) == pytest.approx(1.0) + area_support_path = output_dir / builder.AREA_SUPPORT_FILENAME + assert area_support_path.exists() + assert manifest["outputs"]["area_support_summary"]["path"] == str( + area_support_path + ) + area_support = pd.read_csv(area_support_path) + assert area_support.columns.tolist() == [ + "area_type", + "area_code", + "assigned_households", + "nonzero_households", + "nonzero_source_households", + "weight_sum", + "max_weight", + "effective_sample_size", + ] + assert area_support["area_type"].tolist() == [ + *(["constituency"] * len(np.unique(ladder.constituency_code))), + *(["la"] * len(np.unique(ladder.local_authority_code))), + ] assert (output_dir / "staging_rowwise.h5").exists() @@ -500,6 +544,8 @@ def test_driver_ladder_dry_run_matches_real_assignment( assert manifest["inputs"]["dataset"]["pin_verified"] is True assert manifest["inputs"]["ladder"]["pin_verified"] is True assert manifest["inputs"]["ladder"]["matches_local_area_crosswalk_pin"] is False + assert plan["area_support"] == manifest["rowwise_dataset"]["area_support"] + assert plan["region_mix"] == manifest["rowwise_dataset"]["region_mix"] # The dry-run's realized support is exact: identical draws to the build. realized = { @@ -555,6 +601,11 @@ def test_driver_ladder_spine_lineage_plan_manifest_parity( explicit = plan["source_lineage"]["explicit"] assert explicit == manifest["rowwise_dataset"]["source_lineage"]["explicit"] + assert plan["area_support"]["source_basis"] == "source_household_id" + assert ( + manifest["rowwise_dataset"]["area_support"]["source_basis"] + == "source_household_id" + ) assert explicit == { "basis": "explicit_lineage_columns", "columns_present": [ diff --git a/packages/microcosm-build/tests/test_uk_local_rowwise.py b/packages/microcosm-build/tests/test_uk_local_rowwise.py index 2b4252efa7..69e58fbb8d 100644 --- a/packages/microcosm-build/tests/test_uk_local_rowwise.py +++ b/packages/microcosm-build/tests/test_uk_local_rowwise.py @@ -28,7 +28,9 @@ rowwise_area_support_summary, rowwise_calibration_mass_reason, solve_uk_rowwise_weights_under_doctrine, + uk_area_support_summary, uk_household_weight_kind, + uk_ladder_area_support_summary, uk_national_frame, ) from microcosm.build.uk_runtime.weighted_integrity import ( @@ -407,6 +409,79 @@ def test_rowwise_area_support_summary_reports_all_target_areas() -> None: assert rows["E001"].effective_sample_size == pytest.approx(1.0) +def test_rowwise_area_support_wrapper_equals_frame_agnostic_core() -> None: + problem = build_uk_rowwise_local_matrix(_metrics(), _assigned(), _targets()) + weights = np.array([2.0, 0.0, 5.0]) + source_household_ids = [(2023, 1), (2023, 1), (2023, 2)] + + wrapped = rowwise_area_support_summary( + problem, + weights, + source_household_ids=source_household_ids, + ) + core = uk_area_support_summary( + problem.assigned_areas, + weights, + area_codes=problem.area_codes, + source_household_ids=source_household_ids, + ) + + pd.testing.assert_frame_equal(wrapped, core) + + +def test_ladder_area_support_includes_zeros_and_distinct_cloned_sources() -> None: + ladder = type( + "Ladder", + (), + { + "constituency_code": np.asarray(["C1", "C2", "C3"]), + "local_authority_code": np.asarray(["L1", "L2", "L3"]), + }, + )() + household = pd.DataFrame( + { + "household_id": [101, 102, 201, 202], + "source_household_id": [10, 10, 20, 20], + "household_weight": [2.0, 3.0, 4.0, 0.0], + "constituency_code": ["C1", "C1", "C2", "C2"], + "local_authority_code": ["L1", "L1", "L2", "L2"], + } + ) + + summaries = uk_ladder_area_support_summary(household, ladder) + + constituency = summaries["constituency"].set_index("area_code") + assert constituency.loc["C1", "assigned_households"] == 2 + assert constituency.loc["C1", "nonzero_households"] == 2 + assert constituency.loc["C1", "nonzero_source_households"] == 1 + assert constituency.loc["C2", "nonzero_source_households"] == 1 + assert constituency.loc["C3", "assigned_households"] == 0 + assert constituency.loc["C3", "effective_sample_size"] == 0.0 + assert summaries["la"]["area_code"].tolist() == ["L1", "L2", "L3"] + + +def test_ladder_area_support_refuses_missing_source_column() -> None: + ladder = type( + "Ladder", + (), + { + "constituency_code": np.asarray(["C1"]), + "local_authority_code": np.asarray(["L1"]), + }, + )() + household = pd.DataFrame( + { + "household_id": [1], + "household_weight": [1.0], + "constituency_code": ["C1"], + "local_authority_code": ["L1"], + } + ) + + with pytest.raises(ValueError, match="distinct-source honesty requires it"): + uk_ladder_area_support_summary(household, ladder) + + def test_matrix_builder_fails_closed_on_unreachable_nonzero_targets() -> None: # A target area with no assigned households cannot be hit; a nonzero # target there must refuse at build time, while a zero target is fine. diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index b389da04d8..47d9db72fb 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -181,6 +181,9 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( _write_toy_h5(input_h5) input_sha256 = builder._sha256(input_h5) _crosswalk_frame().to_csv(crosswalk_path, index=False) + output_dir.mkdir() + stale_area_support = output_dir / builder.AREA_SUPPORT_FILENAME + stale_area_support.write_text("stale") pd.DataFrame({"code": ["E14000001", "W07000041", "S14000001", "N05000001"]}).to_csv( constituency_codes, index=False ) @@ -221,6 +224,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( assert output_h5.exists() assert manifest_path.exists() assert coverage_path.exists() + assert not stale_area_support.exists() manifest = json.loads(manifest_path.read_text()) assert manifest["build_kind"] == "uk_rowwise_local_geography_dataset" assert manifest["parameters"]["n_clones"] == 2 @@ -235,6 +239,7 @@ def test_build_uk_rowwise_dataset_writes_manifest_and_outputs( assert manifest["rowwise_dataset"]["assigned_local_authorities"] == 2 assert manifest["coverage"][0]["covered_areas"] == 4 assert manifest["outputs"]["crosswalk"] is None + assert manifest["outputs"]["area_support_summary"] is None with pd.HDFStore(output_h5, mode="r") as store: assert store["household"].shape[0] == 4 assert store["person"].shape[0] == 6 @@ -404,6 +409,7 @@ def fake_clone(*_args, output_path: Path, **_kwargs): "ward_code": ["E05000001", "W05000001"], "constituency_code": ["E14000001", "W07000041"], "region_code": ["E12000007", "W99999999"], + "region": ["LONDON", "WALES"], "itl3_code": ["TLI", "TLL"], "itl2_code": ["TL", "TL"], "itl1_code": ["T", "T"], @@ -431,7 +437,17 @@ def fake_clone(*_args, output_path: Path, **_kwargs): }, )() - monkeypatch.setattr(builder, "load_uk_oa_ladder", lambda _path: object()) + ladder = type( + "Ladder", + (), + { + "constituency_code": pd.Series(["E14000001", "W07000041"]).to_numpy(), + "local_authority_code": pd.Series( + ["E06000063", "W06000001"] + ).to_numpy(), + }, + )() + monkeypatch.setattr(builder, "load_uk_oa_ladder", lambda _path: ladder) monkeypatch.setattr( builder, "clone_uk_dataset_with_ladder_geography", @@ -456,6 +472,10 @@ def fake_clone(*_args, output_path: Path, **_kwargs): assert builder.main() == 0 manifest_path = output_dir / builder.MANIFEST_FILENAME + manifest = json.loads(manifest_path.read_text()) + assert manifest["rowwise_dataset"]["area_support"]["source_basis"] == ( + "household_id" + ) rows = _spool_rows(output_dir) assert len(rows) == 1 assert rows[0].gate_verdicts == { @@ -517,6 +537,7 @@ def fail_clone(*_args, **_kwargs): "/tmp/escaped.h5", "rowwise_build_manifest.json", "geography_coverage_summary.csv", + "area_support_summary.csv", "uk_official_geography_crosswalk.csv.gz", ], ) @@ -556,7 +577,11 @@ def test_validate_output_paths_rejects_crosswalk_collision(tmp_path): @pytest.mark.parametrize( "sidecar_name", - ["rowwise_build_manifest.json", "geography_coverage_summary.csv"], + [ + "rowwise_build_manifest.json", + "geography_coverage_summary.csv", + "area_support_summary.csv", + ], ) def test_validate_output_paths_rejects_supplied_crosswalk_sidecar_collision( sidecar_name, diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 287b4b82a0..ee44a0a4ba 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -61,6 +61,8 @@ read_uk_single_year_weight_metadata, uk_geography_ladder_gate, uk_household_weight_kind, + uk_ladder_area_support_summary, + uk_region_mix, uk_time_period, validate_geography_coverage, write_geography_crosswalk, @@ -76,6 +78,7 @@ MANIFEST_FILENAME = "rowwise_build_manifest.json" COVERAGE_FILENAME = "geography_coverage_summary.csv" DRY_RUN_PLAN_FILENAME = "rowwise_dry_run_plan.json" +AREA_SUPPORT_FILENAME = "area_support_summary.csv" EXPECTED_SUPPORT_BOTTOM_AREAS = 15 _UK_ROWWISE_PIPELINE = "uk-local-rowwise" _REPOSITORY = Path(__file__).resolve().parents[1] @@ -396,6 +399,7 @@ def _main_impl( (args.out / COVERAGE_FILENAME).resolve(), (args.out / DRY_RUN_PLAN_FILENAME).resolve(), (args.out / CROSSWALK_FILENAME).resolve(), + (args.out / AREA_SUPPORT_FILENAME).resolve(), } if args.ladder.resolve() in sidecars: raise ValueError( @@ -415,6 +419,8 @@ def _main_impl( source_year=source_year, attempt=attempt, ) + if not args.dry_run: + (args.out / AREA_SUPPORT_FILENAME).unlink(missing_ok=True) crosswalk_source = _load_or_build_crosswalk(args) crosswalk = crosswalk_source.frame crosswalk_path = crosswalk_source.path @@ -506,6 +512,7 @@ def _main_impl( _artifact_info(crosswalk_path) if crosswalk_source.generated else None ), "coverage_summary": coverage_artifact, + "area_support_summary": None, }, "base_dataset": base_summary, "rowwise_dataset": rowwise_summary, @@ -583,6 +590,7 @@ def _dataset_output_path( MANIFEST_FILENAME, COVERAGE_FILENAME, DRY_RUN_PLAN_FILENAME, + AREA_SUPPORT_FILENAME, } if path.name in reserved: raise ValueError( @@ -601,6 +609,7 @@ def _validate_output_paths( (args.out / MANIFEST_FILENAME).resolve(), (args.out / COVERAGE_FILENAME).resolve(), (args.out / DRY_RUN_PLAN_FILENAME).resolve(), + (args.out / AREA_SUPPORT_FILENAME).resolve(), } generated_crosswalk_path = (args.out / CROSSWALK_FILENAME).resolve() reserved_paths = { @@ -956,6 +965,15 @@ def _run_ladder_route( "passed": bool(result.gate.passed), "details": dict(result.gate.details), } + household = _result_household(result) + area_support, area_support_long = _ladder_area_support_diagnostics( + household, + ladder, + ) + rowwise_summary["area_support"] = area_support + rowwise_summary["region_mix"] = uk_region_mix(household).to_dict("records") + area_support_path = args.out / AREA_SUPPORT_FILENAME + area_support_long.to_csv(area_support_path, index=False) manifest = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dataset", @@ -970,6 +988,7 @@ def _run_ladder_route( "dataset": _artifact_info(output_h5), "crosswalk": None, "coverage_summary": None, + "area_support_summary": _artifact_info(area_support_path), }, "base_dataset": base_summary, "rowwise_dataset": rowwise_summary, @@ -1059,8 +1078,13 @@ def _ladder_dry_run_plan( raise ValueError("household weights must carry positive total mass.") _kind, mass_log = read_uk_single_year_weight_metadata(input_h5) _assert_mass_log_current(mass_log, float(weight_values.sum())) + household_for_clone = household.copy() + if "source_household_id" not in household_for_clone.columns: + household_for_clone["source_household_id"] = household_for_clone[ + "household_id" + ] cloned = clone_entity_frame( - household, + household_for_clone, id_columns=("household_id",), n_clones=args.n_clones, id_multiplier=id_multiplier, @@ -1091,6 +1115,11 @@ def _ladder_dry_run_plan( ladder, n_clones=args.n_clones, ) + area_support, _area_support_long = _ladder_area_support_diagnostics( + assigned, + ladder, + ) + region_mix = uk_region_mix(assigned).to_dict("records") table_rows = { name: base_summary["tables"][name][0] for name in ("person", "benunit", "household") @@ -1145,6 +1174,8 @@ def _ladder_dry_run_plan( for area_type in ("constituency", "la") }, }, + "area_support": area_support, + "region_mix": region_mix, "source_lineage": _source_lineage_report( household, modulus=args.source_lineage_modulus, @@ -1303,6 +1334,79 @@ def _support_summary( } +def _result_household(result: Any) -> pd.DataFrame: + if isinstance(result, UKLadderRowwiseDatasetResult): + return engine_tables(result.frame, weighted_entities=("household",))[ + "household" + ] + return result.household + + +def _ladder_area_support_diagnostics( + household: pd.DataFrame, + ladder: Any, +) -> tuple[dict[str, Any], pd.DataFrame]: + source_basis = ( + "source_household_id" + if "source_household_id" in household.columns + else "household_id" + ) + summaries = uk_ladder_area_support_summary( + household, + ladder, + source_column=source_basis, + ) + block = { + "source_basis": source_basis, + **{ + area_type: _area_support_stats(summaries[area_type]) + for area_type in ("constituency", "la") + }, + } + long_frames = [] + for area_type in ("constituency", "la"): + frame = summaries[area_type].copy() + frame.insert(0, "area_type", area_type) + long_frames.append(frame) + return block, pd.concat(long_frames, ignore_index=True) + + +def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: + rows = support["assigned_households"] + ess = support["effective_sample_size"] + sources = support["nonzero_source_households"] + bottom_by_rows = support.sort_values( + ["assigned_households", "area_code"], + kind="mergesort", + ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) + bottom_by_ess = support.sort_values( + ["effective_sample_size", "area_code"], + kind="mergesort", + ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) + return { + "n_areas": int(len(support)), + "min_rows": int(rows.min()) if len(rows) else 0, + "median_rows": float(rows.median()) if len(rows) else 0.0, + "min_ess": float(ess.min()) if len(ess) else 0.0, + "median_ess": float(ess.median()) if len(ess) else 0.0, + "min_distinct_sources": int(sources.min()) if len(sources) else 0, + "median_distinct_sources": ( + float(sources.median()) if len(sources) else 0.0 + ), + "bottom_by_rows": [ + {"area_code": str(row.area_code), "rows": int(row.assigned_households)} + for row in bottom_by_rows.itertuples(index=False) + ], + "bottom_by_ess": [ + { + "area_code": str(row.area_code), + "ess": float(row.effective_sample_size), + } + for row in bottom_by_ess.itertuples(index=False) + ], + } + + def _pool_lineage_block(household: pd.DataFrame) -> dict[str, Any] | None: if POOL_SOURCE_LINEAGE_COLUMN not in household.columns: return None From 6ce70db217b1ea601a50db74547b2e6b5882d905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:52:35 +0200 Subject: [PATCH 07/14] Plan rows, bytes, and support at each candidate K in one dry-run --candidate-clone-counts (dry-run only, refused on a real build before any side effect) plans every candidate K independently at the build seed: per-K rows and byte estimates on both routes, realized and analytic expected support, and on the ladder route the per-area support stats. The fenced ladder plan body is extracted into _ladder_planned_assignment so the primary plan and every candidate K run the identical clone/assign/gate pipeline. The crosswalk route's candidate blocks omit area_support (no ladder roster) and say so in their basis string. Co-Authored-By: Claude Fable 5 --- .../tests/test_uk_ladder_rowwise_clone.py | 93 +++++++ .../tests/test_uk_rowwise_build_driver.py | 19 ++ .../tests/test_uk_rowwise_dry_run.py | 70 +++++ tools/build_uk_rowwise_dataset.py | 261 ++++++++++++++++-- 4 files changed, 414 insertions(+), 29 deletions(-) diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 831a2d63d3..6f7ff8eca6 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -565,6 +565,99 @@ def test_driver_ladder_dry_run_matches_real_assignment( assert manifest["rowwise_dataset"]["source_lineage"]["explicit"] is None +def test_driver_ladder_candidate_k_matches_independent_single_k_plan( + monkeypatch, toy_ladder, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + import json + import sys + + _, ladder_path = toy_ladder + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + candidate_dir = tmp_path / "candidates" + independent_dir = tmp_path / "independent" + base_argv = [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder", + str(ladder_path), + "--seed", + "7", + ] + + monkeypatch.setattr( + sys, + "argv", + [ + *base_argv, + "--out", + str(candidate_dir), + "--n-clones", + "2", + "--candidate-clone-counts", + "3,1,3", + "--dry-run", + ], + ) + assert builder.main() == 0 + candidate_plan = json.loads( + (candidate_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() + ) + + monkeypatch.setattr( + sys, + "argv", + [ + *base_argv, + "--out", + str(independent_dir), + "--n-clones", + "3", + "--dry-run", + ], + ) + assert builder.main() == 0 + independent_plan = json.loads( + (independent_dir / builder.DRY_RUN_PLAN_FILENAME).read_text() + ) + + assert candidate_plan["plan"]["n_clones"] == 2 + candidates = candidate_plan["candidates"] + assert candidates["clone_counts"] == [1, 3] + assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] + input_bytes = input_h5.stat().st_size + base_rows = {"person": 5, "benunit": 4, "household": 4} + for candidate in candidates["plans"]: + n_clones = candidate["n_clones"] + assert candidate["rows"] == { + name: rows * n_clones for name, rows in base_rows.items() + } + assert candidate["output_bytes_estimate"] == input_bytes * n_clones + assert set(candidate) == { + "n_clones", + "rows", + "output_bytes_estimate", + "realized_support", + "expected_support", + "area_support", + } + + candidate_k3 = candidates["plans"][1] + assert candidate_k3["realized_support"] == { + area_type: independent_plan["realized_support"][area_type] + for area_type in ("constituency", "la") + } + assert candidate_k3["expected_support"] == { + area_type: independent_plan["expected_support"][area_type] + for area_type in ("constituency", "la") + } + assert candidate_k3["area_support"] == independent_plan["area_support"] + + def test_driver_ladder_spine_lineage_plan_manifest_parity( monkeypatch, toy_ladder, tmp_path ): diff --git a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py index 47d9db72fb..d65a6d487f 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_build_driver.py @@ -530,6 +530,25 @@ def fail_clone(*_args, **_kwargs): assert row.gate_verdicts["pipeline_error"]["receipt"].endswith("#/error_type") +def test_candidate_clone_counts_refused_on_real_build(tmp_path) -> None: + builder = _load_builder_module() + output_dir = tmp_path / "out" + + with pytest.raises(ValueError, match="candidate-K planning is a dry-run surface"): + builder.main( + [ + "--input-h5", + str(tmp_path / "not-read.h5"), + "--out", + str(output_dir), + "--candidate-clone-counts", + "1,2,4", + ] + ) + + assert not output_dir.exists() + + @pytest.mark.parametrize( "dataset_filename", [ diff --git a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py index 153cd8ca43..b1cb6c77bc 100644 --- a/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py +++ b/packages/microcosm-build/tests/test_uk_rowwise_dry_run.py @@ -12,6 +12,7 @@ from __future__ import annotations +import argparse import importlib.util import json import sys @@ -141,6 +142,28 @@ def _dry_run_argv( ] +def test_candidate_clone_counts_parser_deduplicates_and_sorts() -> None: + builder = _load_builder_module() + + assert builder._candidate_clone_counts_argument("10, 2,4,2,1") == ( + 1, + 2, + 4, + 10, + ) + + +@pytest.mark.parametrize( + "value", + ["", " ", ",", "0", "-1", "1,0", "1,-2", "1,junk", "1,,2"], +) +def test_candidate_clone_counts_parser_refuses_invalid_values(value) -> None: + builder = _load_builder_module() + + with pytest.raises(argparse.ArgumentTypeError, match="positive integers"): + builder._candidate_clone_counts_argument(value) + + def test_collision_free_expectation_matches_distribution_math() -> None: support = expected_uk_rowwise_area_support( _household_frame(), @@ -248,6 +271,53 @@ def test_driver_dry_run_writes_plan_only(monkeypatch, tmp_path) -> None: assert plan["source_lineage"]["explicit"] is None +def test_crosswalk_dry_run_reports_candidate_k_rows_bytes_and_support( + monkeypatch, tmp_path +) -> None: + pytest.importorskip("tables") + pytest.importorskip("h5py") + builder = _load_builder_module() + input_h5 = tmp_path / "populace_uk_2023.h5" + crosswalk_path = tmp_path / "crosswalk.csv.gz" + output_dir = tmp_path / "out" + _write_toy_h5(input_h5) + _crosswalk_frame().to_csv(crosswalk_path, index=False) + + monkeypatch.setattr( + sys, + "argv", + _dry_run_argv( + input_h5, + output_dir, + crosswalk_path, + "--candidate-clone-counts", + "3,1,3", + ), + ) + assert builder.main() == 0 + plan = json.loads((output_dir / builder.DRY_RUN_PLAN_FILENAME).read_text()) + + assert plan["parameters"]["n_clones"] == 2 + assert plan["parameters"]["candidate_clone_counts"] == [1, 3] + candidates = plan["candidates"] + assert candidates["clone_counts"] == [1, 3] + assert [candidate["n_clones"] for candidate in candidates["plans"]] == [1, 3] + input_bytes = input_h5.stat().st_size + base_rows = {"person": 3, "benunit": 2, "household": 2} + for candidate in candidates["plans"]: + n_clones = candidate["n_clones"] + assert candidate["rows"] == { + name: rows * n_clones for name, rows in base_rows.items() + } + assert candidate["output_bytes_estimate"] == input_bytes * n_clones + assert set(candidate["realized_support"]) == {"constituency", "la"} + assert set(candidate["collision_free_expected_support"]) == { + "constituency", + "la", + } + assert "area_support" not in candidate + + def test_driver_dry_run_reports_weight_chain_and_pool_lineage( monkeypatch, tmp_path ) -> None: diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index ee44a0a4ba..89154c217b 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -84,6 +84,25 @@ _REPOSITORY = Path(__file__).resolve().parents[1] +def _candidate_clone_counts_argument(value: str) -> tuple[int, ...]: + parts = value.split(",") + if not value.strip() or any(not part.strip() for part in parts): + raise argparse.ArgumentTypeError( + "candidate clone counts must be a non-empty comma list of positive integers" + ) + try: + clone_counts = [int(part.strip()) for part in parts] + except ValueError as error: + raise argparse.ArgumentTypeError( + "candidate clone counts must be a comma list of positive integers" + ) from error + if any(clone_count <= 0 for clone_count in clone_counts): + raise argparse.ArgumentTypeError( + "candidate clone counts must all be positive integers" + ) + return tuple(sorted(set(clone_counts))) + + def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( @@ -145,6 +164,14 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="Optional CSV containing a `code` column for local-authority coverage checks.", ) parser.add_argument("--n-clones", type=int, default=2) + parser.add_argument( + "--candidate-clone-counts", + type=_candidate_clone_counts_argument, + help=( + "Dry-run only: comma-separated candidate clone counts to plan " + "independently at the build seed (deduplicated and sorted)." + ), + ) parser.add_argument("--seed", type=int, default=42) parser.add_argument( "--source-year", @@ -312,6 +339,11 @@ def _record_rowwise_error( def main(argv: list[str] | None = None) -> int: args = _parse_args(argv) + if args.candidate_clone_counts is not None and not args.dry_run: + raise ValueError( + "candidate-K planning is a dry-run surface; a real build has " + "exactly one K from --n-clones." + ) if args.dry_run: return _main_impl(args, attempt=None) @@ -711,6 +743,11 @@ def _parameters(args: argparse.Namespace, *, source_year: int) -> dict[str, Any] ladder_route = getattr(args, "ladder", None) is not None return { "n_clones": args.n_clones, + "candidate_clone_counts": ( + list(args.candidate_clone_counts) + if args.candidate_clone_counts is not None + else None + ), "seed": args.seed, "source_year": source_year, # Crosswalk-sampler knobs are meaningless on the ladder route and are @@ -821,7 +858,7 @@ def _dry_run_plan( for name in ("person", "benunit", "household") } input_bytes = input_h5.stat().st_size - return { + plan = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dry_run", "created_at": datetime.now(UTC).isoformat(), @@ -876,6 +913,81 @@ def _dry_run_plan( ), "coverage": coverage.to_dict("records") if not coverage.empty else [], } + if args.candidate_clone_counts is not None: + plan["candidates"] = _crosswalk_candidate_plans( + args, + household=household, + crosswalk=crosswalk, + source_year=source_year, + id_multiplier=id_multiplier, + table_rows=table_rows, + input_bytes=input_bytes, + ) + return plan + + +def _crosswalk_candidate_plans( + args: argparse.Namespace, + *, + household: pd.DataFrame, + crosswalk: pd.DataFrame, + source_year: int, + id_multiplier: int, + table_rows: dict[str, int], + input_bytes: int, +) -> dict[str, Any]: + clone_counts = args.candidate_clone_counts + assert clone_counts is not None + plans = [] + for n_clones in clone_counts: + assignment = assign_household_geography( + household, + crosswalk, + n_clones=n_clones, + seed=args.seed, + id_multiplier=id_multiplier, + source_year=source_year, + require_all_countries=not args.allow_missing_country, + require_constituency=not args.allow_blank_constituency, + constrain_to_region=not args.allow_cross_region_assignment, + avoid_constituency_collisions=not args.allow_constituency_collisions, + ) + realized = _realized_area_support(assignment.household, crosswalk) + collision_free = expected_uk_rowwise_area_support( + household, + crosswalk, + n_clones=n_clones, + source_year=source_year, + require_all_countries=not args.allow_missing_country, + require_constituency=not args.allow_blank_constituency, + constrain_to_region=not args.allow_cross_region_assignment, + ) + plans.append( + { + "n_clones": n_clones, + "rows": { + name: rows * n_clones for name, rows in table_rows.items() + }, + "output_bytes_estimate": input_bytes * n_clones, + "realized_support": { + area_type: _support_summary(realized, area_type) + for area_type in ("constituency", "la") + }, + "collision_free_expected_support": { + area_type: _support_summary(collision_free, area_type) + for area_type in ("constituency", "la") + }, + } + ) + return { + "basis": ( + "independent crosswalk assignments at the build seed for each " + "candidate K; realized and collision-free expected support only " + "because this route has no ladder roster for area-support stats" + ), + "clone_counts": list(clone_counts), + "plans": plans, + } def _run_ladder_route( @@ -1078,37 +1190,14 @@ def _ladder_dry_run_plan( raise ValueError("household weights must carry positive total mass.") _kind, mass_log = read_uk_single_year_weight_metadata(input_h5) _assert_mass_log_current(mass_log, float(weight_values.sum())) - household_for_clone = household.copy() - if "source_household_id" not in household_for_clone.columns: - household_for_clone["source_household_id"] = household_for_clone[ - "household_id" - ] - cloned = clone_entity_frame( - household_for_clone, - id_columns=("household_id",), - n_clones=args.n_clones, - id_multiplier=id_multiplier, - clone_index_column="clone_index", - ).reset_index(drop=True) - cloned["household_weight"] = ( - pd.to_numeric(cloned["household_weight"], errors="raise").to_numpy(dtype=float) - / args.n_clones - ) - assigned = assign_uk_geography_ladder( - cloned, + assigned = _ladder_planned_assignment( + household, ladder, + n_clones=args.n_clones, seed=args.seed, + id_multiplier=id_multiplier, expected_constituency_vintage=args.expected_constituency_vintage, ) - gate = uk_geography_ladder_gate( - assigned, - assigned["household_weight"].to_numpy(dtype=float), - ) - if not gate.passed: - raise ValueError( - "UK geography ladder gate would fail this build: " - + "; ".join(gate.failures) - ) realized = _ladder_realized_support(assigned, ladder) expected = expected_uk_ladder_area_support( household, @@ -1125,7 +1214,7 @@ def _ladder_dry_run_plan( for name in ("person", "benunit", "household") } input_bytes = input_h5.stat().st_size - return { + plan = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dry_run", "created_at": datetime.now(UTC).isoformat(), @@ -1182,6 +1271,120 @@ def _ladder_dry_run_plan( ), "coverage": [], } + if args.candidate_clone_counts is not None: + plan["candidates"] = _ladder_candidate_plans( + args, + household=household, + ladder=ladder, + id_multiplier=id_multiplier, + table_rows=table_rows, + input_bytes=input_bytes, + ) + return plan + + +def _ladder_planned_assignment( + household: pd.DataFrame, + ladder: Any, + *, + n_clones: int, + seed: int, + id_multiplier: int, + expected_constituency_vintage: str, +) -> pd.DataFrame: + """Execute the fenced ladder assignment used by every dry-run K.""" + + household_for_clone = household.copy() + if "source_household_id" not in household_for_clone.columns: + household_for_clone["source_household_id"] = household_for_clone[ + "household_id" + ] + cloned = clone_entity_frame( + household_for_clone, + id_columns=("household_id",), + n_clones=n_clones, + id_multiplier=id_multiplier, + clone_index_column="clone_index", + ).reset_index(drop=True) + cloned["household_weight"] = ( + pd.to_numeric(cloned["household_weight"], errors="raise").to_numpy(dtype=float) + / n_clones + ) + assigned = assign_uk_geography_ladder( + cloned, + ladder, + seed=seed, + expected_constituency_vintage=expected_constituency_vintage, + ) + gate = uk_geography_ladder_gate( + assigned, + assigned["household_weight"].to_numpy(dtype=float), + ) + if not gate.passed: + raise ValueError( + "UK geography ladder gate would fail this build: " + + "; ".join(gate.failures) + ) + return assigned + + +def _ladder_candidate_plans( + args: argparse.Namespace, + *, + household: pd.DataFrame, + ladder: Any, + id_multiplier: int, + table_rows: dict[str, int], + input_bytes: int, +) -> dict[str, Any]: + clone_counts = args.candidate_clone_counts + assert clone_counts is not None + plans = [] + for n_clones in clone_counts: + assigned = _ladder_planned_assignment( + household, + ladder, + n_clones=n_clones, + seed=args.seed, + id_multiplier=id_multiplier, + expected_constituency_vintage=args.expected_constituency_vintage, + ) + realized = _ladder_realized_support(assigned, ladder) + expected = expected_uk_ladder_area_support( + household, + ladder, + n_clones=n_clones, + ) + area_support, _area_support_long = _ladder_area_support_diagnostics( + assigned, + ladder, + ) + plans.append( + { + "n_clones": n_clones, + "rows": { + name: rows * n_clones for name, rows in table_rows.items() + }, + "output_bytes_estimate": input_bytes * n_clones, + "realized_support": { + area_type: _support_summary(realized, area_type) + for area_type in ("constituency", "la") + }, + "expected_support": { + area_type: _support_summary(expected, area_type) + for area_type in ("constituency", "la") + }, + "area_support": area_support, + } + ) + return { + "basis": ( + "independent fenced ladder assignments at the build seed for each " + "candidate K; summary-only plans" + ), + "clone_counts": list(clone_counts), + "plans": plans, + } def _ladder_realized_support( From 652d2891670bbfe090874905a52c9860825a24b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 29 Aug 2026 02:22:12 +0200 Subject: [PATCH 08/14] Name the spine as the driver's sanctioned input, and the changelog entry Co-Authored-By: Claude Fable 5 --- changelog.d/761-uk-rowwise-pilot.added.md | 1 + tools/build_uk_rowwise_dataset.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changelog.d/761-uk-rowwise-pilot.added.md diff --git a/changelog.d/761-uk-rowwise-pilot.added.md b/changelog.d/761-uk-rowwise-pilot.added.md new file mode 100644 index 0000000000..7db3a1b12e --- /dev/null +++ b/changelog.d/761-uk-rowwise-pilot.added.md @@ -0,0 +1 @@ +The UK rowwise pilot machinery (#761, #495 increment 5) lands: the local-areas line is renamed `uk-local-*` and its `uk/local` Logbook scope is ratified (migration + mirror + README together); the rowwise dataset driver gains fail-closed `--input-sha256` and `--ladder-sha256` pins with a committed-crosswalk cross-check; the pool-only source-lineage modulus is refused on spine inputs carrying explicit lineage columns, which now report their own additive lineage block at pre-clone grain; the ladder clone path refuses pre-assigned geography columns; `expected_uk_ladder_area_support` computes the exact analytic per-constituency and per-LA expectation from the sampler's own stage weights; the per-area support diagnostics (ESS, distinct source households) lift off the solve matrix into `uk_area_support_summary` / `uk_ladder_area_support_summary` plus `uk_region_mix`, wired into the ladder manifest, dry-run plan, and a new `area_support_summary.csv` sidecar; and `--candidate-clone-counts` plans rows, bytes, and support at every candidate K in one dry-run. diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 89154c217b..e042b52f57 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -1,10 +1,13 @@ """Build a Microcosm UK row-wise local-geography dataset. -This is the narrow build driver for the UK local replacement path. It starts -from an existing compact Microcosm UK single-year H5, builds or loads the -official-source geography crosswalk, clones the entity tables, assigns each -household a finest available geography row, and writes diagnostics that prove -coverage and weight preservation. +This is the narrow build driver for the UK local replacement path. Its +sanctioned input is the microcosm-built national spine — the assembly seam's +accepted, sha-pinned single-year H5 with its declared weight kind and mass +log — never the retired certified compact and never anything upstream of the +seam. The driver clones the entity tables, assigns each household a finest +available geography row (the ratified OA ladder, or the crosswalk sampler as +the coverage-check path), and writes diagnostics that prove coverage, weight +preservation, and per-area support. """ from __future__ import annotations From 002cee98b9740a8d6fdb1f307f470a7b98221703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 29 Aug 2026 02:28:13 +0200 Subject: [PATCH 09/14] Archive the uk/local genesis row: the K=1 pilot build on spine-i One attempt, disposition iterating: uk-local-rowwise ladder route, f100, seed 42, spine-i (0cce0399) through the ratified ladder (9c6d56b9) -- gate passed 650/650, zero missing-geography rows, mass conserved exactly. Row digest a4d2c0b2. The chain opens at this row. Co-Authored-By: Claude Fable 5 --- logbook/uk/local.jsonl | 1 + 1 file changed, 1 insertion(+) create mode 100644 logbook/uk/local.jsonl diff --git a/logbook/uk/local.jsonl b/logbook/uk/local.jsonl new file mode 100644 index 0000000000..4e4d14a5b8 --- /dev/null +++ b/logbook/uk/local.jsonl @@ -0,0 +1 @@ +{"artifact_location":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/uk-2024-locals-rowwise-pilot-k1.h5","build_id":"uk-local-rowwise-ladder-f100-s42-20260829T002607Z-42440bbd","code_pin":"652d2891670bbfe090874905a52c9860825a24b6","cost_usd":null,"disposition":"iterating","gate_verdicts":{"uk_geography_ladder":{"receipt":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/rowwise_build_manifest.json#/rowwise_dataset/gate","verdict":"passed"}},"identity_digest":"9547050e6d16833fba9572a1517c4fad36bbeb0ccdbf4261c435884125c3fb1f","input_pins_digest":"eeb124ca3665982bd41e3ebc2820bfc6dd00c54097d4a55d1ff508f04d89dcf6","phases_reached":["attempt_started","configured","inputs_pinned","cloned","manifest_written"],"pipeline":"uk-local-rowwise","prediction_id":null,"prev_row_digest":null,"row_digest":"a4d2c0b2b356493cefb30aca6aa35ba33cfeff584b6946e09164a1cadff4c2de","rung":"f100","seed":42,"ts":"2026-08-29T00:26:07.897707Z","wall_seconds":10.081080500036478} From bfd8ab62a848709d62c57f3e0ffd7b24d162eb29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 29 Aug 2026 02:28:13 +0200 Subject: [PATCH 10/14] Record the Round-A pilot receipts and the K recommendation Measured on spine-i: the multi-K plan (K in {1,2,4,10}) and the real K=1 build. 650/650 constituencies at every K; constituency min-ESS climbs 15.3 -> 28.8 -> 86.4 -> 245; the analytic expectation tracks the realized draw on both grains; the K=1 build conserves mass exactly at 1.51 GB peak RSS. Recommendation: K=4 for the first calibrated candidate; adjudication pending Round B (spine-j) per Maria's ruling. Co-Authored-By: Claude Fable 5 --- experiments/761-uk-rowwise-pilot-receipts.md | 122 + experiments/761-uk-rowwise-pilot-support.json | 2300 +++++++++++++++++ 2 files changed, 2422 insertions(+) create mode 100644 experiments/761-uk-rowwise-pilot-receipts.md create mode 100644 experiments/761-uk-rowwise-pilot-support.json diff --git a/experiments/761-uk-rowwise-pilot-receipts.md b/experiments/761-uk-rowwise-pilot-receipts.md new file mode 100644 index 0000000000..60bcec3b84 --- /dev/null +++ b/experiments/761-uk-rowwise-pilot-receipts.md @@ -0,0 +1,122 @@ +# K=1 rowwise pilot receipts (#761, #495 increment 5) + +Round A: spine-i, 2026-08-29, licensed machine. Round B (spine-j, after the +#804 licensed phase re-cuts the acceptance receipt) re-runs the same two +commands and appends its section below; per María's 2026-08-28 ruling the K +adjudication cites Round B, with Round A as the control. + +## R0 — input identity (Round A) + +| item | value | +|---|---| +| input | `spine-i.h5`, sha256 `0cce03992207b4e16d96483642e2edd943fd25112a6425208d748b31c7742416` (= the committed `uk/spine_candidate_acceptance.json` candidate pin), 153,184,127 bytes | +| pin arm | `--input-sha256` verified fail-closed; manifest `inputs.dataset.pin_verified: true` | +| weight kind | `importance` in, `importance` out; input mass log 6 records | +| ladder | `uk_oa_ladder_2021.npz`, sha256 `9c6d56b90d2e975d750106b175020a54c5ec6acf42ef8909d304a9d7fc3868a7`, `matches_local_area_crosswalk_pin: true` | +| code | `652d2891` (`uk-rowwise-pilot-761` rung-8 tip, on the merged #795+#801 main) | +| receipts | dry-run plan sha `a44db1464e3a50ed…`, build manifest sha `b1a1294afeecff6e…`, evidence dir `data/ukds/acceptance/761-rowwise-pilot/spine-i/` (licensed-side) | + +## R1 — the multi-K dry-run plan + +One invocation (`--n-clones 1 --candidate-clone-counts 1,2,4,10 --dry-run`, +seed 42). Every candidate K runs the identical fenced clone → weight-divide → +ladder-assign → gate pipeline at the build seed; `expected` is the analytic +expectation from the sampler's own stage weights. Bytes are the declared +input×K lower bound; the real K=1 build measured ×1.044 over it (added +geography columns), giving the calibrated ceiling column. + +### Constituency grain (650 areas, all assigned at every K) + +| K | households | H5 est. (calibrated) | rows min / med (realized) | rows min / med (expected) | ESS min / med | distinct sources min / med | +|--:|--:|--:|--:|--:|--:|--:| +| 1 | 52,846 | 153 MB (≈160 MB) | 28 / 72 | 27.3 / 71.7 | 15.3 / 62.2 | 28 / 71 | +| 2 | 105,692 | 306 MB (≈319 MB) | 42 / 144 | 54.6 / 143.5 | 28.8 / 120.7 | 42 / 137 | +| 4 | 211,384 | 613 MB (≈640 MB) | 101 / 287 | 109.3 / 286.9 | 86.4 / 240.4 | 97 / 256 | +| 10 | 528,460 | 1,532 MB (≈1,600 MB) | 301 / 714.5 | 273.2 / 717.3 | 245.0 / 592.3 | 276 / 552.5 | + +### Local-authority grain (361 areas, all assigned at every K) + +| K | rows min / med (realized) | rows min / med (expected) | ESS min / med | distinct sources min / med | +|--:|--:|--:|--:|--:| +| 1 | 1 / 106 | 1.7 / 106.6 | 1.0 / 89.4 | 1 / 102 | +| 2 | 3 / 215 | 3.3 / 213.2 | 2.4 / 182.5 | 3 / 201 | +| 4 | 7 / 420 | 6.7 / 426.4 | 6.8 / 355.4 | 7 / 364 | +| 10 | 17 / 1,064 | 16.6 / 1,066.1 | 12.1 / 892.2 | 17 / 721 | + +The LA minimum is the Isles of Scilly (`E06000053`; ~0.05% of national +households), then the City of London (`E09000001`): population-proportional +assignment gives them almost nothing at any affordable K — a support floor, +if one is wanted there, is target-side or assignment-side work, not a clone +multiplier. The constituency floor is Na h-Eileanan an Iar (`S14000027`), +exactly the area the epic predicted (~240 rows at the old pool's K=1 scale ⇒ +~24 at the spine's, measured 28). + +The analytic expectation tracks the realized draw at every K on both grains +(largest relative gap at the thin tail, e.g. K=2 constituency min 54.6 +expected vs 42 realized — a single seed's draw at a 42-row floor). Full +per-K stats: `experiments/761-uk-rowwise-pilot-support.json`. + +## R2 — the real K=1 build + +`--n-clones 1 --seed 42`, both pins armed, 2026-08-29T00:26Z: + +| check | result | +|---|---| +| `uk_geography_ladder` gate | **passed** — constituency/ward/ITL nonempty weighted shares 1.0, London weighted share 0.1261 ∈ (0.08, 0.20) | +| assignment | 650/650 constituencies, 361/361 local authorities, `missing_geography_rows: 0` | +| mass conservation | `abs_delta: 0.0` (input = output = 29,247,433.0), rtol 1e-9, **exact** | +| weight-kind chain | `importance` → `importance`; mass log 6 → 7 records (the single clone record) | +| lineage | `explicit_lineage_columns` basis: 16,288 distinct `source_household_id` (frs 16,288 / spi 10,000 — the SPI channel's sources nest inside the FRS set); flags: spi_synthetic 20,085, cgt_clone 26,420, band_donor 270; `pool: null`, no modulus | +| duplicate source×constituency pairs | 1,454 (K=1: SPI/CGT copies of one source landing in its own constituency — the honest support discount the lineage columns exist to expose) | +| cost | 17.3 s wall, **1.51 GB peak RSS**, output H5 159,984,959 bytes (sha `8bb2b5fa5f28e03e…`) | +| Logbook | pipeline `uk-local-rowwise`, rung `f100`, build id `…-20260829T002607Z-42440bbd`, row digest `a4d2c0b2b356493c…` — the **genesis row of the ratified `uk/local` chain** (`logbook/uk/local.jsonl`, chain validates) | + +Memory ceiling at K, linear in the clone factor from the measured K=1 point: +≈1.5 GB (K=1) → ≈3 GB (K=2) → ≈6 GB (K=4) → ≈15 GB (K=10) peak RSS, with +H5 sizes per R1. All of K ∈ {1,2,4} fit ordinary hardware; K=10 remains +feasible on the licensed machine. + +## R3 — grain (the #688 evidence) + +Pre-clone: 52,846 households = (16,288 raw FRS + 10,000 SPI-synthetic) × 2 ++ 270 CGT band donors, carrying 16,288 distinct source households. Post-clone +at K=1: row counts unchanged (52,846 / 61,211 / 113,649), ids untouched +(`_remap_ids` is the identity at clone 0), weights divided by 1, one mass +record appended. K is a declared build parameter: `parameters.n_clones` in +the manifest, `candidates.clone_counts` in the plan — the declaration #688's +open half asked for. The old `household_id mod 10^8` lineage rule does not +apply to this spine and is structurally refused; distinct-source accounting +reads `source_household_id` directly. + +## R4 — rung ruling + +The Logbook rung stays `f100` at every K: rungs are sample-fraction tokens +(`DESIGN.md` scale ladder: f001 = 1% smoke, f010 = 10% development, f100 = +full; `logbook.py` `_validate_rung`: "#624 fraction token"), and the pilot +consumes 100% of the spine regardless of the clone multiplier. K lives in +`parameters.n_clones`, not the rung. + +## R5 — K recommendation (adjudication pending Round B) + +Measured support says the epic's "first published candidate at K=2–4" range +survives the spine swap, at the top of the range: + +- **K=1 is a pilot, not a candidate**: constituency min-ESS 15.3 means the + thinnest areas calibrate on ~15 effective households. +- **K=4 is the recommendation for #762's first calibrated candidate**: + constituency floor 101 rows / ESS 86 / 97 distinct sources, median 287 + rows — between the US sparse and dense arms per area — at 640 MB and + ~6 GB solve headroom. LA-grain thinness (Scilly, City of London) is + unfixable by K and should be handled target-side. +- **K=10 buys diminishing returns** (median 714 ≈ US-dense 775, but 1.6 GB + H5 and ~15 GB RSS) and adds zero information; hold it unless #762's solve + diagnostics show support-limited misses at K=4. + +**Adjudication (María):** _pending — rules on the Round B (spine-j) numbers, +this section then records the ruling and #761 carries it._ + +## Round B — spine-j + +_Pending the #804 licensed phase. Re-run R1's dry-run and R2's build with the +re-cut acceptance receipt's sha; support deltas vs Round A should be +region-mix-stable._ diff --git a/experiments/761-uk-rowwise-pilot-support.json b/experiments/761-uk-rowwise-pilot-support.json new file mode 100644 index 0000000000..08a99a0aed --- /dev/null +++ b/experiments/761-uk-rowwise-pilot-support.json @@ -0,0 +1,2300 @@ +{ + "artifact_kind": "uk_rowwise_pilot_support_stats", + "candidates": [ + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "E14001454", + "ess": 15.27149392611927 + }, + { + "area_code": "E14001132", + "ess": 20.095695257804277 + }, + { + "area_code": "E14001515", + "ess": 20.63481711963163 + }, + { + "area_code": "E14001508", + "ess": 21.63829872776318 + }, + { + "area_code": "E14001455", + "ess": 23.0322856228197 + }, + { + "area_code": "E14001360", + "ess": 23.140344869850303 + }, + { + "area_code": "S14000027", + "ess": 24.16263812992307 + }, + { + "area_code": "E14001463", + "ess": 24.317601510355406 + }, + { + "area_code": "E14001186", + "ess": 24.654670866710497 + }, + { + "area_code": "E14001195", + "ess": 24.66316425668849 + }, + { + "area_code": "E14001388", + "ess": 26.286214083844957 + }, + { + "area_code": "E14001550", + "ess": 26.793894830865497 + }, + { + "area_code": "E14001303", + "ess": 29.111499596347695 + }, + { + "area_code": "E14001524", + "ess": 29.16543602979089 + }, + { + "area_code": "E14001189", + "ess": 29.751497224394736 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 28 + }, + { + "area_code": "E14001454", + "rows": 32 + }, + { + "area_code": "E14001309", + "rows": 36 + }, + { + "area_code": "E14001089", + "rows": 38 + }, + { + "area_code": "E14001123", + "rows": 41 + }, + { + "area_code": "E14001270", + "rows": 41 + }, + { + "area_code": "E14001204", + "rows": 42 + }, + { + "area_code": "E14001187", + "rows": 43 + }, + { + "area_code": "E14001276", + "rows": 43 + }, + { + "area_code": "E14001137", + "rows": 44 + }, + { + "area_code": "E14001374", + "rows": 44 + }, + { + "area_code": "E14001448", + "rows": 44 + }, + { + "area_code": "E14001124", + "rows": 45 + }, + { + "area_code": "E14001261", + "rows": 45 + }, + { + "area_code": "E14001360", + "rows": 45 + } + ], + "median_distinct_sources": 71.0, + "median_ess": 62.151628242143616, + "median_rows": 72.0, + "min_distinct_sources": 28, + "min_ess": 15.27149392611927, + "min_rows": 28, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 1.0 + }, + { + "area_code": "E09000001", + "ess": 8.844673393420043 + }, + { + "area_code": "S12000027", + "ess": 17.798842023607254 + }, + { + "area_code": "E07000035", + "ess": 21.016318762353905 + }, + { + "area_code": "E07000133", + "ess": 22.18918146591543 + }, + { + "area_code": "S12000023", + "ess": 23.44623761223246 + }, + { + "area_code": "E06000017", + "ess": 23.94465696601676 + }, + { + "area_code": "S12000013", + "ess": 24.16263812992307 + }, + { + "area_code": "E07000124", + "ess": 29.030121672784347 + }, + { + "area_code": "E07000135", + "ess": 31.560810193240044 + }, + { + "area_code": "E07000142", + "ess": 34.00462465185576 + }, + { + "area_code": "E07000136", + "ess": 35.037877189449375 + }, + { + "area_code": "E07000237", + "ess": 35.25903544081407 + }, + { + "area_code": "E07000218", + "ess": 35.410416897750636 + }, + { + "area_code": "E07000223", + "ess": 35.42512690790566 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 1 + }, + { + "area_code": "E09000001", + "rows": 10 + }, + { + "area_code": "S12000027", + "rows": 21 + }, + { + "area_code": "E07000133", + "rows": 26 + }, + { + "area_code": "S12000023", + "rows": 27 + }, + { + "area_code": "E06000017", + "rows": 28 + }, + { + "area_code": "S12000013", + "rows": 28 + }, + { + "area_code": "E07000124", + "rows": 33 + }, + { + "area_code": "E07000135", + "rows": 37 + }, + { + "area_code": "E07000223", + "rows": 39 + }, + { + "area_code": "E07000218", + "rows": 40 + }, + { + "area_code": "E07000136", + "rows": 42 + }, + { + "area_code": "E07000235", + "rows": 44 + }, + { + "area_code": "E07000074", + "rows": 47 + }, + { + "area_code": "E07000120", + "rows": 49 + } + ], + "median_distinct_sources": 102.0, + "median_ess": 89.3840331485377, + "median_rows": 106.0, + "min_distinct_sources": 1, + "min_ess": 1.0, + "min_rows": 1, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 27.323142724462524 + }, + { + "area_code": "E14001213", + "rows": 43.96348004983817 + }, + { + "area_code": "E14001089", + "rows": 44.79852868761248 + }, + { + "area_code": "S14000051", + "rows": 45.50555403862137 + }, + { + "area_code": "E14001454", + "rows": 45.69263309892105 + }, + { + "area_code": "E14001270", + "rows": 45.81192576146024 + }, + { + "area_code": "E14001300", + "rows": 46.08830678160054 + }, + { + "area_code": "E14001417", + "rows": 46.34933330062193 + }, + { + "area_code": "E14001137", + "rows": 46.3989401503907 + }, + { + "area_code": "E14001187", + "rows": 46.85485072207512 + }, + { + "area_code": "E14001414", + "rows": 46.90918203372663 + }, + { + "area_code": "E14001301", + "rows": 47.16784632180665 + }, + { + "area_code": "E14001153", + "rows": 47.9922649203448 + }, + { + "area_code": "E14001123", + "rows": 48.6076260805717 + }, + { + "area_code": "E14001333", + "rows": 48.61353165792513 + } + ], + "max_rows": 361.91626312250185, + "mean_rows": 81.30153846153846, + "median_rows": 71.72847581522676, + "min_rows": 27.323142724462524, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 1.6627178705139922 + }, + { + "area_code": "E09000001", + "rows": 5.004371002424983 + }, + { + "area_code": "S12000023", + "rows": 22.242684068920017 + }, + { + "area_code": "S12000027", + "rows": 23.262869969701352 + }, + { + "area_code": "S12000013", + "rows": 27.323142724462524 + }, + { + "area_code": "E06000017", + "rows": 27.638664317142478 + }, + { + "area_code": "E07000133", + "rows": 35.05453232779583 + }, + { + "area_code": "E07000218", + "rows": 37.007407555320576 + }, + { + "area_code": "E07000135", + "rows": 37.140158454497374 + }, + { + "area_code": "E07000124", + "rows": 39.36028899235197 + }, + { + "area_code": "E07000199", + "rows": 43.96197631917495 + }, + { + "area_code": "E07000047", + "rows": 44.49059696774796 + }, + { + "area_code": "E07000074", + "rows": 46.41700376343651 + }, + { + "area_code": "E07000223", + "rows": 46.75337968963706 + }, + { + "area_code": "E07000125", + "rows": 46.8050199035923 + } + ], + "max_rows": 1091.7357576570837, + "mean_rows": 146.38781163434902, + "median_rows": 106.60903552779729, + "min_rows": 1.6627178705139922, + "n_areas": 361 + } + }, + "household_rows": 52846, + "n_clones": 1, + "output_bytes_estimate": 153184127, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 28.0 + }, + { + "area_code": "E14001454", + "rows": 32.0 + }, + { + "area_code": "E14001309", + "rows": 36.0 + }, + { + "area_code": "E14001089", + "rows": 38.0 + }, + { + "area_code": "E14001123", + "rows": 41.0 + }, + { + "area_code": "E14001270", + "rows": 41.0 + }, + { + "area_code": "E14001204", + "rows": 42.0 + }, + { + "area_code": "E14001187", + "rows": 43.0 + }, + { + "area_code": "E14001276", + "rows": 43.0 + }, + { + "area_code": "E14001137", + "rows": 44.0 + }, + { + "area_code": "E14001374", + "rows": 44.0 + }, + { + "area_code": "E14001448", + "rows": 44.0 + }, + { + "area_code": "E14001124", + "rows": 45.0 + }, + { + "area_code": "E14001261", + "rows": 45.0 + }, + { + "area_code": "E14001360", + "rows": 45.0 + } + ], + "max_rows": 354.0, + "mean_rows": 81.30153846153846, + "median_rows": 72.0, + "min_rows": 28.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 1.0 + }, + { + "area_code": "E09000001", + "rows": 10.0 + }, + { + "area_code": "S12000027", + "rows": 21.0 + }, + { + "area_code": "E07000133", + "rows": 26.0 + }, + { + "area_code": "S12000023", + "rows": 27.0 + }, + { + "area_code": "E06000017", + "rows": 28.0 + }, + { + "area_code": "S12000013", + "rows": 28.0 + }, + { + "area_code": "E07000124", + "rows": 33.0 + }, + { + "area_code": "E07000135", + "rows": 37.0 + }, + { + "area_code": "E07000223", + "rows": 39.0 + }, + { + "area_code": "E07000218", + "rows": 40.0 + }, + { + "area_code": "E07000136", + "rows": 42.0 + }, + { + "area_code": "E07000235", + "rows": 44.0 + }, + { + "area_code": "E07000074", + "rows": 47.0 + }, + { + "area_code": "E07000120", + "rows": 49.0 + } + ], + "max_rows": 1037.0, + "mean_rows": 146.38781163434902, + "median_rows": 106.0, + "min_rows": 1.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 28.772766859888733 + }, + { + "area_code": "E14001241", + "ess": 48.49213048209239 + }, + { + "area_code": "E14001354", + "ess": 51.87001349651396 + }, + { + "area_code": "E14001537", + "ess": 53.66603237929316 + }, + { + "area_code": "E14001421", + "ess": 54.07387209590336 + }, + { + "area_code": "E14001353", + "ess": 54.25201107246176 + }, + { + "area_code": "E14001575", + "ess": 55.46707664746293 + }, + { + "area_code": "E14001221", + "ess": 57.28923069544969 + }, + { + "area_code": "E14001339", + "ess": 57.53554831458898 + }, + { + "area_code": "E14001574", + "ess": 58.58463195974067 + }, + { + "area_code": "E14001132", + "ess": 59.28504402194878 + }, + { + "area_code": "E14001513", + "ess": 60.17316928039642 + }, + { + "area_code": "E14001445", + "ess": 61.57417297964269 + }, + { + "area_code": "E14001213", + "ess": 61.73672943178649 + }, + { + "area_code": "E14001153", + "ess": 62.18794773807564 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 42 + }, + { + "area_code": "E14001153", + "rows": 74 + }, + { + "area_code": "E14001414", + "rows": 79 + }, + { + "area_code": "E14001221", + "rows": 81 + }, + { + "area_code": "E14001300", + "rows": 81 + }, + { + "area_code": "E14001123", + "rows": 83 + }, + { + "area_code": "E14001187", + "rows": 84 + }, + { + "area_code": "E14001270", + "rows": 84 + }, + { + "area_code": "E14001167", + "rows": 87 + }, + { + "area_code": "E14001089", + "rows": 90 + }, + { + "area_code": "E14001257", + "rows": 91 + }, + { + "area_code": "E14001316", + "rows": 91 + }, + { + "area_code": "E14001333", + "rows": 91 + }, + { + "area_code": "E14001086", + "rows": 92 + }, + { + "area_code": "E14001526", + "rows": 92 + } + ], + "median_distinct_sources": 137.0, + "median_ess": 120.74144833002467, + "median_rows": 144.0, + "min_distinct_sources": 42, + "min_ess": 28.772766859888733, + "min_rows": 42, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 2.4156784854913016 + }, + { + "area_code": "E09000001", + "ess": 6.880763051494756 + }, + { + "area_code": "S12000013", + "ess": 28.772766859888733 + }, + { + "area_code": "S12000023", + "ess": 38.23722968460105 + }, + { + "area_code": "E06000017", + "ess": 39.626883904637666 + }, + { + "area_code": "E07000047", + "ess": 41.21513110073454 + }, + { + "area_code": "E07000133", + "ess": 41.55872805737255 + }, + { + "area_code": "S12000027", + "ess": 45.27906261453043 + }, + { + "area_code": "E07000218", + "ess": 48.75943362865257 + }, + { + "area_code": "E07000124", + "ess": 61.65635714775404 + }, + { + "area_code": "E07000223", + "ess": 68.0240614605536 + }, + { + "area_code": "E07000136", + "ess": 68.82740412657078 + }, + { + "area_code": "E07000135", + "ess": 69.31006171494359 + }, + { + "area_code": "E07000035", + "ess": 73.36159173306376 + }, + { + "area_code": "E07000142", + "ess": 74.38243109597174 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 3 + }, + { + "area_code": "E09000001", + "rows": 7 + }, + { + "area_code": "S12000013", + "rows": 42 + }, + { + "area_code": "S12000023", + "rows": 43 + }, + { + "area_code": "E06000017", + "rows": 46 + }, + { + "area_code": "S12000027", + "rows": 53 + }, + { + "area_code": "E07000133", + "rows": 64 + }, + { + "area_code": "E07000218", + "rows": 65 + }, + { + "area_code": "E07000124", + "rows": 68 + }, + { + "area_code": "E07000223", + "rows": 76 + }, + { + "area_code": "E07000136", + "rows": 81 + }, + { + "area_code": "E07000135", + "rows": 83 + }, + { + "area_code": "E07000125", + "rows": 89 + }, + { + "area_code": "E07000235", + "rows": 92 + }, + { + "area_code": "E07000120", + "rows": 95 + } + ], + "median_distinct_sources": 201.0, + "median_ess": 182.48559250642253, + "median_rows": 215.0, + "min_distinct_sources": 3, + "min_ess": 2.4156784854913016, + "min_rows": 3, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 54.64628544892505 + }, + { + "area_code": "E14001213", + "rows": 87.92696009967634 + }, + { + "area_code": "E14001089", + "rows": 89.59705737522496 + }, + { + "area_code": "S14000051", + "rows": 91.01110807724274 + }, + { + "area_code": "E14001454", + "rows": 91.3852661978421 + }, + { + "area_code": "E14001270", + "rows": 91.62385152292048 + }, + { + "area_code": "E14001300", + "rows": 92.17661356320107 + }, + { + "area_code": "E14001417", + "rows": 92.69866660124386 + }, + { + "area_code": "E14001137", + "rows": 92.7978803007814 + }, + { + "area_code": "E14001187", + "rows": 93.70970144415024 + }, + { + "area_code": "E14001414", + "rows": 93.81836406745326 + }, + { + "area_code": "E14001301", + "rows": 94.3356926436133 + }, + { + "area_code": "E14001153", + "rows": 95.9845298406896 + }, + { + "area_code": "E14001123", + "rows": 97.2152521611434 + }, + { + "area_code": "E14001333", + "rows": 97.22706331585026 + } + ], + "max_rows": 723.8325262450037, + "mean_rows": 162.6030769230769, + "median_rows": 143.45695163045352, + "min_rows": 54.64628544892505, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 3.3254357410279844 + }, + { + "area_code": "E09000001", + "rows": 10.008742004849966 + }, + { + "area_code": "S12000023", + "rows": 44.485368137840034 + }, + { + "area_code": "S12000027", + "rows": 46.525739939402705 + }, + { + "area_code": "S12000013", + "rows": 54.64628544892505 + }, + { + "area_code": "E06000017", + "rows": 55.277328634284956 + }, + { + "area_code": "E07000133", + "rows": 70.10906465559167 + }, + { + "area_code": "E07000218", + "rows": 74.01481511064115 + }, + { + "area_code": "E07000135", + "rows": 74.28031690899475 + }, + { + "area_code": "E07000124", + "rows": 78.72057798470394 + }, + { + "area_code": "E07000199", + "rows": 87.9239526383499 + }, + { + "area_code": "E07000047", + "rows": 88.98119393549592 + }, + { + "area_code": "E07000074", + "rows": 92.83400752687302 + }, + { + "area_code": "E07000223", + "rows": 93.50675937927411 + }, + { + "area_code": "E07000125", + "rows": 93.6100398071846 + } + ], + "max_rows": 2183.4715153141674, + "mean_rows": 292.77562326869804, + "median_rows": 213.21807105559458, + "min_rows": 3.3254357410279844, + "n_areas": 361 + } + }, + "household_rows": 105692, + "n_clones": 2, + "output_bytes_estimate": 306368254, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 42.0 + }, + { + "area_code": "E14001153", + "rows": 74.0 + }, + { + "area_code": "E14001414", + "rows": 79.0 + }, + { + "area_code": "E14001221", + "rows": 81.0 + }, + { + "area_code": "E14001300", + "rows": 81.0 + }, + { + "area_code": "E14001123", + "rows": 83.0 + }, + { + "area_code": "E14001187", + "rows": 84.0 + }, + { + "area_code": "E14001270", + "rows": 84.0 + }, + { + "area_code": "E14001167", + "rows": 87.0 + }, + { + "area_code": "E14001089", + "rows": 90.0 + }, + { + "area_code": "E14001257", + "rows": 91.0 + }, + { + "area_code": "E14001316", + "rows": 91.0 + }, + { + "area_code": "E14001333", + "rows": 91.0 + }, + { + "area_code": "E14001086", + "rows": 92.0 + }, + { + "area_code": "E14001526", + "rows": 92.0 + } + ], + "max_rows": 715.0, + "mean_rows": 162.6030769230769, + "median_rows": 144.0, + "min_rows": 42.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 3.0 + }, + { + "area_code": "E09000001", + "rows": 7.0 + }, + { + "area_code": "S12000013", + "rows": 42.0 + }, + { + "area_code": "S12000023", + "rows": 43.0 + }, + { + "area_code": "E06000017", + "rows": 46.0 + }, + { + "area_code": "S12000027", + "rows": 53.0 + }, + { + "area_code": "E07000133", + "rows": 64.0 + }, + { + "area_code": "E07000218", + "rows": 65.0 + }, + { + "area_code": "E07000124", + "rows": 68.0 + }, + { + "area_code": "E07000223", + "rows": 76.0 + }, + { + "area_code": "E07000136", + "rows": 81.0 + }, + { + "area_code": "E07000135", + "rows": 83.0 + }, + { + "area_code": "E07000125", + "rows": 89.0 + }, + { + "area_code": "E07000235", + "rows": 92.0 + }, + { + "area_code": "E07000120", + "rows": 95.0 + } + ], + "max_rows": 2176.0, + "mean_rows": 292.77562326869804, + "median_rows": 215.0, + "min_rows": 3.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 86.35718423510014 + }, + { + "area_code": "E14001231", + "ess": 108.2130074468227 + }, + { + "area_code": "E14001164", + "ess": 113.10723834593558 + }, + { + "area_code": "E14001417", + "ess": 114.49610093904711 + }, + { + "area_code": "E14001240", + "ess": 115.46871900751577 + }, + { + "area_code": "E14001536", + "ess": 118.08752459304661 + }, + { + "area_code": "E14001399", + "ess": 119.73863447434024 + }, + { + "area_code": "E14001371", + "ess": 125.82891686500399 + }, + { + "area_code": "E14001309", + "ess": 126.42553709302008 + }, + { + "area_code": "E14001454", + "ess": 127.31501498907119 + }, + { + "area_code": "E14001133", + "ess": 128.58281864500086 + }, + { + "area_code": "E14001485", + "ess": 129.49044303779488 + }, + { + "area_code": "E14001434", + "ess": 130.07713437572616 + }, + { + "area_code": "E14001388", + "ess": 130.93269449992587 + }, + { + "area_code": "E14001213", + "ess": 133.52997153756021 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 101 + }, + { + "area_code": "E14001213", + "rows": 154 + }, + { + "area_code": "E14001270", + "rows": 165 + }, + { + "area_code": "S14000051", + "rows": 173 + }, + { + "area_code": "E14001454", + "rows": 174 + }, + { + "area_code": "E14001089", + "rows": 179 + }, + { + "area_code": "E14001534", + "rows": 181 + }, + { + "area_code": "E14001417", + "rows": 182 + }, + { + "area_code": "E14001441", + "rows": 182 + }, + { + "area_code": "E14001558", + "rows": 183 + }, + { + "area_code": "E14001271", + "rows": 184 + }, + { + "area_code": "E14001301", + "rows": 185 + }, + { + "area_code": "E14001556", + "rows": 189 + }, + { + "area_code": "E14001189", + "rows": 191 + }, + { + "area_code": "E14001073", + "rows": 192 + } + ], + "median_distinct_sources": 256.0, + "median_ess": 240.42002819931716, + "median_rows": 287.0, + "min_distinct_sources": 97, + "min_ess": 86.35718423510014, + "min_rows": 101, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 6.803346397579617 + }, + { + "area_code": "E09000001", + "ess": 11.966201758605846 + }, + { + "area_code": "S12000023", + "ess": 71.25580669891772 + }, + { + "area_code": "S12000027", + "ess": 76.45656052703967 + }, + { + "area_code": "S12000013", + "ess": 86.35718423510014 + }, + { + "area_code": "E07000133", + "ess": 87.98764186217531 + }, + { + "area_code": "E06000017", + "ess": 91.98533042323214 + }, + { + "area_code": "E07000080", + "ess": 111.00058075828959 + }, + { + "area_code": "E07000236", + "ess": 129.4452253171101 + }, + { + "area_code": "E07000235", + "ess": 130.5025540814119 + }, + { + "area_code": "E07000135", + "ess": 131.41704492562806 + }, + { + "area_code": "E07000124", + "ess": 134.8185493449547 + }, + { + "area_code": "E07000199", + "ess": 139.0479421625425 + }, + { + "area_code": "E07000047", + "ess": 146.70768863130988 + }, + { + "area_code": "E07000119", + "ess": 150.24866836485344 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 7 + }, + { + "area_code": "E09000001", + "rows": 14 + }, + { + "area_code": "S12000023", + "rows": 86 + }, + { + "area_code": "S12000027", + "rows": 87 + }, + { + "area_code": "S12000013", + "rows": 101 + }, + { + "area_code": "E06000017", + "rows": 107 + }, + { + "area_code": "E07000133", + "rows": 142 + }, + { + "area_code": "E07000124", + "rows": 150 + }, + { + "area_code": "E07000236", + "rows": 151 + }, + { + "area_code": "E07000135", + "rows": 157 + }, + { + "area_code": "E07000199", + "rows": 164 + }, + { + "area_code": "E07000047", + "rows": 172 + }, + { + "area_code": "E07000218", + "rows": 179 + }, + { + "area_code": "E07000074", + "rows": 182 + }, + { + "area_code": "E07000125", + "rows": 186 + } + ], + "median_distinct_sources": 364.0, + "median_ess": 355.4371598394059, + "median_rows": 420.0, + "min_distinct_sources": 7, + "min_ess": 6.803346397579617, + "min_rows": 7, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 109.2925708978501 + }, + { + "area_code": "E14001213", + "rows": 175.85392019935267 + }, + { + "area_code": "E14001089", + "rows": 179.19411475044993 + }, + { + "area_code": "S14000051", + "rows": 182.02221615448548 + }, + { + "area_code": "E14001454", + "rows": 182.7705323956842 + }, + { + "area_code": "E14001270", + "rows": 183.24770304584095 + }, + { + "area_code": "E14001300", + "rows": 184.35322712640215 + }, + { + "area_code": "E14001417", + "rows": 185.39733320248772 + }, + { + "area_code": "E14001137", + "rows": 185.5957606015628 + }, + { + "area_code": "E14001187", + "rows": 187.41940288830048 + }, + { + "area_code": "E14001414", + "rows": 187.63672813490652 + }, + { + "area_code": "E14001301", + "rows": 188.6713852872266 + }, + { + "area_code": "E14001153", + "rows": 191.9690596813792 + }, + { + "area_code": "E14001123", + "rows": 194.4305043222868 + }, + { + "area_code": "E14001333", + "rows": 194.45412663170052 + } + ], + "max_rows": 1447.6650524900074, + "mean_rows": 325.2061538461538, + "median_rows": 286.91390326090703, + "min_rows": 109.2925708978501, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 6.650871482055969 + }, + { + "area_code": "E09000001", + "rows": 20.01748400969993 + }, + { + "area_code": "S12000023", + "rows": 88.97073627568007 + }, + { + "area_code": "S12000027", + "rows": 93.05147987880541 + }, + { + "area_code": "S12000013", + "rows": 109.2925708978501 + }, + { + "area_code": "E06000017", + "rows": 110.55465726856991 + }, + { + "area_code": "E07000133", + "rows": 140.21812931118333 + }, + { + "area_code": "E07000218", + "rows": 148.0296302212823 + }, + { + "area_code": "E07000135", + "rows": 148.5606338179895 + }, + { + "area_code": "E07000124", + "rows": 157.44115596940787 + }, + { + "area_code": "E07000199", + "rows": 175.8479052766998 + }, + { + "area_code": "E07000047", + "rows": 177.96238787099185 + }, + { + "area_code": "E07000074", + "rows": 185.66801505374605 + }, + { + "area_code": "E07000223", + "rows": 187.01351875854823 + }, + { + "area_code": "E07000125", + "rows": 187.2200796143692 + } + ], + "max_rows": 4366.943030628335, + "mean_rows": 585.5512465373961, + "median_rows": 426.43614211118916, + "min_rows": 6.650871482055969, + "n_areas": 361 + } + }, + "household_rows": 211384, + "n_clones": 4, + "output_bytes_estimate": 612736508, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 101.0 + }, + { + "area_code": "E14001213", + "rows": 154.0 + }, + { + "area_code": "E14001270", + "rows": 165.0 + }, + { + "area_code": "S14000051", + "rows": 173.0 + }, + { + "area_code": "E14001454", + "rows": 174.0 + }, + { + "area_code": "E14001089", + "rows": 179.0 + }, + { + "area_code": "E14001534", + "rows": 181.0 + }, + { + "area_code": "E14001417", + "rows": 182.0 + }, + { + "area_code": "E14001441", + "rows": 182.0 + }, + { + "area_code": "E14001558", + "rows": 183.0 + }, + { + "area_code": "E14001271", + "rows": 184.0 + }, + { + "area_code": "E14001301", + "rows": 185.0 + }, + { + "area_code": "E14001556", + "rows": 189.0 + }, + { + "area_code": "E14001189", + "rows": 191.0 + }, + { + "area_code": "E14001073", + "rows": 192.0 + } + ], + "max_rows": 1465.0, + "mean_rows": 325.2061538461538, + "median_rows": 287.0, + "min_rows": 101.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 7.0 + }, + { + "area_code": "E09000001", + "rows": 14.0 + }, + { + "area_code": "S12000023", + "rows": 86.0 + }, + { + "area_code": "S12000027", + "rows": 87.0 + }, + { + "area_code": "S12000013", + "rows": 101.0 + }, + { + "area_code": "E06000017", + "rows": 107.0 + }, + { + "area_code": "E07000133", + "rows": 142.0 + }, + { + "area_code": "E07000124", + "rows": 150.0 + }, + { + "area_code": "E07000236", + "rows": 151.0 + }, + { + "area_code": "E07000135", + "rows": 157.0 + }, + { + "area_code": "E07000199", + "rows": 164.0 + }, + { + "area_code": "E07000047", + "rows": 172.0 + }, + { + "area_code": "E07000218", + "rows": 179.0 + }, + { + "area_code": "E07000074", + "rows": 182.0 + }, + { + "area_code": "E07000125", + "rows": 186.0 + } + ], + "max_rows": 4396.0, + "mean_rows": 585.5512465373961, + "median_rows": 420.0, + "min_rows": 7.0, + "n_areas": 361 + } + } + }, + { + "area_support": { + "constituency": { + "bottom_by_ess": [ + { + "area_code": "S14000027", + "ess": 245.00661031168204 + }, + { + "area_code": "E14001454", + "ess": 309.58798378255244 + }, + { + "area_code": "E14001186", + "ess": 325.0021849429837 + }, + { + "area_code": "E14001300", + "ess": 336.04748855009973 + }, + { + "area_code": "E14001189", + "ess": 338.31029112850126 + }, + { + "area_code": "E14001276", + "ess": 343.75592586340593 + }, + { + "area_code": "E14001126", + "ess": 344.12611223537334 + }, + { + "area_code": "E14001548", + "ess": 345.93438568379673 + }, + { + "area_code": "E14001417", + "ess": 346.9780892361871 + }, + { + "area_code": "E14001429", + "ess": 348.2148890152362 + }, + { + "area_code": "E14001301", + "ess": 348.4567826735376 + }, + { + "area_code": "E14001187", + "ess": 349.9808726586294 + }, + { + "area_code": "E14001460", + "ess": 355.6390207205338 + }, + { + "area_code": "E14001550", + "ess": 355.8382451779587 + }, + { + "area_code": "E14001448", + "ess": 360.1587775232958 + } + ], + "bottom_by_rows": [ + { + "area_code": "S14000027", + "rows": 301 + }, + { + "area_code": "E14001213", + "rows": 443 + }, + { + "area_code": "E14001417", + "rows": 446 + }, + { + "area_code": "E14001270", + "rows": 447 + }, + { + "area_code": "E14001333", + "rows": 449 + }, + { + "area_code": "E14001301", + "rows": 450 + }, + { + "area_code": "E14001448", + "rows": 454 + }, + { + "area_code": "E14001454", + "rows": 454 + }, + { + "area_code": "E14001089", + "rows": 455 + }, + { + "area_code": "E14001187", + "rows": 459 + }, + { + "area_code": "E14001300", + "rows": 459 + }, + { + "area_code": "E14001271", + "rows": 479 + }, + { + "area_code": "S14000051", + "rows": 487 + }, + { + "area_code": "E14001550", + "rows": 491 + }, + { + "area_code": "E14001189", + "rows": 493 + } + ], + "median_distinct_sources": 552.5, + "median_ess": 592.328299460505, + "median_rows": 714.5, + "min_distinct_sources": 276, + "min_ess": 245.00661031168204, + "min_rows": 301, + "n_areas": 650 + }, + "la": { + "bottom_by_ess": [ + { + "area_code": "E06000053", + "ess": 12.113919922730993 + }, + { + "area_code": "E09000001", + "ess": 42.0438939071751 + }, + { + "area_code": "E06000017", + "ess": 180.75401191345054 + }, + { + "area_code": "S12000023", + "ess": 197.0121926202139 + }, + { + "area_code": "S12000027", + "ess": 198.28988727221227 + }, + { + "area_code": "E07000133", + "ess": 222.9318295202106 + }, + { + "area_code": "E07000135", + "ess": 244.23156569686483 + }, + { + "area_code": "S12000013", + "ess": 245.00661031168204 + }, + { + "area_code": "E07000218", + "ess": 319.2242286745908 + }, + { + "area_code": "E07000223", + "ess": 332.43048742715064 + }, + { + "area_code": "E07000124", + "ess": 344.6432429352619 + }, + { + "area_code": "E07000199", + "ess": 374.24511694345773 + }, + { + "area_code": "E07000235", + "ess": 374.33815974358714 + }, + { + "area_code": "E07000047", + "ess": 374.8395900190861 + }, + { + "area_code": "S12000005", + "ess": 379.3890742905147 + } + ], + "bottom_by_rows": [ + { + "area_code": "E06000053", + "rows": 17 + }, + { + "area_code": "E09000001", + "rows": 49 + }, + { + "area_code": "S12000027", + "rows": 238 + }, + { + "area_code": "S12000023", + "rows": 249 + }, + { + "area_code": "E06000017", + "rows": 287 + }, + { + "area_code": "S12000013", + "rows": 301 + }, + { + "area_code": "E07000135", + "rows": 350 + }, + { + "area_code": "E07000133", + "rows": 357 + }, + { + "area_code": "E07000218", + "rows": 389 + }, + { + "area_code": "E07000124", + "rows": 398 + }, + { + "area_code": "E07000199", + "rows": 422 + }, + { + "area_code": "E07000047", + "rows": 447 + }, + { + "area_code": "E07000074", + "rows": 456 + }, + { + "area_code": "E07000125", + "rows": 456 + }, + { + "area_code": "E07000223", + "rows": 463 + } + ], + "median_distinct_sources": 721.0, + "median_ess": 892.1532218859292, + "median_rows": 1064.0, + "min_distinct_sources": 17, + "min_ess": 12.113919922730993, + "min_rows": 17, + "n_areas": 361 + }, + "source_basis": "source_household_id" + }, + "expected_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 273.23142724462525 + }, + { + "area_code": "E14001213", + "rows": 439.63480049838165 + }, + { + "area_code": "E14001089", + "rows": 447.9852868761248 + }, + { + "area_code": "S14000051", + "rows": 455.05554038621364 + }, + { + "area_code": "E14001454", + "rows": 456.9263309892105 + }, + { + "area_code": "E14001270", + "rows": 458.11925761460236 + }, + { + "area_code": "E14001300", + "rows": 460.88306781600534 + }, + { + "area_code": "E14001417", + "rows": 463.49333300621925 + }, + { + "area_code": "E14001137", + "rows": 463.98940150390695 + }, + { + "area_code": "E14001187", + "rows": 468.54850722075116 + }, + { + "area_code": "E14001414", + "rows": 469.0918203372663 + }, + { + "area_code": "E14001301", + "rows": 471.6784632180665 + }, + { + "area_code": "E14001153", + "rows": 479.92264920344803 + }, + { + "area_code": "E14001123", + "rows": 486.076260805717 + }, + { + "area_code": "E14001333", + "rows": 486.13531657925125 + } + ], + "max_rows": 3619.1626312250182, + "mean_rows": 813.0153846153846, + "median_rows": 717.2847581522676, + "min_rows": 273.23142724462525, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 16.627178705139926 + }, + { + "area_code": "E09000001", + "rows": 50.043710024249826 + }, + { + "area_code": "S12000023", + "rows": 222.42684068920013 + }, + { + "area_code": "S12000027", + "rows": 232.62869969701347 + }, + { + "area_code": "S12000013", + "rows": 273.23142724462525 + }, + { + "area_code": "E06000017", + "rows": 276.3866431714248 + }, + { + "area_code": "E07000133", + "rows": 350.54532327795835 + }, + { + "area_code": "E07000218", + "rows": 370.0740755532057 + }, + { + "area_code": "E07000135", + "rows": 371.4015845449737 + }, + { + "area_code": "E07000124", + "rows": 393.6028899235196 + }, + { + "area_code": "E07000199", + "rows": 439.61976319174954 + }, + { + "area_code": "E07000047", + "rows": 444.90596967747956 + }, + { + "area_code": "E07000074", + "rows": 464.17003763436514 + }, + { + "area_code": "E07000223", + "rows": 467.5337968963706 + }, + { + "area_code": "E07000125", + "rows": 468.0501990359231 + } + ], + "max_rows": 10917.35757657084, + "mean_rows": 1463.8781163434903, + "median_rows": 1066.0903552779728, + "min_rows": 16.627178705139926, + "n_areas": 361 + } + }, + "household_rows": 528460, + "n_clones": 10, + "output_bytes_estimate": 1531841270, + "realized_support": { + "constituency": { + "bottom": [ + { + "area_code": "S14000027", + "rows": 301.0 + }, + { + "area_code": "E14001213", + "rows": 443.0 + }, + { + "area_code": "E14001417", + "rows": 446.0 + }, + { + "area_code": "E14001270", + "rows": 447.0 + }, + { + "area_code": "E14001333", + "rows": 449.0 + }, + { + "area_code": "E14001301", + "rows": 450.0 + }, + { + "area_code": "E14001448", + "rows": 454.0 + }, + { + "area_code": "E14001454", + "rows": 454.0 + }, + { + "area_code": "E14001089", + "rows": 455.0 + }, + { + "area_code": "E14001187", + "rows": 459.0 + }, + { + "area_code": "E14001300", + "rows": 459.0 + }, + { + "area_code": "E14001271", + "rows": 479.0 + }, + { + "area_code": "S14000051", + "rows": 487.0 + }, + { + "area_code": "E14001550", + "rows": 491.0 + }, + { + "area_code": "E14001189", + "rows": 493.0 + } + ], + "max_rows": 3634.0, + "mean_rows": 813.0153846153846, + "median_rows": 714.5, + "min_rows": 301.0, + "n_areas": 650 + }, + "la": { + "bottom": [ + { + "area_code": "E06000053", + "rows": 17.0 + }, + { + "area_code": "E09000001", + "rows": 49.0 + }, + { + "area_code": "S12000027", + "rows": 238.0 + }, + { + "area_code": "S12000023", + "rows": 249.0 + }, + { + "area_code": "E06000017", + "rows": 287.0 + }, + { + "area_code": "S12000013", + "rows": 301.0 + }, + { + "area_code": "E07000135", + "rows": 350.0 + }, + { + "area_code": "E07000133", + "rows": 357.0 + }, + { + "area_code": "E07000218", + "rows": 389.0 + }, + { + "area_code": "E07000124", + "rows": 398.0 + }, + { + "area_code": "E07000199", + "rows": 422.0 + }, + { + "area_code": "E07000047", + "rows": 447.0 + }, + { + "area_code": "E07000074", + "rows": 456.0 + }, + { + "area_code": "E07000125", + "rows": 456.0 + }, + { + "area_code": "E07000223", + "rows": 463.0 + } + ], + "max_rows": 10929.0, + "mean_rows": 1463.8781163434903, + "median_rows": 1064.0, + "min_rows": 17.0, + "n_areas": 361 + } + } + } + ], + "input_sha256": "0cce03992207b4e16d96483642e2edd943fd25112a6425208d748b31c7742416", + "ladder_sha256": "9c6d56b90d2e975d750106b175020a54c5ec6acf42ef8909d304a9d7fc3868a7", + "plan_git_commit": "652d2891670bbfe090874905a52c9860825a24b6", + "region_mix": [ + { + "region_code": "E12000001", + "row_share": 0.04087348143662718, + "rows": 2160, + "weight_share": 0.04132138137843869 + }, + { + "region_code": "E12000002", + "row_share": 0.09105703364493055, + "rows": 4812, + "weight_share": 0.11135883275636532 + }, + { + "region_code": "E12000003", + "row_share": 0.09870188850622563, + "rows": 5216, + "weight_share": 0.08260608717353075 + }, + { + "region_code": "E12000004", + "row_share": 0.061291299246868255, + "rows": 3239, + "weight_share": 0.07270689362721167 + }, + { + "region_code": "E12000005", + "row_share": 0.062067138477841274, + "rows": 3280, + "weight_share": 0.08658163834982258 + }, + { + "region_code": "E12000006", + "row_share": 0.08346894750785301, + "rows": 4411, + "weight_share": 0.09236710335114426 + }, + { + "region_code": "E12000007", + "row_share": 0.07652424024524089, + "rows": 4044, + "weight_share": 0.12610049116675187 + }, + { + "region_code": "E12000008", + "row_share": 0.12224198614843129, + "rows": 6460, + "weight_share": 0.13547538798818118 + }, + { + "region_code": "E12000009", + "row_share": 0.0822578813912122, + "rows": 4347, + "weight_share": 0.08746195492324632 + }, + { + "region_code": "N99999999", + "row_share": 0.1072171971388563, + "rows": 5666, + "weight_share": 0.027166840020911694 + }, + { + "region_code": "S99999999", + "row_share": 0.10222154940771298, + "rows": 5402, + "weight_share": 0.08843631963643898 + }, + { + "region_code": "W99999999", + "row_share": 0.07207735684820044, + "rows": 3809, + "weight_share": 0.04841706962795675 + } + ], + "round": "spine-i", + "seed": 42 +} From dca31857a5c5d48b9faf38a40d9f43ecae605931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sat, 29 Aug 2026 03:26:37 +0200 Subject: [PATCH 11/14] Re-pin the committed-archive roster for the opened uk/local chain test_committed_archives_are_scoped_by_country pins the committed archive inventory; archiving the genesis uk/local row legitimately extends it. Caught by CI's engine-free lanes -- the local runs predated the archive commit. Co-Authored-By: Claude Fable 5 --- packages/microcosm-build/tests/test_logbook_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/test_logbook_cli.py b/packages/microcosm-build/tests/test_logbook_cli.py index de3e0b752f..64444560f8 100644 --- a/packages/microcosm-build/tests/test_logbook_cli.py +++ b/packages/microcosm-build/tests/test_logbook_cli.py @@ -112,13 +112,13 @@ def test_cli_defaults_to_the_ratified_root_paths() -> None: def test_committed_archives_are_scoped_by_country() -> None: - # One chain per country: the US pool lineage and the UK migration - # chain never share an archive (microcosm#665). + # One chain per scope: the US pool lineage, and each UK line under its + # ratified scope path (microcosm#665; uk/local opened by #761). archives = sorted( path.relative_to(ROOT / "logbook").as_posix() for path in (ROOT / "logbook").rglob("*.jsonl") ) - assert archives == ["us.jsonl"] + assert archives == ["uk/local.jsonl", "us.jsonl"] assert (ROOT / "logbook" / "README.md").is_file() From 5422de7a00391e961239bc9258c0994ca144d8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:59:21 +0200 Subject: [PATCH 12/14] Disposition Vahid's review round: support basis, pin coverage, dead concat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finding 1 (confirmed as hardening, premise measured false here): the area-support stats and thinness ranking now read nonzero_households with the basis recorded and the assigned count kept per bottom area — a zero-weight row can never look like support. On spine-i assigned == nonzero for all 1,011 areas (the spine's SPI channel carries positive importance mass), so Round A's published figures stand unchanged. Finding 2 (confirmed): --ladder-sha256 now refuses whenever --ladder is absent — both crosswalk routes — instead of silently ignoring a pin the caller believes was verified. Finding 3 (confirmed): the per-area long frame is built only at the CSV write site; the dry-run paths no longer build and discard it per candidate K. Co-Authored-By: Claude Fable 5 --- experiments/761-uk-rowwise-pilot-receipts.md | 7 +++ .../tests/test_uk_ladder_rowwise_clone.py | 62 ++++++++++++++++++- tools/build_uk_rowwise_dataset.py | 43 +++++++++---- 3 files changed, 100 insertions(+), 12 deletions(-) diff --git a/experiments/761-uk-rowwise-pilot-receipts.md b/experiments/761-uk-rowwise-pilot-receipts.md index 60bcec3b84..fa685425b0 100644 --- a/experiments/761-uk-rowwise-pilot-receipts.md +++ b/experiments/761-uk-rowwise-pilot-receipts.md @@ -56,6 +56,13 @@ The analytic expectation tracks the realized draw at every K on both grains expected vs 42 realized — a single seed's draw at a 42-row floor). Full per-K stats: `experiments/761-uk-rowwise-pilot-support.json`. +Basis note (post review round): the stats block now reads +`nonzero_households` (`rows_basis` recorded), so a zero-weight row can never +count as support. Round A measured `assigned == nonzero` for all 1,011 areas +— the spine's SPI channel carries positive importance mass, unlike the old +pool's zero-weight convention — so every figure above stands unchanged; +Round B emits the labelled shape natively. + ## R2 — the real K=1 build `--n-clones 1 --seed 42`, both pins armed, 2026-08-29T00:26Z: diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index 6f7ff8eca6..ddf1315850 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -447,6 +447,7 @@ def test_driver_ladder_route_builds_with_gate(monkeypatch, toy_ladder, tmp_path) ) assert set(summary["area_support"]["constituency"]) == { "n_areas", + "rows_basis", "min_rows", "median_rows", "min_ess", @@ -770,10 +771,69 @@ def test_driver_ladder_sha256_refuses_crosswalk(monkeypatch, tmp_path) -> None: ], ) - with pytest.raises(ValueError, match="ladder-sha256.*crosswalk"): + with pytest.raises(ValueError, match="ladder-sha256.*ladder"): builder.main() +def test_driver_ladder_sha256_refuses_generated_crosswalk_route( + monkeypatch, tmp_path +) -> None: + # Neither --ladder nor --crosswalk: the driver would download and build a + # crosswalk, and there is no ladder artifact the pin could verify. A + # silently ignored pin would report verification that never happened. + pytest.importorskip("tables") + import sys + + builder = _load_builder_module() + input_h5 = tmp_path / "staging.h5" + _write_seam_h5(input_h5) + monkeypatch.setattr( + sys, + "argv", + [ + "build_uk_rowwise_dataset.py", + "--input-h5", + str(input_h5), + "--ladder-sha256", + "0" * 64, + "--out", + str(tmp_path / "out"), + "--dry-run", + ], + ) + + with pytest.raises(ValueError, match="ladder-sha256 requires --ladder"): + builder.main() + + +def test_area_support_stats_read_nonzero_rows_not_assigned() -> None: + # An area whose only rows are zero-weight synthetics shapes no estimate + # there: the stats and the thinness ranking must read mass-carrying rows, + # with the assigned count still visible per bottom area. + builder = _load_builder_module() + support = pd.DataFrame( + { + "area_code": ["A1", "A2"], + "assigned_households": [40, 3], + "nonzero_households": [0, 3], + "nonzero_source_households": [0, 3], + "weight_sum": [0.0, 30.0], + "max_weight": [0.0, 10.0], + "effective_sample_size": [0.0, 3.0], + } + ) + + stats = builder._area_support_stats(support) + + assert stats["rows_basis"] == "nonzero_households" + assert stats["min_rows"] == 0 + assert stats["bottom_by_rows"][0] == { + "area_code": "A1", + "rows": 0, + "assigned": 40, + } + + def test_driver_ladder_pin_mismatch_refuses_before_parse( monkeypatch, toy_ladder, tmp_path ) -> None: diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index e042b52f57..16142f6141 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -419,8 +419,12 @@ def _main_impl( source_year=source_year, ) _validate_output_paths(input_h5=input_h5, output_h5=output_h5, args=args) - if args.crosswalk is not None and args.ladder_sha256 is not None: - raise ValueError("--ladder-sha256 and --crosswalk are mutually exclusive.") + if args.ladder_sha256 is not None and args.ladder is None: + raise ValueError( + "--ladder-sha256 requires --ladder; the crosswalk routes have no " + "ladder artifact to pin, and silently ignoring a supplied pin " + "would report verification that never happened." + ) if args.ladder is not None: if args.crosswalk is not None: raise ValueError("--ladder and --crosswalk are mutually exclusive.") @@ -1081,14 +1085,16 @@ def _run_ladder_route( "details": dict(result.gate.details), } household = _result_household(result) - area_support, area_support_long = _ladder_area_support_diagnostics( + area_support, area_support_summaries = _ladder_area_support_diagnostics( household, ladder, ) rowwise_summary["area_support"] = area_support rowwise_summary["region_mix"] = uk_region_mix(household).to_dict("records") area_support_path = args.out / AREA_SUPPORT_FILENAME - area_support_long.to_csv(area_support_path, index=False) + _area_support_long_frame(area_support_summaries).to_csv( + area_support_path, index=False + ) manifest = { "schema_version": 1, "build_kind": "uk_rowwise_local_geography_dataset", @@ -1207,7 +1213,7 @@ def _ladder_dry_run_plan( ladder, n_clones=args.n_clones, ) - area_support, _area_support_long = _ladder_area_support_diagnostics( + area_support, _ = _ladder_area_support_diagnostics( assigned, ladder, ) @@ -1358,7 +1364,7 @@ def _ladder_candidate_plans( ladder, n_clones=n_clones, ) - area_support, _area_support_long = _ladder_area_support_diagnostics( + area_support, _ = _ladder_area_support_diagnostics( assigned, ladder, ) @@ -1551,7 +1557,7 @@ def _result_household(result: Any) -> pd.DataFrame: def _ladder_area_support_diagnostics( household: pd.DataFrame, ladder: Any, -) -> tuple[dict[str, Any], pd.DataFrame]: +) -> tuple[dict[str, Any], dict[str, pd.DataFrame]]: source_basis = ( "source_household_id" if "source_household_id" in household.columns @@ -1569,20 +1575,30 @@ def _ladder_area_support_diagnostics( for area_type in ("constituency", "la") }, } + return block, summaries + + +def _area_support_long_frame(summaries: dict[str, pd.DataFrame]) -> pd.DataFrame: + """The full per-area table, built only where a consumer writes it.""" + long_frames = [] for area_type in ("constituency", "la"): frame = summaries[area_type].copy() frame.insert(0, "area_type", area_type) long_frames.append(frame) - return block, pd.concat(long_frames, ignore_index=True) + return pd.concat(long_frames, ignore_index=True) def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: - rows = support["assigned_households"] + # Support means rows that carry mass: a row assigned to an area at zero + # weight shapes no estimate there, so the headline stats and the thinness + # ranking read nonzero_households. The assigned count stays visible per + # bottom area and in the full per-area CSV. + rows = support["nonzero_households"] ess = support["effective_sample_size"] sources = support["nonzero_source_households"] bottom_by_rows = support.sort_values( - ["assigned_households", "area_code"], + ["nonzero_households", "area_code"], kind="mergesort", ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) bottom_by_ess = support.sort_values( @@ -1591,6 +1607,7 @@ def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: ).head(EXPECTED_SUPPORT_BOTTOM_AREAS) return { "n_areas": int(len(support)), + "rows_basis": "nonzero_households", "min_rows": int(rows.min()) if len(rows) else 0, "median_rows": float(rows.median()) if len(rows) else 0.0, "min_ess": float(ess.min()) if len(ess) else 0.0, @@ -1600,7 +1617,11 @@ def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: float(sources.median()) if len(sources) else 0.0 ), "bottom_by_rows": [ - {"area_code": str(row.area_code), "rows": int(row.assigned_households)} + { + "area_code": str(row.area_code), + "rows": int(row.nonzero_households), + "assigned": int(row.assigned_households), + } for row in bottom_by_rows.itertuples(index=False) ], "bottom_by_ess": [ From baf1e3a87db79d7c55a64ac8b0bbe041bae372a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:12:28 +0200 Subject: [PATCH 13/14] Round B on spine-k: the K decision surface is invariant across vintages Spine-k (built 2026-08-30, 14/14 gates, same 52,846-row identity, sha b4403ea4) reproduces Round A's support surface bit-for-bit at every K on both grains -- the recert changes live in value columns the seeded assignment never reads. Gate 650/650 + 361/361, mass abs_delta 0.0, lineage identical; second uk/local Logbook row chained onto genesis, the committed archive validates at 2 rows. The K=4 recommendation now stands on both spine vintages; adjudication section awaits Maria's ruling. Co-Authored-By: Claude Fable 5 --- experiments/761-uk-rowwise-pilot-receipts.md | 44 ++++++++++++++++---- logbook/uk/local.jsonl | 1 + 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/experiments/761-uk-rowwise-pilot-receipts.md b/experiments/761-uk-rowwise-pilot-receipts.md index fa685425b0..8cce62c04c 100644 --- a/experiments/761-uk-rowwise-pilot-receipts.md +++ b/experiments/761-uk-rowwise-pilot-receipts.md @@ -119,11 +119,39 @@ survives the spine swap, at the top of the range: H5 and ~15 GB RSS) and adds zero information; hold it unless #762's solve diagnostics show support-limited misses at K=4. -**Adjudication (María):** _pending — rules on the Round B (spine-j) numbers, -this section then records the ruling and #761 carries it._ - -## Round B — spine-j - -_Pending the #804 licensed phase. Re-run R1's dry-run and R2's build with the -re-cut acceptance receipt's sha; support deltas vs Round A should be -region-mix-stable._ +**Adjudication (María):** _pending — Round B below confirms the Round-A +numbers bit-for-bit, so the recommendation stands on both spine vintages; +this section records the ruling and #761 carries it._ + +## Round B — spine-k (2026-08-30) + +The spine line moved past spine-i in the recert lane before any acceptance +re-cut: spine-j (`spine-j-remeasure`) and then **spine-k** +(`data/ukds/acceptance/spine-k-stack/spine-k.h5`, built 2026-08-30, release +id `uk-frs-spine-20260830T120239Z`, **14/14 spine gates passed**, importance +weights, mass total 29,247,433.0, the same 52,846/61,211/113,649 row +identity). Round B ran against spine-k, sha-pinned at the artifact: +`b4403ea4b2d345de06e9f297fdeb9f144519407e98d653b09cf77b16c1e436cb`. The +committed `spine_candidate_acceptance.json` still names spine-i — the +acceptance re-mint is the recert lane's step; this pilot pins bytes, not the +receipt, and records that status honestly. + +Same two commands as Round A (multi-K dry-run at seed 42; real K=1 build, +Logbook row chained onto the genesis digest `a4d2c0b2…`): + +- **Support is bit-identical to Round A at every K on both grains** — every + min/median rows, ESS, and distinct-sources figure in R1's tables + reproduces exactly. Spine-k preserves the household roster, region mix, + and importance-weight allocation, so the seeded two-stage draw reproduces; + the pension-wealth split and sibling recert changes live in value columns + the assignment never reads. This is the strongest possible control: the K + decision surface is invariant across the spine vintages. +- Build receipts: gate passed 650/650 + 361/361, `missing_geography_rows: 0`, + mass `abs_delta: 0.0`, `importance` carried, mass log 6 → 7, lineage + identical (16,288 distinct sources; frs 16,288 / spi 10,000; flags + 20,085 / 26,420 / 270). 11.4 s wall, 1.51 GB peak RSS. +- Receipts: dry-run plan sha `5edc378eeac7e1ab…`, build manifest sha + `ded07a6122a3c36e…`, evidence `…/761-rowwise-pilot/spine-k/`; Logbook row + `15f6b09944f749cd…` (chain tail; `logbook/uk/local.jsonl` validates at 2 + rows). Stats emitted in the post-review `rows_basis: nonzero_households` + shape; `assigned == nonzero` for all 1,011 areas here too. diff --git a/logbook/uk/local.jsonl b/logbook/uk/local.jsonl index 4e4d14a5b8..d6bbe4143b 100644 --- a/logbook/uk/local.jsonl +++ b/logbook/uk/local.jsonl @@ -1 +1,2 @@ {"artifact_location":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/uk-2024-locals-rowwise-pilot-k1.h5","build_id":"uk-local-rowwise-ladder-f100-s42-20260829T002607Z-42440bbd","code_pin":"652d2891670bbfe090874905a52c9860825a24b6","cost_usd":null,"disposition":"iterating","gate_verdicts":{"uk_geography_ladder":{"receipt":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-i/k1/rowwise_build_manifest.json#/rowwise_dataset/gate","verdict":"passed"}},"identity_digest":"9547050e6d16833fba9572a1517c4fad36bbeb0ccdbf4261c435884125c3fb1f","input_pins_digest":"eeb124ca3665982bd41e3ebc2820bfc6dd00c54097d4a55d1ff508f04d89dcf6","phases_reached":["attempt_started","configured","inputs_pinned","cloned","manifest_written"],"pipeline":"uk-local-rowwise","prediction_id":null,"prev_row_digest":null,"row_digest":"a4d2c0b2b356493cefb30aca6aa35ba33cfeff584b6946e09164a1cadff4c2de","rung":"f100","seed":42,"ts":"2026-08-29T00:26:07.897707Z","wall_seconds":10.081080500036478} +{"artifact_location":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-k/k1/uk-2024-locals-rowwise-pilot-k1.h5","build_id":"uk-local-rowwise-ladder-f100-s42-20260830T181120Z-3e7bd722","code_pin":"5422de7a00391e961239bc9258c0994ca144d8ee","cost_usd":null,"disposition":"iterating","gate_verdicts":{"uk_geography_ladder":{"receipt":"local://~/Desktop/PolicyEngine/data/ukds/acceptance/761-rowwise-pilot/spine-k/k1/rowwise_build_manifest.json#/rowwise_dataset/gate","verdict":"passed"}},"identity_digest":"0f474ce0fbfc525a8bffac6e8ca1080bde75aaab7f269ee56660a664533447cc","input_pins_digest":"8d4c069b1350cdda9b0abfd63ab05b9d1bd7294fcc4273dd88434a96f46731c2","phases_reached":["attempt_started","configured","inputs_pinned","cloned","manifest_written"],"pipeline":"uk-local-rowwise","prediction_id":null,"prev_row_digest":"a4d2c0b2b356493cefb30aca6aa35ba33cfeff584b6946e09164a1cadff4c2de","row_digest":"15f6b09944f749cd2954a38daa78f52c9a24a9f61bbb9ab768af5b4133d22fcc","rung":"f100","seed":42,"ts":"2026-08-30T18:11:20.735779Z","wall_seconds":6.7020839168690145} From 09bccdf67c30c26ce1068dc0c71abb1ba7d9defc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:42:14 +0200 Subject: [PATCH 14/14] Disposition the second review round, and record Maria's K=4 ruling Finding 1: _support_summary now requires a rows_basis and every block labels its semantics (assigned_rows for realized draws, expected_rows for the analytics) -- same-named stats keys can no longer be silently compared across bases; pinned in the plan test. Finding 2: the Round-B receipts now claim exactly what the evidence supports -- measured, that spine-k preserves the support inputs and that the assignment surface is value-column-independent; entailed from that, the bit-identical tables; value-dependent questions are named as #762's fit diagnostics. Finding 3: row-count roster vs artifact-bytes identity disambiguated, with the identity_digest stated as re-derived. The adjudication section records Maria's K=4 approval (2026-08-30). Co-Authored-By: Claude Fable 5 --- experiments/761-uk-rowwise-pilot-receipts.md | 35 +++++++++---- .../tests/test_uk_ladder_rowwise_clone.py | 4 ++ tools/build_uk_rowwise_dataset.py | 51 +++++++++++-------- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/experiments/761-uk-rowwise-pilot-receipts.md b/experiments/761-uk-rowwise-pilot-receipts.md index 8cce62c04c..ccfcfb7393 100644 --- a/experiments/761-uk-rowwise-pilot-receipts.md +++ b/experiments/761-uk-rowwise-pilot-receipts.md @@ -119,9 +119,14 @@ survives the spine swap, at the top of the range: H5 and ~15 GB RSS) and adds zero information; hold it unless #762's solve diagnostics show support-limited misses at K=4. -**Adjudication (María):** _pending — Round B below confirms the Round-A -numbers bit-for-bit, so the recommendation stands on both spine vintages; -this section records the ruling and #761 carries it._ +**Adjudication (María, 2026-08-30): K=4 approved** for #762's first +calibrated rowwise candidate, on the measurement above — constituency floor +101 rows / ESS 86.4 / 97 distinct sources, median 287 rows, at ≈640 MB and +≈6 GB solve headroom — confirmed unchanged on spine-k (Round B: support +inputs measured-preserved, tables reproduce exactly). K=10 stays held unless +#762's solve diagnostics show support-limited misses at K=4; the LA-grain +floor (Scilly, City of London) is target-side work at any K. The ruled K +becomes #762's declared `--n-clones`, recorded in that build's manifest. ## Round B — spine-k (2026-08-30) @@ -129,8 +134,10 @@ The spine line moved past spine-i in the recert lane before any acceptance re-cut: spine-j (`spine-j-remeasure`) and then **spine-k** (`data/ukds/acceptance/spine-k-stack/spine-k.h5`, built 2026-08-30, release id `uk-frs-spine-20260830T120239Z`, **14/14 spine gates passed**, importance -weights, mass total 29,247,433.0, the same 52,846/61,211/113,649 row -identity). Round B ran against spine-k, sha-pinned at the artifact: +weights, mass total 29,247,433.0, the same 52,846/61,211/113,649 **row-count +roster** — the artifact bytes differ, and the Logbook `identity_digest` is +re-derived for this run, never carried over). Round B ran against spine-k, +sha-pinned at the artifact: `b4403ea4b2d345de06e9f297fdeb9f144519407e98d653b09cf77b16c1e436cb`. The committed `spine_candidate_acceptance.json` still names spine-i — the acceptance re-mint is the recert lane's step; this pilot pins bytes, not the @@ -141,11 +148,19 @@ Logbook row chained onto the genesis digest `a4d2c0b2…`): - **Support is bit-identical to Round A at every K on both grains** — every min/median rows, ESS, and distinct-sources figure in R1's tables - reproduces exactly. Spine-k preserves the household roster, region mix, - and importance-weight allocation, so the seeded two-stage draw reproduces; - the pension-wealth split and sibling recert changes live in value columns - the assignment never reads. This is the strongest possible control: the K - decision surface is invariant across the spine vintages. + reproduces exactly. Read precisely, this establishes two things and + entails a third. Measured: (a) spine-k **preserves the support inputs** — + household roster, region mix, and the importance-weight vector — which was + not knowable a priori (the recert changes could have moved any of them); + (b) the seeded two-stage draw is **reproducible** and the assignment + surface is **value-column-independent**. Entailed, once (a) holds: the + support tables reproduce bit-for-bit — the metric is a function of those + inputs only, so identity here is arithmetic, not an independent check. + The K adjudication is defined over the support surface (#761 items 6–7), + so it carries to spine-k because its inputs are measured-unchanged. + Anything value-dependent — whether the solve at the ruled K fits spine-k's + revised wealth columns — is invisible to this metric by design and belongs + to #762's fit diagnostics, not this pilot. - Build receipts: gate passed 650/650 + 361/361, `missing_geography_rows: 0`, mass `abs_delta: 0.0`, `importance` carried, mass log 6 → 7, lineage identical (16,288 distinct sources; frs 16,288 / spi 10,000; flags diff --git a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py index ddf1315850..6f7934c744 100644 --- a/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py +++ b/packages/microcosm-build/tests/test_uk_ladder_rowwise_clone.py @@ -530,6 +530,10 @@ def test_driver_ladder_dry_run_matches_real_assignment( ) for area_type in ("constituency", "la"): area_support = expected_support[area_type] + # Same-named stats keys carry their semantics: an expected-mass + # figure must never be silently compared against a realized count. + assert area_support["rows_basis"] == "expected_rows" + assert plan["realized_support"][area_type]["rows_basis"] == "assigned_rows" assert area_support["n_areas"] <= builder.EXPECTED_SUPPORT_BOTTOM_AREAS assert len(area_support["bottom"]) == area_support["n_areas"] assert sum(row["rows"] for row in area_support["bottom"]) == pytest.approx( diff --git a/tools/build_uk_rowwise_dataset.py b/tools/build_uk_rowwise_dataset.py index 16142f6141..6f3bb24827 100644 --- a/tools/build_uk_rowwise_dataset.py +++ b/tools/build_uk_rowwise_dataset.py @@ -899,7 +899,9 @@ def _dry_run_plan( "sampleable areas included" ), **{ - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) for area_type in ("constituency", "la") }, }, @@ -910,7 +912,9 @@ def _dry_run_plan( "constituency count" ), **{ - area_type: _support_summary(collision_free, area_type) + area_type: _support_summary( + collision_free, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, }, @@ -972,16 +976,18 @@ def _crosswalk_candidate_plans( plans.append( { "n_clones": n_clones, - "rows": { - name: rows * n_clones for name, rows in table_rows.items() - }, + "rows": {name: rows * n_clones for name, rows in table_rows.items()}, "output_bytes_estimate": input_bytes * n_clones, "realized_support": { - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) for area_type in ("constituency", "la") }, "collision_free_expected_support": { - area_type: _support_summary(collision_free, area_type) + area_type: _support_summary( + collision_free, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, } @@ -1257,7 +1263,9 @@ def _ladder_dry_run_plan( "ladder areas included" ), **{ - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) for area_type in ("constituency", "la") }, }, @@ -1268,7 +1276,9 @@ def _ladder_dry_run_plan( "population shares within constituency for LA support" ), **{ - area_type: _support_summary(expected, area_type) + area_type: _support_summary( + expected, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, }, @@ -1305,9 +1315,7 @@ def _ladder_planned_assignment( household_for_clone = household.copy() if "source_household_id" not in household_for_clone.columns: - household_for_clone["source_household_id"] = household_for_clone[ - "household_id" - ] + household_for_clone["source_household_id"] = household_for_clone["household_id"] cloned = clone_entity_frame( household_for_clone, id_columns=("household_id",), @@ -1371,16 +1379,18 @@ def _ladder_candidate_plans( plans.append( { "n_clones": n_clones, - "rows": { - name: rows * n_clones for name, rows in table_rows.items() - }, + "rows": {name: rows * n_clones for name, rows in table_rows.items()}, "output_bytes_estimate": input_bytes * n_clones, "realized_support": { - area_type: _support_summary(realized, area_type) + area_type: _support_summary( + realized, area_type, rows_basis="assigned_rows" + ) for area_type in ("constituency", "la") }, "expected_support": { - area_type: _support_summary(expected, area_type) + area_type: _support_summary( + expected, area_type, rows_basis="expected_rows" + ) for area_type in ("constituency", "la") }, "area_support": area_support, @@ -1513,12 +1523,14 @@ def _support_summary( support: pd.DataFrame, area_type: str, *, + rows_basis: str, bottom: int = EXPECTED_SUPPORT_BOTTOM_AREAS, ) -> dict[str, Any]: subset = support[support["area_type"] == area_type] if subset.empty: return { "n_areas": 0, + "rows_basis": rows_basis, "min_rows": 0.0, "median_rows": 0.0, "mean_rows": 0.0, @@ -1532,6 +1544,7 @@ def _support_summary( ).head(bottom) return { "n_areas": int(len(subset)), + "rows_basis": rows_basis, "min_rows": float(values.min()), "median_rows": float(values.median()), "mean_rows": float(values.mean()), @@ -1613,9 +1626,7 @@ def _area_support_stats(support: pd.DataFrame) -> dict[str, Any]: "min_ess": float(ess.min()) if len(ess) else 0.0, "median_ess": float(ess.median()) if len(ess) else 0.0, "min_distinct_sources": int(sources.min()) if len(sources) else 0, - "median_distinct_sources": ( - float(sources.median()) if len(sources) else 0.0 - ), + "median_distinct_sources": (float(sources.median()) if len(sources) else 0.0), "bottom_by_rows": [ { "area_code": str(row.area_code),