diff --git a/README.md b/README.md index 34b552f7..95f8715c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ These flags are grouped into themes: | Theme | Example flags | |---|---| -| Schema / taxonomy | `use_cornerstone_2026_model_schema`, `implement_waste_disaggregation`, `implement_electricity_reallocation` | +| Schema / taxonomy | `implement_waste_disaggregation`, `implement_electricity_reallocation` | | Economic IOT (input-output tables) | `apply_io_year_adjustments`, `iot_before_or_after_redefinition` | | GHG attribution | `use_cornerstone_ghg_model` | | Data vintage | `model_base_year`, `usa_base_io_data_year`, `ipcc_ar_version` | diff --git a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py index 19d12c11..df7e36b7 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py @@ -13,11 +13,11 @@ _weighted_ef, ) from bedrock.publish.model_objects import get_B, get_D, get_L, get_N, get_q +from bedrock.transform.allocation.derived import derive_E_usa from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( electricity_conversion_factors, electricity_mixed_units_enabled, ) -from bedrock.transform.eeio.derived import derive_E_usa from bedrock.transform.eeio.derived_cornerstone import ( derive_cornerstone_Aq_mixed_units, derive_cornerstone_Aq_scaled, diff --git a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/full_trace.py b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/full_trace.py index 03da88cf..fd4d3804 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/full_trace.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/full_trace.py @@ -11,12 +11,12 @@ from bedrock.analysis.electricity_disagg_diagnostics.paths import OUT_DIR from bedrock.publish.model_objects import get_B, get_D, get_L, get_N, get_q +from bedrock.transform.allocation.derived import derive_E_usa from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( derive_disagg_Ytot_with_trade, electricity_conversion_factors, electricity_mixed_units_enabled, ) -from bedrock.transform.eeio.derived import derive_E_usa from bedrock.transform.eeio.derived_cornerstone import ( derive_cornerstone_Aq_scaled, derive_cornerstone_U_set, diff --git a/bedrock/extract/allocation/__tests__/test_bea_use_table_schema.py b/bedrock/extract/allocation/__tests__/test_bea_use_table_schema.py index d7bfe4c2..6d9d911f 100644 --- a/bedrock/extract/allocation/__tests__/test_bea_use_table_schema.py +++ b/bedrock/extract/allocation/__tests__/test_bea_use_table_schema.py @@ -1,64 +1,20 @@ -"""Tests for schema-aligned BEA use table (load_bea_use_table).""" +"""Tests for the Cornerstone-frame BEA use table (load_bea_use_table).""" from __future__ import annotations -from unittest.mock import patch - import pytest -from bedrock.extract.allocation.bea import ( - _load_bea_use_table_cached, - load_bea_use_table, -) -from bedrock.utils.config.usa_config import USAConfig -from bedrock.utils.taxonomy.bea.ceda_v7 import CEDA_V7_SECTORS +from bedrock.extract.allocation.bea import load_bea_use_table from bedrock.utils.taxonomy.cornerstone.industries import INDUSTRIES -def _clear_use_table_cache() -> None: - _load_bea_use_table_cached.cache_clear() - - -@pytest.mark.eeio_integration -def test_load_bea_use_table_ceda_shape() -> None: - """With CEDA config, table has CEDA v7 industry rows + PCE row.""" - _clear_use_table_cache() - config = USAConfig(use_cornerstone_2026_model_schema=False) - with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config): - table = load_bea_use_table() - # Rows = CEDA industries + one PCE row - industry_rows = [i for i in table.index if i in CEDA_V7_SECTORS] - assert len(industry_rows) == len(CEDA_V7_SECTORS) - assert "221200" in table.columns - assert table.shape[0] >= len(CEDA_V7_SECTORS) - assert table.shape[1] > 0 - - @pytest.mark.eeio_integration def test_load_bea_use_table_cornerstone_shape() -> None: - """With Cornerstone config, table has Cornerstone industry rows + PCE row.""" - _clear_use_table_cache() - config = USAConfig(use_cornerstone_2026_model_schema=True) - with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config): - table = load_bea_use_table() + """Table has Cornerstone industry rows + PCE row.""" + load_bea_use_table.cache_clear() + table = load_bea_use_table() industry_rows = [i for i in table.index if i in INDUSTRIES] assert len(industry_rows) == len(INDUSTRIES) assert "221200" in table.columns assert table.shape[0] >= len(INDUSTRIES) assert table.shape[1] > 0 - - -@pytest.mark.eeio_integration -def test_load_bea_use_table_cache_per_schema() -> None: - """CEDA and Cornerstone calls return different shapes (cache keyed by schema).""" - _clear_use_table_cache() - config_ceda = USAConfig(use_cornerstone_2026_model_schema=False) - config_cs = USAConfig(use_cornerstone_2026_model_schema=True) - with patch( - "bedrock.extract.allocation.bea.get_usa_config", return_value=config_ceda - ): - table_ceda = load_bea_use_table() - with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config_cs): - table_cs = load_bea_use_table() - assert table_ceda.shape[0] != table_cs.shape[0] - assert set(table_ceda.index) != set(table_cs.index) diff --git a/bedrock/extract/allocation/bea.py b/bedrock/extract/allocation/bea.py index 9ac73775..af5e397e 100644 --- a/bedrock/extract/allocation/bea.py +++ b/bedrock/extract/allocation/bea.py @@ -6,9 +6,11 @@ import pandas as pd from bedrock.transform.eeio.derived_2017 import ( - derive_2017_U_set_usa, derive_2017_V_usa, - derive_2017_Y_personal_consumption_expenditure_usa, +) +from bedrock.transform.eeio.derived_cornerstone import ( + derive_cornerstone_U_set, + derive_cornerstone_Y_personal_consumption_expenditure, ) from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.io.gcp import load_from_gcs @@ -45,34 +47,16 @@ def load_bea_make_table() -> pd.DataFrame: @functools.cache -def _load_bea_use_table_cached(use_cornerstone: bool) -> pd.DataFrame: - """Inner loader keyed by schema so both CEDA and Cornerstone can be cached.""" - if use_cornerstone: - from bedrock.transform.eeio import derived_cornerstone # noqa: PLC0415 - - uset = derived_cornerstone.derive_cornerstone_U_set() - U_combined = (uset.Udom + uset.Uimp).T - Y_cs = ( - derived_cornerstone.derive_cornerstone_Y_personal_consumption_expenditure() - .to_frame() - .T - ) - return pd.concat([U_combined, Y_cs]) - U_set = derive_2017_U_set_usa() - Y_usa = derive_2017_Y_personal_consumption_expenditure_usa().to_frame() - return pd.concat([(U_set.Udom + U_set.Uimp).T, Y_usa.T]) - - def load_bea_use_table() -> pd.DataFrame: """ - Load BEA Use and Final Demand tables aligned to the model schema. + Load BEA Use and Final Demand tables in the Cornerstone frame. - When use_cornerstone_2026_model_schema is False, returns CEDA v7 industry rows. - When True, returns Cornerstone industry rows (from derive_cornerstone_U_set and Y). - Rows = industries + one PCE row; columns = commodities. Result is cached per schema. + Rows = Cornerstone industries + one PCE row; columns = commodities. """ - use_cornerstone = get_usa_config().use_cornerstone_2026_model_schema - return _load_bea_use_table_cached(use_cornerstone) + uset = derive_cornerstone_U_set() + U_combined = (uset.Udom + uset.Uimp).T + Y_cs = derive_cornerstone_Y_personal_consumption_expenditure().to_frame().T + return pd.concat([U_combined, Y_cs]) def _use_table_value_ceda_sector_cornerstone_aligned( @@ -81,12 +65,11 @@ def _use_table_value_ceda_sector_cornerstone_aligned( ceda_sector: str, ) -> float: """ - Value for one CEDA allocator sector from a use table (CEDA or Cornerstone shaped). + Value for one CEDA-vocabulary allocator sector from the Cornerstone use table. - When the table is CEDA-shaped (Cornerstone schema not active), the sector is - in the index and we return it directly; alignment rules are skipped. When the - table is Cornerstone-shaped, we apply alignment: 562* → 562000, 335220 ↔ 4 - appliance sectors, 331313 → 331313+33131B. + Sectors present in the index are returned directly; otherwise alignment + rules apply: 562* → 562000, 335220 ↔ 4 appliance sectors, + 331313 → 331313+33131B. """ if ceda_sector in table_idx: return float(col.loc[ceda_sector]) @@ -120,11 +103,11 @@ def use_table_series_ceda_allocator_to_cornerstone_schema( commodity: str, ) -> pd.Series: """ - Use-table series for CEDA allocator sectors, aligned to Cornerstone schema. + Use-table series for CEDA-vocabulary allocator sectors, aligned to the + Cornerstone-shaped use table. - When the use table is CEDA-shaped (Cornerstone schema not active), sectors - are looked up directly; alignment is skipped. When Cornerstone-shaped, - alignment applies: 562* → 562000, 335220 ↔ 4 appliance sectors, 331313 → + Sectors present in the table are looked up directly; otherwise alignment + applies: 562* → 562000, 335220 ↔ 4 appliance sectors, 331313 → 331313+33131B. Missing sectors get 0. Safe to normalize (e.g. pct = s / s.sum()). """ table_idx = use_table.index diff --git a/bedrock/publish/emission_factors/writer.py b/bedrock/publish/emission_factors/writer.py index e23baa73..90779b24 100644 --- a/bedrock/publish/emission_factors/writer.py +++ b/bedrock/publish/emission_factors/writer.py @@ -10,7 +10,7 @@ build_purchaser_matrices, finalize_cornerstone_ef_table, ) -from bedrock.publish.model_objects import apply_loc_suffix, require_cornerstone_config +from bedrock.publish.model_objects import apply_loc_suffix logger = logging.getLogger(__name__) @@ -24,7 +24,6 @@ def write_emission_factors( write_matrices: bool = False, ) -> dict[str, str]: """Write CO2e SEF CSV (and optional M/N purchaser matrices) under ``output_dir``.""" - require_cornerstone_config() os.makedirs(output_dir, exist_ok=True) table = finalize_cornerstone_ef_table( diff --git a/bedrock/publish/excel/writer.py b/bedrock/publish/excel/writer.py index b6608359..16b8c62e 100644 --- a/bedrock/publish/excel/writer.py +++ b/bedrock/publish/excel/writer.py @@ -90,7 +90,6 @@ get_Udom, get_V, get_x, - require_cornerstone_config, ) from bedrock.utils.config.settings import GIT_HASH_LONG from bedrock.utils.config.usa_config import get_usa_config @@ -370,11 +369,7 @@ def write_model_to_xlsx(out_path: str, *, config_name: str) -> None: Sheets are produced from `_build_matrix_registry(config_name)`. Any registry entry whose getter returns `None` is omitted from the workbook (useeior-style "skip if NULL"). - - Raises `NotImplementedError` on legacy (non-cornerstone) configs -- - publish is wired only for cornerstone-schema configs today. """ - require_cornerstone_config() registry = _build_matrix_registry(config_name) materialized = _materialize(registry) os.makedirs(os.path.dirname(out_path) or '.', exist_ok=True) diff --git a/bedrock/publish/model_objects.py b/bedrock/publish/model_objects.py index ca382272..36922c6d 100644 --- a/bedrock/publish/model_objects.py +++ b/bedrock/publish/model_objects.py @@ -9,7 +9,6 @@ import pandas as pd -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.math.formulas import compute_L_matrix, compute_M_matrix PUBLISH_LOCATION: str = 'US' @@ -67,14 +66,6 @@ def assemble_extended_U( return out -def require_cornerstone_config() -> None: - if not get_usa_config().use_cornerstone_2026_model_schema: - raise NotImplementedError( - 'bedrock.publish only supports cornerstone-schema configs ' - '(use_cornerstone_2026_model_schema=True).' - ) - - @functools.cache def get_V() -> pd.DataFrame: from bedrock.transform.eeio.derived_cornerstone import derive_cornerstone_V diff --git a/bedrock/transform/allocation/co2/industrial_coal.py b/bedrock/transform/allocation/co2/industrial_coal.py index 7f34d5f4..499b23cf 100644 --- a/bedrock/transform/allocation/co2/industrial_coal.py +++ b/bedrock/transform/allocation/co2/industrial_coal.py @@ -18,12 +18,9 @@ CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, ) from bedrock.transform.allocation.mappings.v7.ceda_mecs import ( - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, NON_MECS_INDUSTRIES, ) from bedrock.transform.allocation.utils import get_allocation_sectors -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.economic.units import COAL_MMBTU_PER_SHORT_TONNE, MEGATONNE_TO_KG load_table_a17_tbtu = functools.cache(_load_table_a17_tbtu) @@ -38,15 +35,10 @@ def _get_mecs_3_1_naics_mappings() -> tuple[ dict[tuple[str, ...], tuple[str, ...]], dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]], ]: - """Return (mapping, subtraction_mapping) for MECS 3.1 NAICS; use CORNERSTONE when schema flag is on.""" - if get_usa_config().use_cornerstone_2026_model_schema: - return ( - CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, - ) + """Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 3.1 NAICS.""" return ( - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, ) diff --git a/bedrock/transform/allocation/co2/industrial_natural_gas.py b/bedrock/transform/allocation/co2/industrial_natural_gas.py index 4e8b59ba..ff28f4a5 100644 --- a/bedrock/transform/allocation/co2/industrial_natural_gas.py +++ b/bedrock/transform/allocation/co2/industrial_natural_gas.py @@ -17,12 +17,9 @@ CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, ) from bedrock.transform.allocation.mappings.v7.ceda_mecs import ( - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, NON_MECS_INDUSTRIES, ) from bedrock.transform.allocation.utils import get_allocation_sectors -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.economic.units import MEGATONNE_TO_KG, NAT_GAS_BCF_TO_TRILLION_BTU load_table_a17_tbtu = functools.cache(_load_table_a17_tbtu) @@ -47,15 +44,10 @@ def _get_mecs_3_1_naics_mappings() -> tuple[ dict[tuple[str, ...], tuple[str, ...]], dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]], ]: - """Return (mapping, subtraction_mapping) for MECS 3.1 NAICS; use CORNERSTONE when schema flag is on.""" - if get_usa_config().use_cornerstone_2026_model_schema: - return ( - CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, - ) + """Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 3.1 NAICS.""" return ( - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING, ) diff --git a/bedrock/transform/allocation/co2/non_energy_fuels_natural_gas.py b/bedrock/transform/allocation/co2/non_energy_fuels_natural_gas.py index 4ff8b1d4..df61a42a 100644 --- a/bedrock/transform/allocation/co2/non_energy_fuels_natural_gas.py +++ b/bedrock/transform/allocation/co2/non_energy_fuels_natural_gas.py @@ -17,12 +17,7 @@ CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, ) -from bedrock.transform.allocation.mappings.v7.ceda_mecs import ( - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, -) from bedrock.transform.allocation.utils import get_allocation_sectors -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.economic.units import MEGATONNE_TO_KG logger = logging.getLogger(__name__) @@ -32,15 +27,10 @@ def _get_mecs_2_1_naics_mappings() -> tuple[ dict[tuple[str, ...], tuple[str, ...]], dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]], ]: - """Return (mapping, subtraction_mapping) for MECS 2.1 NAICS; use CORNERSTONE when schema flag is on.""" - if get_usa_config().use_cornerstone_2026_model_schema: - return ( - CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, - ) + """Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 2.1 NAICS.""" return ( - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, ) diff --git a/bedrock/transform/allocation/co2/non_energy_fuels_petrol.py b/bedrock/transform/allocation/co2/non_energy_fuels_petrol.py index 44e8956f..280ac646 100644 --- a/bedrock/transform/allocation/co2/non_energy_fuels_petrol.py +++ b/bedrock/transform/allocation/co2/non_energy_fuels_petrol.py @@ -17,12 +17,7 @@ CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, ) -from bedrock.transform.allocation.mappings.v7.ceda_mecs import ( - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, -) from bedrock.transform.allocation.utils import get_allocation_sectors -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.economic.units import MEGATONNE_TO_KG logger = logging.getLogger(__name__) @@ -32,15 +27,10 @@ def _get_mecs_2_1_naics_mappings() -> tuple[ dict[tuple[str, ...], tuple[str, ...]], dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]], ]: - """Return (mapping, subtraction_mapping) for MECS 2.1 NAICS; use CORNERSTONE when schema flag is on.""" - if get_usa_config().use_cornerstone_2026_model_schema: - return ( - CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, - ) + """Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 2.1 NAICS.""" return ( - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, - CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING, + CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING, ) @@ -99,7 +89,6 @@ def allocate_non_energy_fuels_petrol() -> pd.Series[float]: use = use_table_series_ceda_allocator_to_cornerstone_schema( load_bea_use_table(), get_allocation_sectors(), "324110" ) - use_cornerstone = get_usa_config().use_cornerstone_2026_model_schema mapping, subtraction_mapping = _get_mecs_2_1_naics_mappings() for ( ceda_industries, @@ -121,15 +110,6 @@ def allocate_non_energy_fuels_petrol() -> pd.Series[float]: for ceda_industry in ceda_industries: industry_use = float(total_use_ser[ceda_industry]) - if not use_cornerstone: - if len(ceda_industries) == 1: - assert ( - industry_use == total_use - ), f"There is only one sector in ceda_industries {ceda_industries}, but use by the industry {industry_use} != total_use ({total_use})" - else: - assert ( - industry_use <= total_use - ), f"There are more than one sector in ceda_industries {ceda_industries}, but use by a child industry {ceda_industry} ({industry_use}) > total_use ({total_use})" if ceda_industry in ("324121", "324122"): # Allocate asphalt and HGL emissions to asphalt industries (324121 and 324122) allocated[ceda_industry] = ( diff --git a/bedrock/transform/allocation/derived.py b/bedrock/transform/allocation/derived.py index d587552a..797ae32e 100644 --- a/bedrock/transform/allocation/derived.py +++ b/bedrock/transform/allocation/derived.py @@ -14,14 +14,10 @@ get_activitytosector_mapping, ) from bedrock.utils.schemas.cornerstone_schemas import CORNERSTONE_INDUSTRIES_ELEC -from bedrock.utils.taxonomy.bea.ceda_v7 import CEDA_V7_SECTORS from bedrock.utils.taxonomy.cornerstone.industries import ( INDUSTRIES, WASTE_DISAGG_INDUSTRIES, ) -from bedrock.utils.taxonomy.mappings.bea_v2017_industry__bea_v2017_commodity import ( - load_bea_v2017_industry_to_bea_v2017_commodity, -) logger = logging.getLogger(__name__) @@ -198,18 +194,12 @@ def map_fbs_sectors_to_model_schema(fbs: pd.DataFrame) -> pd.DataFrame: ) fbs2['NAICS_6'] = fbs2['NAICS_6'].fillna(fbs2['SectorProducedBy']) - if get_usa_config().use_cornerstone_2026_model_schema: - mapping = _build_mapping_with_allocations( - get_activitytosector_mapping('Cornerstone_2025'), - use_output_weights=False, - ) - if get_usa_config().implement_electricity_disaggregation: - mapping = _apply_electricity_disagg_cornerstone_mapping(mapping) - else: - mapping = _build_mapping_with_allocations( - get_activitytosector_mapping('CEDA_2025'), - use_output_weights=False, - ) + mapping = _build_mapping_with_allocations( + get_activitytosector_mapping('Cornerstone_2025'), + use_output_weights=False, + ) + if get_usa_config().implement_electricity_disaggregation: + mapping = _apply_electricity_disagg_cornerstone_mapping(mapping) pre_total = float(fbs2['FlowAmount'].sum()) fbs2 = ( @@ -433,41 +423,12 @@ def load_E_from_flowsa() -> pd.DataFrame: new_index = E_usa.index.map(lambda x: reverse.get(x, x)) E_usa = E_usa.groupby(new_index).agg('sum') - # Collapse across sectors (when CEDA: group BEA→CEDA; when Cornerstone: already in schema) - if get_usa_config().use_cornerstone_2026_model_schema: - if get_usa_config().implement_electricity_disaggregation: - target_columns = [str(sector) for sector in CORNERSTONE_INDUSTRIES_ELEC] - else: - target_columns = [str(sector) for sector in INDUSTRIES] - # E_usa already has Cornerstone columns from derive_E_usa_emissions_sources - E_usa = E_usa.reindex(columns=target_columns, fill_value=0) + # Collapse across sectors (already in Cornerstone schema from + # map_fbs_sectors_to_model_schema). + if usa.implement_electricity_disaggregation: + target_columns = [str(sector) for sector in CORNERSTONE_INDUSTRIES_ELEC] else: - mapping = load_bea_v2017_industry_to_bea_v2017_commodity() - target_columns = [str(sector) for sector in CEDA_V7_SECTORS] - col_to_target = {k: v[0] for k, v in mapping.items()} - for c in E_usa.columns: - if c not in col_to_target and c in target_columns: - col_to_target[c] = c # type: ignore - dropped_by_groupby = sorted(set(E_usa.columns) - set(col_to_target.keys())) - if dropped_by_groupby: - logger.warning( - 'E_usa columns with no mapping (dropped by groupby): %s', - dropped_by_groupby, - ) - E_usa = E_usa.groupby(col_to_target, axis=1).sum() # type: ignore - target_set = set(target_columns) - extra = sorted(set(E_usa.columns) - target_set) - missing = sorted(target_set - set(E_usa.columns)) - if extra: - logger.warning( - 'E_usa columns not in target schema (will be dropped by reindex): %s', - extra, - ) - if missing: - logger.debug( - 'Target schema columns missing from E_usa (will be filled with 0): %s', - missing, - ) - E_usa = E_usa.reindex(columns=target_columns, fill_value=0) + target_columns = [str(sector) for sector in INDUSTRIES] + E_usa = E_usa.reindex(columns=target_columns, fill_value=0) return E_usa diff --git a/bedrock/transform/allocation/utils.py b/bedrock/transform/allocation/utils.py index 0c06f782..a3a9b953 100644 --- a/bedrock/transform/allocation/utils.py +++ b/bedrock/transform/allocation/utils.py @@ -5,21 +5,15 @@ import pandas as pd -from bedrock.utils.config.usa_config import get_usa_config -from bedrock.utils.taxonomy.bea.ceda_v7 import CEDA_V7_SECTORS from bedrock.utils.taxonomy.cornerstone.industries import INDUSTRIES def get_allocation_sectors() -> list[str]: - """ - Return the sector list (taxonomy) for allocation based on model config. + """Return the allocation sector list: Cornerstone INDUSTRIES. - When use_cornerstone_2026_model_schema is True, returns Cornerstone INDUSTRIES; - otherwise returns CEDA v7 sectors. + The Cornerstone frame matches the pinned allocation FBA parquets on GCS. """ - if get_usa_config().use_cornerstone_2026_model_schema: - return list(INDUSTRIES) - return list(CEDA_V7_SECTORS) + return list(INDUSTRIES) def parse_index_with_aggregates( diff --git a/bedrock/transform/eeio/derived.py b/bedrock/transform/eeio/derived.py index 4f3d75da..391e22c0 100644 --- a/bedrock/transform/eeio/derived.py +++ b/bedrock/transform/eeio/derived.py @@ -13,17 +13,6 @@ load_2012_URdom_usa, load_2012_YR_usa, ) -from bedrock.extract.iot.io_2017 import load_summary_Uimp_usa -from bedrock.transform.allocation.derived import derive_E_usa -from bedrock.transform.eeio.derived_2017 import ( - derive_2017_Aq_usa, - derive_2017_U_set_usa, - derive_2017_Vnorm_scrap_corrected, - derive_2017_x_usa, - derive_2017_Ytot_usa_matrix_set, - derive_summary_Yimp_usa, - derive_summary_Ytot_usa_matrix_set, -) from bedrock.transform.eeio.derived_cornerstone import ( derive_cornerstone_Aq_scaled, derive_cornerstone_B_non_finetuned, @@ -32,36 +21,12 @@ derive_cornerstone_y_nab, derive_cornerstone_ydom_and_yimp, ) -from bedrock.transform.eeio.scale_abq_via_summary import ( - scale_detail_A_based_on_summary_A, - scale_detail_B_based_on_summary_q, - scale_detail_q_based_on_summary_q, -) -from bedrock.utils.config.usa_config import get_usa_config -from bedrock.utils.economic.inflation_helpers_ceda import ( - inflate_A_matrix, - inflate_B_matrix, - inflate_q_or_y, -) from bedrock.utils.emissions.characterization import build_ghg_characterization_matrix from bedrock.utils.emissions.ghg import GHG -from bedrock.utils.math.disaggregation import disaggregate_vector -from bedrock.utils.math.formulas import ( - compute_B_ind_matrix, - compute_B_matrix, - compute_y_for_national_accounting_balance, - compute_y_imp, -) from bedrock.utils.math.handle_negatives import handle_negative_vector_values -from bedrock.utils.math.split_using_aggregated_weights import ( - split_vector_using_agg_ratio, -) from bedrock.utils.schemas.single_region_schemas import ( - AMatrix, - BMatrix, ExportsVectorSchema, ImportsVectorSchema, - QVectorSchema, UMatrix, YVectorSchema, ) @@ -75,12 +40,6 @@ USA_2017_FINAL_DEMAND_EXPORT_CODE, USA_2017_FINAL_DEMAND_IMPORT_CODE, ) -from bedrock.utils.taxonomy.bea.v2017_industry_summary import ( - USA_2017_SUMMARY_INDUSTRY_CODES, -) -from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( - get_bea_v2017_summary_to_ceda_corresp_df, -) from bedrock.utils.taxonomy.mappings.ceda_v7__ceda_v5 import CEDA_V5_TO_CEDA_V7_CODES logger = logging.getLogger(__name__) @@ -88,254 +47,31 @@ @functools.cache def derive_B_usa_non_finetuned() -> pd.DataFrame: - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_B_non_finetuned() - else: - E_usa = derive_E_usa() - # B_usa_2017 has 2022 emissions but 2017 economic data - b_usa_2017 = derive_B_usa_via_vnorm(E_usa=E_usa) - # Scale the economic data part of B_usa_2017 to 2022 - B_usa = inflate_B_matrix( - scale_detail_B_based_on_summary_q( - B=b_usa_2017, - original_year=get_usa_config().usa_detail_original_year, - target_year=get_usa_config().usa_io_data_year, - ), - original_year=get_usa_config().usa_io_data_year, - target_year=get_usa_config().model_base_year, - ) - - return pt.DataFrame[BMatrix](BMatrix.validate(B_usa)) + return derive_cornerstone_B_non_finetuned() def derive_Y_and_trade_matrix_usa_from_summary_target_year_ytot_and_structural_reflection() -> ( SingleRegionYtotAndTradeVectorSet ): - """ - We get detail Y and Trade Matrix in the following steps: - - get `target_year` summary Y and Trade Matrix and 2017 detail Y and Trade Matrix - - structurally reflect `target_year` summary Y and Trade Matrix into `target_year` detail Y and Trade Matrix using 2017 detail Y and Trade Matrix - - split the reflected `target_year` detail Y and Trade Matrix into `target_year` detail Y and Trade Matrix, - using weights derived from `target_year` summary Y and Trade Matrix - """ - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_Y_and_trade_scaled() - else: - detail_2017_YandTradeset = derive_2017_Ytot_usa_matrix_set() - - summary_to_ceda_corresp_df = get_bea_v2017_summary_to_ceda_corresp_df() - - summary_Y_matrix_set = derive_summary_Ytot_usa_matrix_set( - get_usa_config().usa_io_data_year - ) - - ytot = inflate_q_or_y( - disaggregate_vector( - base_series=summary_Y_matrix_set.ytot, - weight_series=detail_2017_YandTradeset.ytot, - corresp_df=summary_to_ceda_corresp_df, - ), - original_year=get_usa_config().usa_io_data_year, - target_year=get_usa_config().model_base_year, - ) - exports = inflate_q_or_y( - disaggregate_vector( - base_series=summary_Y_matrix_set.exports, - weight_series=detail_2017_YandTradeset.exports, - corresp_df=summary_to_ceda_corresp_df, - ), - original_year=get_usa_config().usa_io_data_year, - target_year=get_usa_config().model_base_year, - ) - imports = inflate_q_or_y( - handle_negative_vector_values( - disaggregate_vector( - base_series=summary_Y_matrix_set.imports, - weight_series=detail_2017_YandTradeset.imports, - corresp_df=summary_to_ceda_corresp_df, - ) - ), - original_year=get_usa_config().usa_io_data_year, - target_year=get_usa_config().model_base_year, - ) - - return SingleRegionYtotAndTradeVectorSet( - ytot=YVectorSchema.validate(ytot), - exports=ExportsVectorSchema.validate(exports), - imports=ImportsVectorSchema.validate(imports), - ) + """Detail Y and trade vectors, structurally reflected to the target year.""" + return derive_cornerstone_Y_and_trade_scaled() @functools.cache def derive_y_for_national_accounting_balance_usa() -> pd.Series[float]: - """ - We get Y for national accounting balance via the following equations: - - y_nab = y_dom + exports - y_nab = (y_tot - y_imp) + exports - y_nab = (y_tot - (imports - Uimp_row_sum)) + exports - - Because we only have Uimp in 2017 detail, we get 2022 detail y in the - following steps: - - calculate 2017 detail y_nab - - calculate 2022 summary y_nab - - scale 2017 detail y_nab to 2022 detail y_nab using 2022 summary y_nab - """ - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_y_nab() - else: - detail_2017_YandTradeset = derive_2017_Ytot_usa_matrix_set() - - y_national_acct_balance_detail_2017 = compute_y_for_national_accounting_balance( - y_tot=detail_2017_YandTradeset.ytot, - y_imp=compute_y_imp( - imports=detail_2017_YandTradeset.imports, - Uimp=derive_2017_U_set_usa().Uimp, - ), - exports=detail_2017_YandTradeset.exports, - ) - - summary_2022_Y_matrix_set = derive_summary_Ytot_usa_matrix_set( - get_usa_config().usa_io_data_year - ) - - y_national_acct_balance_summary_2022 = ( - compute_y_for_national_accounting_balance( - y_tot=summary_2022_Y_matrix_set.ytot, - y_imp=compute_y_imp( - imports=summary_2022_Y_matrix_set.imports, - Uimp=load_summary_Uimp_usa(get_usa_config().usa_io_data_year).loc[ - USA_2017_SUMMARY_INDUSTRY_CODES, - USA_2017_SUMMARY_INDUSTRY_CODES, - ], - ), - exports=summary_2022_Y_matrix_set.exports, - ) - ) - - summary_to_ceda_corresp_df = get_bea_v2017_summary_to_ceda_corresp_df() - y_national_acct_balance_detail_2022 = inflate_q_or_y( - disaggregate_vector( - corresp_df=summary_to_ceda_corresp_df, - base_series=y_national_acct_balance_summary_2022, - weight_series=y_national_acct_balance_detail_2017, - ), - original_year=get_usa_config().usa_io_data_year, - target_year=get_usa_config().model_base_year, - ) - - # NOTE: original y values have some negative values - # that will distort the scaling process that uses the ytot here in non-US countries. - # We make a data assumption here to set them to 0 in order to make the scaling process valid. - # TODO: this is a temporary solution, we need a y pandera type to enforce this data assumption. - return YVectorSchema.validate( - handle_negative_vector_values(y_national_acct_balance_detail_2022) - ) + """y for the national accounting balance: y_nab = (y_tot - y_imp) + exports.""" + return derive_cornerstone_y_nab() def derive_ydom_and_yimp_usa() -> SingleRegionYVectorSet: - """ - This function is only used in derivation of Y_oecd, where we need y_dom and y_imp separately - to populate the diagonal and off-diagonal elements of Y_oecd. - We get ydom and yimp in the following steps: - 1. Get 2022 detail ytot - 2. Get 2022 summary ydom and yimp - 3. Split 2022 detail ytot to ydom and yimp using 2022 summary ydom and yimp ratios - """ - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_ydom_and_yimp() - else: - # Load summary 2022 ytot and yimp - summary_2022_ytot = derive_summary_Ytot_usa_matrix_set(2022).ytot - summary_2022_yimp = derive_summary_Yimp_usa(2022).sum(axis=1) - # Derive ydom over ytot ratio - # NOTE: in case some yimp values are larger than ytot values, causing ydom to be negative, - # which could be reasonable but we don't want to take negative ydom values. - # We handle this by setting ydom to 0 when yimp is larger than ytot. - summary_2022_ydom_over_ytot_ratio = handle_negative_vector_values( - 1 - (summary_2022_yimp / summary_2022_ytot).fillna(0.0) - ) - - # Derive 2022 detail ytot - detail_2022_ytot = disaggregate_vector( - corresp_df=get_bea_v2017_summary_to_ceda_corresp_df(), - base_series=summary_2022_ytot, - weight_series=derive_2017_Ytot_usa_matrix_set().ytot, - ) - # Split detail 2022 ytot to ydom and yimp - ydom, yimp = split_vector_using_agg_ratio( - base_series=detail_2022_ytot, - agg_ratio_series=summary_2022_ydom_over_ytot_ratio, - corresp_df=get_bea_v2017_summary_to_ceda_corresp_df(), - ) - return SingleRegionYVectorSet( - ydom=YVectorSchema.validate(ydom), yimp=YVectorSchema.validate(yimp) - ) + """ydom and yimp split, used to populate diagonal/off-diagonal Y_oecd.""" + return derive_cornerstone_ydom_and_yimp() @functools.cache def derive_Aq_usa() -> SingleRegionAqMatrixSet: - """ - This function derives Aq_usa in `target_year` USD. - - For Adom and Aimp, we get `target_year` detail matrices by - year-scaling 2017 detail Adom and Aimp separately using the - `target_year` summary Adom and Aimp, respectively - - For q, we get 2017 detail q from 2017 detail IOTs, then inflate it to `target_year` USD. - """ - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_Aq_scaled() - else: - detail_2017_Aq_set = derive_2017_Aq_usa() - - target_year = get_usa_config().usa_io_data_year - original_year = get_usa_config().usa_detail_original_year - - Adom = scale_detail_A_based_on_summary_A( - A=detail_2017_Aq_set.Adom, - target_year=target_year, - original_year=original_year, - dom_or_imp_or_total='dom', - ) - Aimp = scale_detail_A_based_on_summary_A( - A=detail_2017_Aq_set.Aimp, - target_year=target_year, - original_year=original_year, - dom_or_imp_or_total='imp', - ) - - q = scale_detail_q_based_on_summary_q( - q=detail_2017_Aq_set.scaled_q, - target_year=target_year, - original_year=original_year, - ) - assert q is not None, 'q in derive_Aq_usa() is None' - - # NOTE: the Adom/Aimp/q being passed in are already in `target_year`. - # We just need to inflate them to CEDA base year. - # TODO: type inflate_A_matrix as DataFrame[AMatrix] - Adom = inflate_A_matrix( # type: ignore[assignment] - Adom, - target_year=get_usa_config().model_base_year, - original_year=target_year, - ) - Aimp = inflate_A_matrix( # type: ignore[assignment] - Aimp, - target_year=get_usa_config().model_base_year, - original_year=target_year, - ) - q = inflate_q_or_y( - q, - target_year=get_usa_config().model_base_year, - original_year=target_year, - ) - - return SingleRegionAqMatrixSet( - Adom=pt.DataFrame[AMatrix](Adom), - Aimp=pt.DataFrame[AMatrix](Aimp), - scaled_q=QVectorSchema.validate(q), - ) + """Year-scaled and inflated A matrices and q (model_base_year USD).""" + return derive_cornerstone_Aq_scaled() @functools.cache @@ -362,18 +98,6 @@ def derive_D_usa() -> pd.DataFrame: return D -def derive_B_usa_via_vnorm(*, E_usa: pd.DataFrame) -> pd.DataFrame: - x = derive_2017_x_usa() - Vnorm = derive_2017_Vnorm_scrap_corrected() - - Bi = compute_B_ind_matrix(E=E_usa, x=x) - Bc = compute_B_matrix(B_ind=Bi, V_norm=Vnorm) - - Bc.columns.name = 'sector' - Bc.index.name = 'ghg' - return Bc - - def derive_v5_U_usa() -> SingleRegionUMatrixSet: URtot_usa = load_2012_UR_usa() URdom_usa = load_2012_URdom_usa() @@ -434,40 +158,5 @@ def derive_v5_detail_Ytot_usa_matrix_set() -> SingleRegionYtotAndTradeVectorSet: @functools.cache def derive_v7_detail_Ytot_usa_matrix_set() -> SingleRegionYtotAndTradeVectorSet: - """ - Derive US Ytot and trade vectors using v7 USA IO tables. - - NOTE: Ytot_usa can't be negative, because we need to use it to ABSR Ytot of other countries. - """ - if get_usa_config().use_cornerstone_2026_model_schema: - return derive_cornerstone_detail_Ytot_matrix_set() - else: - detail_2017_YandTradeset = derive_2017_Ytot_usa_matrix_set() - - summary_to_ceda_corresp_df = get_bea_v2017_summary_to_ceda_corresp_df() - - summary_Y_matrix_set = derive_summary_Ytot_usa_matrix_set(year=2022) - - ytot = disaggregate_vector( - base_series=summary_Y_matrix_set.ytot, - weight_series=detail_2017_YandTradeset.ytot, - corresp_df=summary_to_ceda_corresp_df, - ) - exports = disaggregate_vector( - base_series=summary_Y_matrix_set.exports, - weight_series=detail_2017_YandTradeset.exports, - corresp_df=summary_to_ceda_corresp_df, - ) - imports = handle_negative_vector_values( - disaggregate_vector( - base_series=summary_Y_matrix_set.imports, - weight_series=detail_2017_YandTradeset.imports, - corresp_df=summary_to_ceda_corresp_df, - ) - ) - - return SingleRegionYtotAndTradeVectorSet( - ytot=YVectorSchema.validate(ytot), - exports=ExportsVectorSchema.validate(exports), - imports=ImportsVectorSchema.validate(imports), - ) + """US Ytot and trade vectors (non-negative ytot, used to ABSR other countries).""" + return derive_cornerstone_detail_Ytot_matrix_set() diff --git a/bedrock/transform/eeio/nowcast.py b/bedrock/transform/eeio/nowcast.py index ab0f4baa..b0a3bb3b 100644 --- a/bedrock/transform/eeio/nowcast.py +++ b/bedrock/transform/eeio/nowcast.py @@ -35,14 +35,11 @@ from __future__ import annotations import functools -from collections.abc import Generator -from contextlib import contextmanager import pandas as pd from bedrock.transform.allocation.derived import map_fbs_sectors_to_model_schema from bedrock.transform.flowbysector import FlowBySector -from bedrock.utils.config.usa_config import get_usa_config from bedrock.utils.taxonomy.bea.v2017_final_demand import BEA_2017_FINAL_DEMAND_CODES _SECTOR_SWAP = { @@ -51,33 +48,13 @@ } -@contextmanager -def _force_cornerstone_model_schema() -> Generator[None, None, None]: - """ - ``map_fbs_sectors_to_model_schema`` branches on - ``get_usa_config().use_cornerstone_2026_model_schema`` (default False) to - pick the Cornerstone_2025 vs. CEDA_2025 crosswalk. Temporarily forces it - True on the live global config (rather than swapping to a whole different - named config via ``temp_usa_config``, which would also reset unrelated - fields like ``model_base_year``) and restores the prior value after. - """ - cfg = get_usa_config() - previous = cfg.use_cornerstone_2026_model_schema - cfg.use_cornerstone_2026_model_schema = True - try: - yield - finally: - cfg.use_cornerstone_2026_model_schema = previous - - def _resolve_both_sector_columns(fbs: pd.DataFrame) -> pd.DataFrame: """Apply ``map_fbs_sectors_to_model_schema`` to both SectorProducedBy and SectorConsumedBy (via a temporary swap - see module docstring).""" - with _force_cornerstone_model_schema(): - fbs = map_fbs_sectors_to_model_schema(fbs.rename(columns=_SECTOR_SWAP)).rename( - columns=_SECTOR_SWAP - ) - return map_fbs_sectors_to_model_schema(fbs) + fbs = map_fbs_sectors_to_model_schema(fbs.rename(columns=_SECTOR_SWAP)).rename( + columns=_SECTOR_SWAP + ) + return map_fbs_sectors_to_model_schema(fbs) @functools.cache diff --git a/bedrock/transform/eeio/scale_abq_via_summary.py b/bedrock/transform/eeio/scale_abq_via_summary.py deleted file mode 100644 index b0f8bd62..00000000 --- a/bedrock/transform/eeio/scale_abq_via_summary.py +++ /dev/null @@ -1,141 +0,0 @@ -from __future__ import annotations - -import typing as ta - -import numpy as np -import pandas as pd -import pandera.pandas as pa -import pandera.typing as pt - -from bedrock.transform.eeio.derived_2017 import ( - derive_summary_Adom_usa, - derive_summary_Aimp_usa, - derive_summary_q_usa, -) -from bedrock.utils.math.formulas import compute_total_industry_inputs -from bedrock.utils.schemas.single_region_schemas import AMatrix -from bedrock.utils.taxonomy.bea.ceda_v7 import CEDA_V7_SECTORS -from bedrock.utils.taxonomy.bea.matrix_mappings import USA_SUMMARY_MUT_YEARS -from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( - load_bea_v2017_summary_to_ceda_v7, -) - - -@pa.check_output(AMatrix.to_schema()) -def scale_detail_A_based_on_summary_A( - A: pd.DataFrame, - target_year: USA_SUMMARY_MUT_YEARS, - original_year: USA_SUMMARY_MUT_YEARS, - dom_or_imp_or_total: ta.Literal["dom", "imp", "total"], -) -> pt.DataFrame[AMatrix]: - """ - Derive A_ratio from the summary tables and scale A element-wise based on A_ratio. - """ - - match dom_or_imp_or_total: - case "dom": - A_summary_base = derive_summary_Adom_usa(original_year) - A_summary_target = derive_summary_Adom_usa(target_year) - case "imp": - A_summary_base = derive_summary_Aimp_usa(original_year) - A_summary_target = derive_summary_Aimp_usa(target_year) - case "total": - A_summary_base = derive_summary_Adom_usa( - original_year - ) + derive_summary_Aimp_usa(original_year) - A_summary_target = derive_summary_Adom_usa( - target_year - ) + derive_summary_Aimp_usa(target_year) - - summary_to_ceda_v7 = load_bea_v2017_summary_to_ceda_v7() - summary_ratios = (A_summary_target / A_summary_base).fillna(1.0) - summary_ratios[np.isinf(summary_ratios)] = 1.0 - - A_scaled = A.copy() - - block_rows = [] - for i, row in summary_ratios.iterrows(): - if i not in summary_to_ceda_v7: - continue - block_mat = pd.DataFrame( - index=summary_to_ceda_v7[i], # type: ignore - columns=CEDA_V7_SECTORS, - data=0, - dtype=float, - ) - for col_summary_sector, val in row.items(): - if val == 0: - continue - if col_summary_sector in ["Used", "Other"]: - continue - col_ceda_sectors = summary_to_ceda_v7[col_summary_sector] # type: ignore - block_mat.loc[:, col_ceda_sectors] = val - block_rows.append(block_mat) - ratio_multiplier = pd.concat(block_rows, axis=0).loc[ - A_scaled.index, A_scaled.columns - ] - A_scaled_by_A_summary = A_scaled * ratio_multiplier - - # Adjust A if column sum exceeds 1, force the column sum to be 0.98 - A_scaled_by_A_summary = _rescale_A_matrix_by_capping_total_industry_inputs( - A=A_scaled_by_A_summary, max_total_industry_inputs=0.98 - ) - - assert ( - compute_total_industry_inputs(A=A_scaled_by_A_summary) <= 1 - ).all(), "A column sums exceed 1 after scaled via A_ratio." - - return pt.DataFrame[AMatrix](A_scaled_by_A_summary) - - -def scale_detail_q_based_on_summary_q( - q: pd.Series[float], - target_year: USA_SUMMARY_MUT_YEARS, - original_year: USA_SUMMARY_MUT_YEARS, -) -> pd.Series[float]: - q_scaled = q.copy() - q_summary_base = derive_summary_q_usa(original_year) - q_summary_target = derive_summary_q_usa(target_year) - q_ratio = (q_summary_target / q_summary_base).fillna(1.0) - - summary_to_ceda_v7 = load_bea_v2017_summary_to_ceda_v7() - for i, val in q_ratio.items(): - if i not in summary_to_ceda_v7: - continue - ceda_sectors = summary_to_ceda_v7[i] # type: ignore - q_scaled.loc[ceda_sectors] *= val - - return q_scaled - - -def scale_detail_B_based_on_summary_q( - B: pd.DataFrame, - target_year: USA_SUMMARY_MUT_YEARS, - original_year: USA_SUMMARY_MUT_YEARS, -) -> pd.DataFrame: - B_scaled = B.copy() - q_summary_base = derive_summary_q_usa(original_year) - q_summary_target = derive_summary_q_usa(target_year) - # b_ratio is the opposite of q_ratio because - # the price value for B is in the denominator - b_ratio = (q_summary_base / q_summary_target).fillna(1.0) - - summary_to_ceda_v7 = load_bea_v2017_summary_to_ceda_v7() - for i, val in b_ratio.items(): - if i not in summary_to_ceda_v7: - continue - ceda_sectors = summary_to_ceda_v7[i] # type: ignore - B_scaled.loc[:, ceda_sectors] *= val - - return B_scaled - - -def _rescale_A_matrix_by_capping_total_industry_inputs( - *, A: pd.DataFrame, max_total_industry_inputs: float # noqa: ARG001 -) -> pd.DataFrame: - A_rescaled = A.copy() - total_industry_inputs = compute_total_industry_inputs(A=A) - oob_idx = total_industry_inputs[total_industry_inputs > 1].index - for col in oob_idx: - A_rescaled[col] *= 0.98 / total_industry_inputs[col] - return A_rescaled diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_full_model_electricity_disaggregation.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_full_model_electricity_disaggregation.yaml index a80a07a3..76ca203b 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_full_model_electricity_disaggregation.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_full_model_electricity_disaggregation.yaml @@ -1,5 +1,4 @@ # Fully implemented Cornerstone 2025 model + waste + electricity reallocation + disaggregation. -use_cornerstone_2026_model_schema: True use_cornerstone_ghg_model: true use_E_data_year_for_x_in_B: True implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy_and_B_transformation.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy_and_B_transformation.yaml index 3dab08f2..054d461c 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy_and_B_transformation.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy_and_B_transformation.yaml @@ -11,7 +11,6 @@ ##### # Methodology selection ##### -use_cornerstone_2026_model_schema: True use_E_data_year_for_x_in_B: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_disaggregation.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_disaggregation.yaml index 283b53dc..75cf4395 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_disaggregation.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_disaggregation.yaml @@ -1,5 +1,4 @@ # v0.2 + waste + electricity reallocation + 3-way split (no mixed units, no margins). -use_cornerstone_2026_model_schema: True use_cornerstone_ghg_model: true use_E_data_year_for_x_in_B: True implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_mixed_units.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_mixed_units.yaml index 840b6eab..2ca190f8 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_mixed_units.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_mixed_units.yaml @@ -1,5 +1,4 @@ # v0.2 electricity-disagg config + mixed units (221110 in MWh). -use_cornerstone_2026_model_schema: True use_cornerstone_ghg_model: true use_E_data_year_for_x_in_B: True implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_reallocation.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_reallocation.yaml index 0abaab18..87c76dc3 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_reallocation.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_electricity_reallocation.yaml @@ -1,5 +1,4 @@ # Fully implemented Cornerstone 2025 model + waste disagg + electricity co-production reallocation. -use_cornerstone_2026_model_schema: True use_cornerstone_ghg_model: true use_E_data_year_for_x_in_B: True implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_3.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_3.yaml index 2d44ac50..2775a23e 100644 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_3.yaml +++ b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_3.yaml @@ -24,7 +24,6 @@ usa_ghg_data_year: 2024 ##### # Methodology selection ##### -use_cornerstone_2026_model_schema: True # GHG model: Cornerstone GHG FBS (2024 = UMD GHGIA inventory + 2022 MECS attribution). use_cornerstone_ghg_model: True diff --git a/bedrock/utils/config/configs/test_usa_config_waste_disagg.yaml b/bedrock/utils/config/configs/test_usa_config_waste_disagg.yaml index 7ced4da3..de38b204 100644 --- a/bedrock/utils/config/configs/test_usa_config_waste_disagg.yaml +++ b/bedrock/utils/config/configs/test_usa_config_waste_disagg.yaml @@ -1,4 +1,3 @@ model_base_year: 2023 usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: True implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity.yaml b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity.yaml index 61bf5589..ccbc84eb 100644 --- a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity.yaml +++ b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity.yaml @@ -1,5 +1,4 @@ model_base_year: 2023 usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: True implement_waste_disaggregation: True implement_electricity_reallocation: True diff --git a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_disaggregation.yaml b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_disaggregation.yaml index 2dfbfb3f..18281052 100644 --- a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_disaggregation.yaml +++ b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_disaggregation.yaml @@ -1,6 +1,5 @@ model_base_year: 2023 usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: True implement_waste_disaggregation: True implement_electricity_reallocation: True implement_electricity_disaggregation: True diff --git a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_mixed_units.yaml b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_mixed_units.yaml index dc01945b..13a21496 100644 --- a/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_mixed_units.yaml +++ b/bedrock/utils/config/configs/test_usa_config_waste_disagg_electricity_mixed_units.yaml @@ -1,6 +1,5 @@ model_base_year: 2023 usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: True implement_waste_disaggregation: True implement_electricity_reallocation: True implement_electricity_disaggregation: True diff --git a/bedrock/utils/config/configs/useeio_phoebe_23.yaml b/bedrock/utils/config/configs/useeio_phoebe_23.yaml index d2ae2a7e..aa1cc93f 100644 --- a/bedrock/utils/config/configs/useeio_phoebe_23.yaml +++ b/bedrock/utils/config/configs/useeio_phoebe_23.yaml @@ -16,7 +16,6 @@ usa_ghg_data_year: 2023 ##### # Methodology selection ##### -use_cornerstone_2026_model_schema: true use_E_data_year_for_x_in_B: true implement_waste_disaggregation: true diff --git a/bedrock/utils/config/configs/v03_waterfall_ceda_g1a_schema_ghg.yaml b/bedrock/utils/config/configs/v03_waterfall_ceda_g1a_schema_ghg.yaml index 5be21c1b..82415403 100644 --- a/bedrock/utils/config/configs/v03_waterfall_ceda_g1a_schema_ghg.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_ceda_g1a_schema_ghg.yaml @@ -7,7 +7,6 @@ price_type: producer usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: true use_cornerstone_ghg_model: true apply_io_year_adjustments: false implement_waste_disaggregation: false diff --git a/bedrock/utils/config/configs/v03_waterfall_ceda_g1b_waste_disagg.yaml b/bedrock/utils/config/configs/v03_waterfall_ceda_g1b_waste_disagg.yaml index 67a4d90c..5af3937d 100644 --- a/bedrock/utils/config/configs/v03_waterfall_ceda_g1b_waste_disagg.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_ceda_g1b_waste_disagg.yaml @@ -6,7 +6,6 @@ price_type: producer usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: true use_cornerstone_ghg_model: true apply_io_year_adjustments: false implement_waste_disaggregation: true diff --git a/bedrock/utils/config/configs/v03_waterfall_final.yaml b/bedrock/utils/config/configs/v03_waterfall_final.yaml index a476fec5..2c9bbf24 100644 --- a/bedrock/utils/config/configs/v03_waterfall_final.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_final.yaml @@ -6,7 +6,6 @@ price_type: producer usa_ghg_data_year: 2024 -use_cornerstone_2026_model_schema: true use_cornerstone_ghg_model: true diff --git a/bedrock/utils/config/configs/v03_waterfall_g2_methods.yaml b/bedrock/utils/config/configs/v03_waterfall_g2_methods.yaml index c72272fb..bae5e1d7 100644 --- a/bedrock/utils/config/configs/v03_waterfall_g2_methods.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_g2_methods.yaml @@ -6,7 +6,6 @@ price_type: producer usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: true use_cornerstone_ghg_model: true apply_io_year_adjustments: true implement_waste_disaggregation: true diff --git a/bedrock/utils/config/configs/v03_waterfall_g3_data.yaml b/bedrock/utils/config/configs/v03_waterfall_g3_data.yaml index 5a5ae0d5..e7b0f706 100644 --- a/bedrock/utils/config/configs/v03_waterfall_g3_data.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_g3_data.yaml @@ -6,7 +6,6 @@ price_type: producer usa_ghg_data_year: 2024 -use_cornerstone_2026_model_schema: true use_cornerstone_ghg_model: true apply_io_year_adjustments: true implement_waste_disaggregation: true diff --git a/bedrock/utils/config/configs/v03_waterfall_useeio_g1_schema_ghg.yaml b/bedrock/utils/config/configs/v03_waterfall_useeio_g1_schema_ghg.yaml index 5314e2b9..b5cfee48 100644 --- a/bedrock/utils/config/configs/v03_waterfall_useeio_g1_schema_ghg.yaml +++ b/bedrock/utils/config/configs/v03_waterfall_useeio_g1_schema_ghg.yaml @@ -7,7 +7,6 @@ iot_before_or_after_redefinition: before usa_ghg_data_year: 2023 -use_cornerstone_2026_model_schema: true use_E_data_year_for_x_in_B: true implement_waste_disaggregation: true diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index a788d2b2..2c9ed350 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -61,7 +61,6 @@ class USAConfig(BaseModel): # Methodology selection ##### ### Schema/Taxonomy selection - use_cornerstone_2026_model_schema: bool = False # DRI: mo.li use_useeio_schema: bool = False ### IO Methodology selection # "IO year adjustments" bucket: CEDA A/q scaling to usa_io_data_year with diff --git a/bedrock/utils/economic/inflation_helpers_ceda.py b/bedrock/utils/economic/inflation_helpers_ceda.py index bf04923d..7d17f734 100644 --- a/bedrock/utils/economic/inflation_helpers_ceda.py +++ b/bedrock/utils/economic/inflation_helpers_ceda.py @@ -3,7 +3,6 @@ import functools import os -import numpy as np import pandas as pd from bedrock.utils.io.gcp import download_gcs_file_if_not_exists @@ -52,34 +51,3 @@ def obtain_inflation_factors_from_reference_data() -> pd.DataFrame: get_price_index = functools.cache( lambda: obtain_inflation_factors_from_reference_data() ) - - -def inflate_A_matrix( - A: pd.DataFrame, original_year: int, target_year: int -) -> pd.DataFrame: - price_index = get_price_index() - - price_ratio = price_index[target_year] / price_index[original_year] - return pd.DataFrame( - (np.diag(price_ratio) @ A @ np.diag(1 / price_ratio)).values, - index=A.index, - columns=A.columns, - ) - - -def inflate_B_matrix( - B: pd.DataFrame, original_year: int, target_year: int -) -> pd.DataFrame: - price_index = get_price_index() - - price_ratio = price_index[original_year] / price_index[target_year] - return B * price_ratio.loc[B.columns].values - - -def inflate_q_or_y( - q_or_y: pd.Series[float], original_year: int, target_year: int -) -> pd.Series[float]: - price_index = get_price_index() - - price_ratio = price_index[target_year] / price_index[original_year] - return q_or_y * price_ratio.loc[q_or_y.index] diff --git a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py index 41d8141a..df90f9f7 100644 --- a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py @@ -69,7 +69,6 @@ def mock_load(name: str) -> pd.DataFrame: ] mock_config = MagicMock() - mock_config.use_cornerstone_2026_model_schema = False mock_config.use_ghg_year_x_in_B = False mock_config.diagnostics_baseline_source = "gcs_snapshot" @@ -98,6 +97,14 @@ def mock_load(name: str) -> pd.DataFrame: "bedrock.utils.validation.calculate_ef_diagnostics.SIGNIFICANT_SECTORS", mock_significant_sectors, ), + patch( + "bedrock.utils.validation.calculate_ef_diagnostics.align_efs_across_schemas", + return_value=(efs, {}), + ), + patch( + "bedrock.utils.validation.calculate_ef_diagnostics.get_aligned_sector_desc", + return_value={s: s for s in SECTORS}, + ), ): calculate_ef_diagnostics(sheet_id="test_sheet") @@ -114,6 +121,7 @@ def test_tab_names_are_correct(self) -> None: "D_and_diffs", "D_and_N_significant_sectors", "N_and_D_summary_stats", + "sector_mapping_notes", "output_contrib_new_vs_old", ] @@ -172,7 +180,6 @@ def mock_load(name: str) -> pd.DataFrame: mock_significant_sectors = [{"sector": "1111A0"}, {"sector": "221110"}] mock_config = MagicMock() - mock_config.use_cornerstone_2026_model_schema = True mock_config.use_E_data_year_for_x_in_B = False mock_config.diagnostics_baseline_source = "gcs_snapshot" mock_config.usa_ghg_data_year = 2023 @@ -254,7 +261,7 @@ def mock_load(name: str) -> pd.DataFrame: def test_output_contribution_has_expected_columns(self) -> None: tabs = self._run_ef_diagnostics() - oc = tabs[4][2] + oc = tabs[5][2] expected_cols = { "EF_sector", diff --git a/bedrock/utils/validation/__tests__/test_diagnostics_helpers.py b/bedrock/utils/validation/__tests__/test_diagnostics_helpers.py index b9f582b8..570a7a89 100644 --- a/bedrock/utils/validation/__tests__/test_diagnostics_helpers.py +++ b/bedrock/utils/validation/__tests__/test_diagnostics_helpers.py @@ -21,7 +21,6 @@ class TestDNNewInflatedEligibility: def test_emit_when_deflate_path(self) -> None: cfg = USAConfig( - use_cornerstone_2026_model_schema=True, use_E_data_year_for_x_in_B=True, deflate_x_to_detail_io_year_for_B=True, ) @@ -31,7 +30,6 @@ def test_emit_when_deflate_path(self) -> None: def test_skip_when_model_base_year_equals_detail_original_year(self) -> None: cfg = USAConfig( - use_cornerstone_2026_model_schema=True, use_E_data_year_for_x_in_B=True, deflate_x_to_detail_io_year_for_B=True, model_base_year=2017, @@ -42,7 +40,6 @@ def test_skip_when_model_base_year_equals_detail_original_year(self) -> None: def test_skip_use_e_without_deflate(self) -> None: cfg = USAConfig( - use_cornerstone_2026_model_schema=True, use_E_data_year_for_x_in_B=True, deflate_x_to_detail_io_year_for_B=False, ) @@ -52,7 +49,6 @@ def test_skip_use_e_without_deflate(self) -> None: def test_skip_default_b_inflation_path(self) -> None: cfg = USAConfig( - use_cornerstone_2026_model_schema=True, deflate_x_to_detail_io_year_for_B=False, use_E_data_year_for_x_in_B=False, ) @@ -60,16 +56,6 @@ def test_skip_default_b_inflation_path(self) -> None: assert ok is False assert 'double-apply' in reason - def test_skip_legacy_non_cornerstone(self) -> None: - cfg = USAConfig( - use_cornerstone_2026_model_schema=False, - use_E_data_year_for_x_in_B=True, - deflate_x_to_detail_io_year_for_B=True, - ) - ok, reason = d_n_new_inflated_eligibility(cfg) - assert ok is False - assert 'legacy' in reason.lower() - SECTORS = ["1111A0", "1111B0", "221100"] diff --git a/bedrock/utils/validation/calculate_ef_diagnostics.py b/bedrock/utils/validation/calculate_ef_diagnostics.py index 0a7c0fa8..67c7ee0b 100644 --- a/bedrock/utils/validation/calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/calculate_ef_diagnostics.py @@ -152,13 +152,12 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: - N_and_D_summary_stats: Summary statistics of percent diffs. - output_contrib_new_vs_old: Top N contributing sectors to each EF's change, derived from the output contribution matrix. - - sector_mapping_notes (cornerstone only): Documents mapped/excluded sectors. + - sector_mapping_notes: Documents mapped/excluded sectors. Old EFs are inflation-adjusted to the current base year before comparison. - When ``use_cornerstone_2026_model_schema`` is active, old (CEDA v7) and new - (cornerstone) EF vectors are aligned before comparison so that sectors with - different granularity are still comparable. + Old (CEDA v7) and new (Cornerstone) EF vectors are aligned before comparison + so that sectors with different granularity are still comparable. Args: sheet_id: Google Sheets spreadsheet ID to write results to. @@ -175,7 +174,6 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: from bedrock.utils.validation.diagnostics_helpers import pull_efs_for_diagnostics config = get_usa_config() - use_cornerstone = config.use_cornerstone_2026_model_schema t0 = time.time() efs_raw = pull_efs_for_diagnostics() @@ -183,15 +181,10 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: f'[TIMING] pull_efs_for_diagnostics completed in {time.time() - t0:.1f}s' ) - # When the cornerstone schema is active, align old/new sector indices - active_mappings: ta.Dict[str, str] = {} - if use_cornerstone: - logger.info('Aligning EF vectors across CEDA v7 / cornerstone schemas') - efs, active_mappings = align_efs_across_schemas(efs_raw) - sector_desc: ta.Optional[ta.Dict[str, str]] = get_aligned_sector_desc() - else: - efs = efs_raw - sector_desc = None # use default CEDA_V7_SECTOR_DESC + # Align old (CEDA v7) / new (Cornerstone) sector indices before comparison. + logger.info('Aligning EF vectors across CEDA v7 / cornerstone schemas') + efs, active_mappings = align_efs_across_schemas(efs_raw) + sector_desc: ta.Optional[ta.Dict[str, str]] = get_aligned_sector_desc() logger.info('------ Calculating EF Diagnostics ------') @@ -211,9 +204,8 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: sector_desc=sector_desc, ) - if use_cornerstone: - _add_comparison_type_column(N_comparison, active_mappings) - _add_comparison_type_column(D_comparison, active_mappings) + _add_comparison_type_column(N_comparison, active_mappings) + _add_comparison_type_column(D_comparison, active_mappings) if efs.D_new_inflated is not None: assert efs.N_new_inflated is not None @@ -352,9 +344,7 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: # When aligned, some significant sectors may not be in the index (e.g. if # they were removed). Filter to those present. available_significant = [s for s in significant_sectors if s in D_comparison.index] - drop_cols = ['sector_name'] - if use_cornerstone: - drop_cols.append('comparison_type') + drop_cols = ['sector_name', 'comparison_type'] d_sig = D_comparison.loc[available_significant] n_sig = N_comparison.loc[available_significant].drop( columns=drop_cols, errors='ignore' @@ -408,17 +398,14 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: f'[TIMING] Write N_and_D_summary_stats to Google Sheets in {time.time() - t0:.1f}s' ) - # Sector mapping notes (cornerstone only) - if use_cornerstone: - mapping_notes = _build_sector_mapping_notes( - active_mappings, - old_ef=efs_raw.D_old.raw, - new_ef=efs_raw.D_new, - ) - update_sheet_tab( - sheet_id, 'sector_mapping_notes', mapping_notes, clean_nans=True - ) - logger.info('Wrote sector_mapping_notes tab') + # Sector mapping notes + mapping_notes = _build_sector_mapping_notes( + active_mappings, + old_ef=efs_raw.D_old.raw, + new_ef=efs_raw.D_new, + ) + update_sheet_tab(sheet_id, 'sector_mapping_notes', mapping_notes, clean_nans=True) + logger.info('Wrote sector_mapping_notes tab') # Compare output contribution (parquet baseline only; omitted for gcs_useeio_xlsx) if config.diagnostics_baseline_source != 'gcs_useeio_xlsx': @@ -449,11 +436,10 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: L=L_old, D=ta.cast('pd.Series[float]', efs_raw.D_old.inflated.squeeze()) ) - if use_cornerstone: - full_idx = OC_new.index.union(OC_old.index).sort_values() - full_cols = OC_new.columns.union(OC_old.columns).sort_values() - OC_new = OC_new.reindex(index=full_idx, columns=full_cols, fill_value=0.0) - OC_old = OC_old.reindex(index=full_idx, columns=full_cols, fill_value=0.0) + full_idx = OC_new.index.union(OC_old.index).sort_values() + full_cols = OC_new.columns.union(OC_old.columns).sort_values() + OC_new = OC_new.reindex(index=full_idx, columns=full_cols, fill_value=0.0) + OC_old = OC_old.reindex(index=full_idx, columns=full_cols, fill_value=0.0) OC_comparison = diff_and_perc_diff_two_output_contribution_matrices( OC_old, diff --git a/bedrock/utils/validation/diagnostics_helpers.py b/bedrock/utils/validation/diagnostics_helpers.py index 2fd92edf..e03cb1e9 100644 --- a/bedrock/utils/validation/diagnostics_helpers.py +++ b/bedrock/utils/validation/diagnostics_helpers.py @@ -129,12 +129,6 @@ def d_n_new_inflated_eligibility(cfg: USAConfig) -> tuple[bool, str]: ``derive_cornerstone_B_non_finetuned`` already inflates ``B`` to ``model_base_year``. """ - if not cfg.use_cornerstone_2026_model_schema: - return ( - False, - 'use_cornerstone_2026_model_schema is false (legacy B path; no ' - 'cornerstone D_new_inflated hook)', - ) if cfg.deflate_x_to_detail_io_year_for_B: if cfg.model_base_year == cfg.usa_detail_original_year: return (