diff --git a/bedrock/analysis/electricity/class_prices/compare_paths.py b/bedrock/analysis/electricity/class_prices/compare_paths.py index b103344d..9787c539 100644 --- a/bedrock/analysis/electricity/class_prices/compare_paths.py +++ b/bedrock/analysis/electricity/class_prices/compare_paths.py @@ -9,7 +9,7 @@ from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( MixedUnitEfResult, compute_mixed_unit_ef_vectors, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.transform.eeio.derived_cornerstone import ( derive_cornerstone_Aq_scaled, @@ -31,7 +31,7 @@ class ClassPricesComparison: def _equal_prices_from_table() -> dict[str, float]: cfg = get_usa_config() - prices = table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year) + prices = electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year) total = float(prices['Total']) return {k: total for k in prices} diff --git a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py index 151d57c9..dbef3121 100644 --- a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py @@ -19,7 +19,9 @@ from bedrock.analysis.electricity.d_85.disagg_weights import ( build_ugo_col_table83_row_intersection_matrix, ) -from bedrock.analysis.electricity.d_85.eia_inputs import table_2_4_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import ( + electricity_end_use_retail_prices_cents_kwh, +) from bedrock.transform.eeio.electricity_disaggregation import ELECTRICITY_AGGREGATE from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS @@ -92,7 +94,7 @@ def test_t83_production_diag_step2_uses_table83_weights( @mock.patch( - 'bedrock.analysis.electricity.d_85.disagg_scenarios.table_2_4_prices_cents_kwh' + 'bedrock.analysis.electricity.d_85.disagg_scenarios.electricity_end_use_retail_prices_cents_kwh' ) @mock.patch('bedrock.analysis.electricity.d_85.disagg_scenarios.build_end_use_map') @mock.patch('bedrock.analysis.electricity.d_85.disagg_scenarios.ugo305_go_weights') @@ -109,7 +111,7 @@ def test_p24_weights_sum_to_one_per_column( checkpoint_mock.return_value = (V, Udom, Uimp, VA, Y) ugo_mock.return_value = pd.Series({'221110': 0.34, '221121': 0.04, '221122': 0.62}) map_mock.return_value = {'541000': 'Commercial', 'F01000': 'Residential'} - prices_mock.return_value = table_2_4_prices_cents_kwh( + prices_mock.return_value = electricity_end_use_retail_prices_cents_kwh( 2017, fba=mock_fba_table83_table24() ) diff --git a/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py b/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py index 345c6f4e..1c50592d 100644 --- a/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py +++ b/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py @@ -6,7 +6,7 @@ import pytest from bedrock.analysis.electricity.d_85.eia_inputs import ( - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, table_8_3_gtd_expenses_musd, table_8_3_purchased_power_gtd_expenses_musd, ) @@ -28,8 +28,8 @@ def test_table_8_3_purchased_power_gtd_expenses_musd(mock_fba: pd.DataFrame) -> assert out['Distribution'] == pytest.approx(4358.0) -def test_table_2_4_prices_cents_kwh(mock_fba: pd.DataFrame) -> None: - prices = table_2_4_prices_cents_kwh(2017, fba=mock_fba) +def test_electricity_end_use_retail_prices_cents_kwh(mock_fba: pd.DataFrame) -> None: + prices = electricity_end_use_retail_prices_cents_kwh(2017, fba=mock_fba) assert prices['Residential'] == pytest.approx(12.89) assert prices['Total'] == pytest.approx(10.54) diff --git a/bedrock/analysis/electricity/d_85/disagg_scenarios.py b/bedrock/analysis/electricity/d_85/disagg_scenarios.py index 49ebe11a..880790d4 100644 --- a/bedrock/analysis/electricity/d_85/disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/disagg_scenarios.py @@ -12,7 +12,9 @@ table83_purchased_power_weights, ugo305_go_weights, ) -from bedrock.analysis.electricity.d_85.eia_inputs import table_2_4_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import ( + electricity_end_use_retail_prices_cents_kwh, +) from bedrock.analysis.electricity.d_85.end_use_mapping import ( build_end_use_map, build_price_tilt_weights_by_column, @@ -326,7 +328,7 @@ def _weights_t83_purchased_power_diag_compensated() -> ScenarioWeights: def _weights_p24(price_year: int) -> ScenarioWeights: w_ugo = ugo305_go_weights() - raw_prices = table_2_4_prices_cents_kwh(price_year) + raw_prices = electricity_end_use_retail_prices_cents_kwh(price_year) prices: dict[str, float] = {str(k): float(v) for k, v in raw_prices.items()} end_use_map = build_end_use_map() _, Udom, Uimp, _, Y = derive_post_reallocation_checkpoint() diff --git a/bedrock/analysis/electricity/d_85/eia_inputs.py b/bedrock/analysis/electricity/d_85/eia_inputs.py index 32b99025..60a243da 100644 --- a/bedrock/analysis/electricity/d_85/eia_inputs.py +++ b/bedrock/analysis/electricity/d_85/eia_inputs.py @@ -13,7 +13,7 @@ TABLE_2_4_DESCRIPTION, TABLE_2_4_PROVIDER, EPAEndUse, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) TABLE_8_3_DESCRIPTION = ( @@ -108,7 +108,7 @@ def table_8_3_purchased_power_gtd_expenses_musd( 'TABLE_8_3_GENERATION_FLOWNAMES', 'TABLE_8_3_PRODUCER', 'TABLE_8_3_SHARED_FLOWNAMES', - 'table_2_4_prices_cents_kwh', + 'electricity_end_use_retail_prices_cents_kwh', 'table_8_3_gtd_expenses_musd', 'table_8_3_purchased_power_gtd_expenses_musd', ] diff --git a/bedrock/analysis/electricity/d_85/end_use_mapping.py b/bedrock/analysis/electricity/d_85/end_use_mapping.py index 3193507b..a70b48d2 100644 --- a/bedrock/analysis/electricity/d_85/end_use_mapping.py +++ b/bedrock/analysis/electricity/d_85/end_use_mapping.py @@ -14,7 +14,7 @@ build_end_use_map, build_end_use_map_resolved, classify_industry_end_use, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS @@ -26,7 +26,7 @@ 'build_end_use_map_resolved', 'build_price_tilt_weights_by_column', 'classify_industry_end_use', - 'table_2_4_prices_cents_kwh', + 'electricity_end_use_retail_prices_cents_kwh', 'write_default_overrides_csv', ] diff --git a/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md b/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md index 69a238f9..88899765 100644 --- a/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md +++ b/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md @@ -175,8 +175,9 @@ Indirect fuel-chain emissions remain upstream (fuels assigned to generation in s After disaggregation, BEA summary sector `"22"` (Utilities) would apply one price index to all three children. Production applies the standard summary scaling, then a **D7 correction** using UGO305 detail gross-output ratios between `original_year` and -`target_year` per child (`build_electricity_ugo305_scaling_ratios` / -`apply_electricity_d7_scaling_correction_to_A` and `_q` in +`target_year` per child (`build_electricity_detail_GO_growth_ratios` / +`rescale_electricity_children_to_detail_GO_growth_A` / +`rescale_electricity_children_to_detail_GO_growth_q` in `cornerstone_year_scaling.py`). For 2017 → 2022, detail GO ratios differentiate G/T/D (generation ~1.62×, transmission @@ -287,6 +288,6 @@ Runtime ~1–2 minutes on a typical developer machine. | 4 | `disaggregate_use_industry_columns`, `_enforce_go_identity_precondition` | | 5 | `get_electricity_commodity_row_weights`, `disaggregate_use_commodity_rows`, Y split in `cornerstone_disagg_pipeline.py` | | 6 | `_apply_electricity_disagg_cornerstone_mapping` in `derived.py` | -| 7 | `build_electricity_ugo305_scaling_ratios`, `apply_electricity_d7_scaling_correction_*` in `cornerstone_year_scaling.py` | +| 7 | `build_electricity_detail_GO_growth_ratios`, `rescale_electricity_children_to_detail_GO_growth_*` in `cornerstone_year_scaling.py` | All in `bedrock/transform/eeio/electricity_disaggregation.py` unless noted. diff --git a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py index dc17ae6f..ac357867 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py @@ -59,7 +59,7 @@ ) from bedrock.transform.eeio.electricity_end_use_mapping import ( build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.config.usa_config import reset_usa_config, set_global_usa_config from bedrock.utils.schemas.cornerstone_schemas import ( @@ -337,7 +337,9 @@ def analyze() -> dict[str, Any]: egrid_mwh = float(us_total_net_generation_mwh(model_year)) end_use_map = build_end_use_map() - prices = cast(dict[str, float], table_2_4_prices_cents_kwh(ghg_year)) + prices = cast( + dict[str, float], electricity_end_use_retail_prices_cents_kwh(ghg_year) + ) eia = _eia_table_2_2_sales_mwh(model_year) w_go = build_electricity_disagg_go_weights() w_trans, w_dist = _td_national_shares() diff --git a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py index 19d12c11..38916b52 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py @@ -753,7 +753,7 @@ def _conversion_factor_detail(config: str) -> dict[str, Any]: ) from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) reset_usa_config() @@ -764,7 +764,10 @@ def _conversion_factor_detail(config: str) -> dict[str, Any]: q_usd = float(aq.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) c_col = electricity_output_factor(q_usd, mwh) - prices = cast(dict[str, float], table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year)) + prices = cast( + dict[str, float], + electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year), + ) end_use_map = build_end_use_map() y_row = _model_year_y_row_221110(aq) adom_row = cast(pd.Series, aq.Adom.loc[GENERATION_SECTOR]) diff --git a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py index 0b245103..086f9d20 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py @@ -39,7 +39,7 @@ from bedrock.transform.eeio.electricity_end_use_mapping import ( EPA_END_USES, build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS from bedrock.utils.taxonomy.cornerstone.commodities import COMMODITY_DESC @@ -504,7 +504,7 @@ def analyze() -> dict[str, Any]: eia = _eia_values(int(cfg.model_base_year)) prices = cast( Mapping[str, float], - table_2_4_prices_cents_kwh(int(cfg.usa_ghg_data_year)), + electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)), ) return { diff --git a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py index de45f17c..9d372ec1 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py @@ -50,7 +50,7 @@ from bedrock.transform.eeio.electricity_end_use_mapping import ( EPA_END_USES, build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.math.formulas import ( backcompute_y_from_A_and_q, @@ -232,7 +232,8 @@ def analyze() -> dict[str, Any]: intermediate_usd, y_row_usd, end_use_map, eia_classes ) prices_2_4 = cast( - dict[str, float], table_2_4_prices_cents_kwh(int(cfg.usa_ghg_data_year)) + dict[str, float], + electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)), ) # Production path (Table 2.4 + eGRID total) diff --git a/bedrock/publish/cache_reset.py b/bedrock/publish/cache_reset.py index 932ec9fc..6f6d8b1a 100644 --- a/bedrock/publish/cache_reset.py +++ b/bedrock/publish/cache_reset.py @@ -2,7 +2,9 @@ from __future__ import annotations +import sys from collections.abc import Callable +from types import ModuleType from bedrock.extract.iot.io_2017 import ( load_2017_margins_after_redef_usa, @@ -10,13 +12,11 @@ ) from bedrock.publish.model_objects import clear_publish_caches from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( - build_end_use_map, cornerstone_sector_disagg_active, derive_disagg_io_bundle, derive_disagg_Ytot_with_trade, electricity_mixed_units_enabled, get_waste_disagg_weights, - table_2_4_prices_cents_kwh, ) from bedrock.transform.eeio.derived import ( derive_Aq_usa, @@ -44,12 +44,6 @@ derive_cornerstone_y_nab_mixed_units, derive_cornerstone_Ytot_matrix_set, ) -from bedrock.transform.eeio.electricity_disaggregation import ( - _derive_post_reallocation_checkpoint_for_disagg, - build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, - get_electricity_commodity_row_weights, -) from bedrock.transform.iot.derive_PRO_to_PUR_ratio import ( derive_margins_cornerstone_usa_at_year, derive_phi_cornerstone_usa_at_year, @@ -61,6 +55,17 @@ get_price_index_ratio, ) +# Cached electricity helpers cleared only if their modules are already loaded. +# Never import electricity_disaggregation / electricity_end_use_mapping here — +# that would re-couple v0.3 / waste-only publish clears. +_ELECTRICITY_DISAGG_CACHED_ATTRS: tuple[str, ...] = ( + 'get_electricity_commodity_row_weights', + '_derive_post_reallocation_checkpoint_for_disagg', + 'build_electricity_disagg_use_intersection_weights', + 'build_electricity_detail_GO_growth_ratios', + 'build_electricity_disagg_go_weights', +) + UPSTREAM_CACHED_DERIVES: list[Callable[..., object]] = [ derive_B_usa_non_finetuned, derive_C_usa, @@ -70,14 +75,8 @@ cornerstone_sector_disagg_active, electricity_mixed_units_enabled, get_waste_disagg_weights, - build_end_use_map, - table_2_4_prices_cents_kwh, derive_disagg_io_bundle, derive_disagg_Ytot_with_trade, - get_electricity_commodity_row_weights, - _derive_post_reallocation_checkpoint_for_disagg, - build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, derive_cornerstone_V, derive_cornerstone_x, derive_cornerstone_x_after_redefinition, @@ -105,9 +104,39 @@ ] +def _clear_cached_attrs(mod: ModuleType, names: tuple[str, ...]) -> None: + for name in names: + fn = getattr(mod, name, None) + cache_clear = getattr(fn, 'cache_clear', None) + if callable(cache_clear): + cache_clear() + + +def _clear_electricity_caches_if_loaded() -> None: + """Flush elec ``@cache``s only when those modules are already in ``sys.modules``. + + After a flag-on run, leftover caches must still clear when switching to v0.3; + under a never-loaded v0.3 process, this is a no-op and never imports elec. + """ + ed = sys.modules.get('bedrock.transform.eeio.electricity_disaggregation') + if ed is not None: + _clear_cached_attrs(ed, _ELECTRICITY_DISAGG_CACHED_ATTRS) + eum = sys.modules.get('bedrock.transform.eeio.electricity_end_use_mapping') + if eum is not None: + # No @cache today; keep for future-proofing if helpers become cached. + _clear_cached_attrs( + eum, + ( + 'build_end_use_map', + 'electricity_end_use_retail_prices_cents_kwh', + ), + ) + + def clear_all_publish_caches() -> None: clear_cornerstone_inflation_caches() for fn in UPSTREAM_CACHED_DERIVES: if hasattr(fn, 'cache_clear'): fn.cache_clear() + _clear_electricity_caches_if_loaded() clear_publish_caches() diff --git a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py index 3cff672d..9fab73dc 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py @@ -35,11 +35,11 @@ _compute_w_row, _derive_post_reallocation_checkpoint_for_disagg, _float_ndarray, - _table83_purchased_power_expenses, - _weights_from_table83_expenses, + _iou_utility_gtd_operating_expenses, + _normalize_gtd_expense_weights, + build_electricity_detail_GO_growth_ratios, build_electricity_disagg_go_weights, build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, disaggregate_use_industry_columns, get_electricity_commodity_row_weights, ) @@ -63,7 +63,7 @@ derive_disagg_Ytot_with_trade, build_electricity_disagg_go_weights, build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, + build_electricity_detail_GO_growth_ratios, get_electricity_commodity_row_weights, _derive_post_reallocation_checkpoint_for_disagg, derive_cornerstone_V, @@ -219,10 +219,10 @@ def test_purchased_power_weights_sum_to_one(self) -> None: mock_fba_table83_table24, ) - expenses = _table83_purchased_power_expenses( + expenses = _iou_utility_gtd_operating_expenses( 2017, fba=mock_fba_table83_table24() ) - w = _weights_from_table83_expenses(expenses) + w = _normalize_gtd_expense_weights(expenses) assert set(w.index) == set(ELECTRICITY_DISAGG_SECTORS) np.testing.assert_allclose(float(w.sum()), 1.0, rtol=1e-9, atol=1e-12) assert w['221110'] == pytest.approx(49030.0 / (49030.0 + 10804.0 + 4358.0)) @@ -274,7 +274,7 @@ def test_differentiated_child_q_scaling( _setup_config(electricity_disagg_config) try: aq = derive_cornerstone_Aq_scaled() - ratios = build_electricity_ugo305_scaling_ratios(2017, 2022) + ratios = build_electricity_detail_GO_growth_ratios(2017, 2022) q_vals = [float(aq.scaled_q[c]) for c in ELECTRICITY_DISAGG_SECTORS] assert len(set(round(v, 6) for v in q_vals)) == 3 assert ratios['221110'] != pytest.approx(ratios['221121']) diff --git a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py index e25e6635..97290229 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py @@ -183,7 +183,7 @@ def test_mixed_units_flag_off_is_noop() -> None: return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_output_mwh_anchor( mock_prices: Mock, @@ -269,7 +269,7 @@ def test_y_nab_stays_monetary_under_mixed_gate(mixed_units_config: str) -> None: return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_y_nab_mixed_differs_from_monetary_under_gate( mock_prices: Mock, @@ -302,7 +302,7 @@ def test_y_nab_mixed_differs_from_monetary_under_gate( return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_d_scalar_bridge_under_gate( mock_prices: Mock, diff --git a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py index c8704c8d..db5714bd 100644 --- a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py +++ b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py @@ -3,6 +3,11 @@ Returns uninflated 2017-chain-dollar IO matrices only. Public entry points in ``derived_cornerstone`` apply inflation and year-scaling after routing here. + +Electricity modules are loaded only when electricity flags are on (or when +callers invoke the lazy end-use facade). Importing this module under +waste-only / v0.3 configs does not load ``electricity_disaggregation`` or +``electricity_end_use_mapping``. """ from __future__ import annotations @@ -11,7 +16,7 @@ import pathlib from collections.abc import Mapping from dataclasses import dataclass -from typing import cast +from typing import Any, cast import pandas as pd import pandera.typing as pt @@ -32,26 +37,6 @@ commodity_corresp, industry_corresp, ) -from bedrock.transform.eeio.electricity_disaggregation import ( - GENERATION_SECTOR, - apply_electricity_unit_conversion_to_A, - apply_electricity_unit_conversion_to_B, - apply_electricity_unit_conversion_to_q, - disaggregate_electricity_commodity_row_in_y, - disaggregate_electricity_make_use_va, - distribute_electricity_aggregate_x_using_v_row_shares, - electricity_class_row_factors, - electricity_output_factor, - get_electricity_commodity_row_weights, - reallocate_electricity_coproduction, -) -from bedrock.transform.eeio.electricity_end_use_mapping import ( - END_USE_MAPPING_REVIEW_STATUS, # noqa: F401 — re-export - build_end_use_map, - build_end_use_map_resolved, # noqa: F401 — re-export - classify_industry_end_use, # noqa: F401 — re-export - table_2_4_prices_cents_kwh, -) from bedrock.transform.eeio.waste_disaggregation import ( apply_waste_disagg_to_U, apply_waste_disagg_to_V, @@ -205,8 +190,16 @@ def derive_disagg_io_bundle() -> CornerstoneDisaggIOBundle: Udom, Uimp = derive_cornerstone_U_after_waste() VA = derive_cornerstone_VA_after_waste() if electricity_reallocation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + reallocate_electricity_coproduction, + ) + V, Udom, Uimp, VA = reallocate_electricity_coproduction(V, Udom, Uimp, VA) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + disaggregate_electricity_make_use_va, + ) + V, Udom, Uimp, VA = disaggregate_electricity_make_use_va(V, Udom, Uimp, VA) return CornerstoneDisaggIOBundle(V=V, Udom=Udom, Uimp=Uimp, VA=VA) @@ -222,6 +215,11 @@ def derive_disagg_Ytot_with_trade() -> pd.DataFrame: Ytot = apply_waste_disagg_to_Ytot(Ytot, weights) Ytot.index.name = 'sector' if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + disaggregate_electricity_commodity_row_in_y, + get_electricity_commodity_row_weights, + ) + w_row = get_electricity_commodity_row_weights() Ytot = disaggregate_electricity_commodity_row_in_y(Ytot, w_row) Ytot.index.name = 'sector' @@ -251,6 +249,10 @@ def distribute_waste_parent_x_using_v_row_shares( for code in present: x.loc[code] = parent_go * float(shares.loc[code]) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + distribute_electricity_aggregate_x_using_v_row_shares, + ) + return distribute_electricity_aggregate_x_using_v_row_shares( x, derive_disagg_io_bundle().V ) @@ -262,8 +264,84 @@ def electricity_mixed_units_enabled() -> bool: return get_usa_config().implement_electricity_mixed_units +# --- Lazy end-use facade (importing this module must not load elec) ------------- + + +def build_end_use_map() -> dict[str, str]: + """Lazy re-export of ``electricity_end_use_mapping.build_end_use_map``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + build_end_use_map as _impl, + ) + + return _impl() + + +def electricity_end_use_retail_prices_cents_kwh( + year: int, + provider: str | None = None, + *, + fba: pd.DataFrame | None = None, +) -> dict[str, float]: + """Lazy re-export of ``electricity_end_use_mapping.electricity_end_use_retail_prices_cents_kwh``. + + Kept as a module-level callable so + ``@patch('…cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh')`` continues + to intercept calls from ``electricity_conversion_factors``. + """ + from bedrock.transform.eeio import ( # noqa: PLC0415 + electricity_end_use_mapping as eum, + ) + + return cast( + dict[str, float], + eum.electricity_end_use_retail_prices_cents_kwh( + year, + eum.TABLE_2_4_PROVIDER if provider is None else provider, + fba=fba, + ), + ) + + +def classify_industry_end_use(industry_code: str) -> tuple[str, str]: + """Lazy re-export of ``electricity_end_use_mapping.classify_industry_end_use``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + classify_industry_end_use as _impl, + ) + + return _impl(industry_code) + + +def build_end_use_map_resolved( + prices_by_class: dict[str, float] | None = None, + *, + c_col: float | None = None, + c_row: pd.Series[float] | None = None, +) -> pd.DataFrame: + """Lazy re-export of ``electricity_end_use_mapping.build_end_use_map_resolved``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + build_end_use_map_resolved as _impl, + ) + + return _impl(prices_by_class, c_col=c_col, c_row=c_row) + + +def __getattr__(name: str) -> Any: + """Lazy attribute access for end-use constants re-exported from this module.""" + if name == 'END_USE_MAPPING_REVIEW_STATUS': + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + END_USE_MAPPING_REVIEW_STATUS, + ) + + return END_USE_MAPPING_REVIEW_STATUS + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + def _model_year_y_row_221110(aq_scaled: SingleRegionAqMatrixSet) -> pd.Series[float]: """Model-year 221110 FD row from backcompute total × 2017 share split.""" + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + GENERATION_SECTOR, + ) + y_2017 = derive_disagg_Ytot_with_trade().loc[GENERATION_SECTOR] y_total = float( backcompute_y_from_A_and_q(A=aq_scaled.Adom, q=aq_scaled.scaled_q).loc[ @@ -287,15 +365,20 @@ def electricity_conversion_factors( from bedrock.extract.disaggregation.egrid_generation import ( # noqa: PLC0415 us_total_net_generation_mwh, ) + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + GENERATION_SECTOR, + electricity_class_row_factors, + electricity_output_factor, + ) + # Call module-level facade for end-use helpers so unittest patches on + # ``cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh`` still apply. cfg = get_usa_config() q_usd = float(aq_scaled.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) c_col = electricity_output_factor(q_usd, mwh) if prices_by_class is None: - prices = cast( - dict[str, float], table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year) - ) + prices = electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year) else: prices = dict(prices_by_class) end_use_map = build_end_use_map() @@ -320,6 +403,11 @@ def build_electricity_mixed_units_aq( """Return mixed-unit A/q when gate is on; else pass-through.""" if not electricity_mixed_units_enabled(): return aq_scaled + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_A, + apply_electricity_unit_conversion_to_q, + ) + c_col, c_row = electricity_conversion_factors( aq_scaled, prices_by_class=prices_by_class ) @@ -344,6 +432,10 @@ def build_electricity_mixed_units_b( """Return mixed-unit B when gate is on; else pass-through.""" if not electricity_mixed_units_enabled(): return b + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_B, + ) + return apply_electricity_unit_conversion_to_B(b, c_col) @@ -365,6 +457,10 @@ def compute_mixed_unit_ef_vectors( prices_by_class: Mapping[str, float] | None = None, ) -> MixedUnitEfResult: """Apply mixed conversion to monetary scaled A/q/B; never use cached mixed derives.""" + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_A, + apply_electricity_unit_conversion_to_B, + ) from bedrock.utils.math.formulas import ( # noqa: PLC0415 compute_d, compute_L_matrix, diff --git a/bedrock/transform/eeio/cornerstone_year_scaling.py b/bedrock/transform/eeio/cornerstone_year_scaling.py index c5e9efa5..e09bb87c 100644 --- a/bedrock/transform/eeio/cornerstone_year_scaling.py +++ b/bedrock/transform/eeio/cornerstone_year_scaling.py @@ -142,12 +142,13 @@ def scale_cornerstone_A( from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, ) - from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_A, - ) if electricity_disaggregation_enabled(): - A_scaled = apply_electricity_d7_scaling_correction_to_A( + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + rescale_electricity_children_to_detail_GO_growth_A, + ) + + A_scaled = rescale_electricity_children_to_detail_GO_growth_A( A_scaled, original_year, target_year ) @@ -182,12 +183,13 @@ def scale_cornerstone_q( from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, ) - from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_q, - ) if electricity_disaggregation_enabled(): - q_scaled = apply_electricity_d7_scaling_correction_to_q( + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + rescale_electricity_children_to_detail_GO_growth_q, + ) + + q_scaled = rescale_electricity_children_to_detail_GO_growth_q( q_scaled, original_year, target_year ) diff --git a/bedrock/transform/eeio/electricity_disaggregation.py b/bedrock/transform/eeio/electricity_disaggregation.py index 9691b176..e9bcdebe 100644 --- a/bedrock/transform/eeio/electricity_disaggregation.py +++ b/bedrock/transform/eeio/electricity_disaggregation.py @@ -316,7 +316,7 @@ def build_electricity_disagg_go_weights() -> pd.Series[float]: ) -def _table83_purchased_power_expenses( +def _iou_utility_gtd_operating_expenses( year: int, *, fba: pd.DataFrame | None = None, @@ -348,7 +348,7 @@ def _table83_purchased_power_expenses( return out -def _weights_from_table83_expenses(expenses: dict[str, float]) -> pd.Series[float]: +def _normalize_gtd_expense_weights(expenses: dict[str, float]) -> pd.Series[float]: keys = list(expenses.keys()) if len(keys) != 3: raise ValueError(f'expected 3 expense buckets, got {keys!r}') @@ -369,8 +369,8 @@ def _weights_from_table83_expenses(expenses: dict[str, float]) -> pd.Series[floa @functools.cache def build_electricity_disagg_use_intersection_weights() -> pd.Series[float]: """Return Table 8.3 Purchased Power + T/D shares for step 2 intersection.""" - expenses = _table83_purchased_power_expenses(IO_ACCOUNT_YEAR) - return _weights_from_table83_expenses(expenses) + expenses = _iou_utility_gtd_operating_expenses(IO_ACCOUNT_YEAR) + return _normalize_gtd_expense_weights(expenses) def _diagonal_intersection_weights(w: pd.Series[float]) -> pd.DataFrame: @@ -913,7 +913,7 @@ def _go_levels_for_year(year: int) -> pd.Series[float]: @functools.cache -def build_electricity_ugo305_scaling_ratios( +def build_electricity_detail_GO_growth_ratios( original_year: int, target_year: int, ) -> pd.Series[float]: @@ -924,8 +924,11 @@ def build_electricity_ugo305_scaling_ratios( return ratios.fillna(1.0).reindex(ELECTRICITY_DISAGG_SECTORS) -def utilities_summary_ratio_22(original_year: int, target_year: int) -> float: - """Summary Utilities sector-22 q ratio used as D7 base scaling factor.""" +def utilities_summary_q_growth_ratio(original_year: int, target_year: int) -> float: + """BEA summary Make-Use-Trade (MUT) Utilities sector-22 q growth ratio. + + Used as the denominator when rescaling electricity children to detail GO growth. + """ orig = cast(USA_SUMMARY_MUT_YEARS, original_year) tgt = cast(USA_SUMMARY_MUT_YEARS, target_year) ratio = (derive_summary_q_usa(tgt) / derive_summary_q_usa(orig)).fillna(1.0) @@ -933,14 +936,14 @@ def utilities_summary_ratio_22(original_year: int, target_year: int) -> float: return val if np.isfinite(val) else 1.0 -def apply_electricity_d7_scaling_correction_to_A( +def rescale_electricity_children_to_detail_GO_growth_A( a: pd.DataFrame, original_year: int, target_year: int, ) -> pd.DataFrame: """Rescale electricity child rows after summary-ratio A scaling (D7 pure).""" - ratios = build_electricity_ugo305_scaling_ratios(original_year, target_year) - base = utilities_summary_ratio_22(original_year, target_year) + ratios = build_electricity_detail_GO_growth_ratios(original_year, target_year) + base = utilities_summary_q_growth_ratio(original_year, target_year) out = a.copy() for code in ELECTRICITY_DISAGG_SECTORS: if code not in out.index: @@ -950,14 +953,14 @@ def apply_electricity_d7_scaling_correction_to_A( return out -def apply_electricity_d7_scaling_correction_to_q( +def rescale_electricity_children_to_detail_GO_growth_q( q: pd.Series[float], original_year: int, target_year: int, ) -> pd.Series[float]: """Rescale electricity child q rows after summary-ratio q scaling (D7 pure).""" - ratios = build_electricity_ugo305_scaling_ratios(original_year, target_year) - base = utilities_summary_ratio_22(original_year, target_year) + ratios = build_electricity_detail_GO_growth_ratios(original_year, target_year) + base = utilities_summary_q_growth_ratio(original_year, target_year) out = q.copy() for code in ELECTRICITY_DISAGG_SECTORS: if code not in out.index: diff --git a/bedrock/transform/eeio/electricity_end_use_mapping.py b/bedrock/transform/eeio/electricity_end_use_mapping.py index cf78c40b..49fb1a65 100644 --- a/bedrock/transform/eeio/electricity_end_use_mapping.py +++ b/bedrock/transform/eeio/electricity_end_use_mapping.py @@ -170,13 +170,13 @@ def _load_eia_fba(year: int) -> pd.DataFrame: return getFlowByActivity('EIA_ElectricPowerAnnual', year) -def table_2_4_prices_cents_kwh( +def electricity_end_use_retail_prices_cents_kwh( year: int, provider: str = TABLE_2_4_PROVIDER, *, fba: pd.DataFrame | None = None, ) -> dict[EPAEndUse, float]: - """Return end-use retail prices (cents/kWh) from Table 2.4.""" + """source is EIA Electric Power Annual Table 2.4 (Average price to ultimate customers); used to build class-specific MWh/$ factors.""" df = fba if fba is not None else _load_eia_fba(year) mask = ( (df['Year'] == year) diff --git a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py index 51997d02..da13f992 100644 --- a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py @@ -215,7 +215,7 @@ def mock_load(name: str) -> pd.DataFrame: "bedrock.transform.eeio.cornerstone_disagg_pipeline.compute_mixed_unit_ef_vectors", ) as mock_uniform, patch( - "bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh", + "bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh", return_value={"Total": 10.0}, ), patch( diff --git a/bedrock/utils/validation/calculate_ef_diagnostics.py b/bedrock/utils/validation/calculate_ef_diagnostics.py index f3d4233c..acdfeb84 100644 --- a/bedrock/utils/validation/calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/calculate_ef_diagnostics.py @@ -271,8 +271,8 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 compute_mixed_unit_ef_vectors, + electricity_end_use_retail_prices_cents_kwh, electricity_mixed_units_enabled, - table_2_4_prices_cents_kwh, ) from bedrock.transform.eeio.derived import ( # noqa: PLC0415 derive_B_usa_non_finetuned, @@ -301,7 +301,9 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: n_mon = ta.cast('pd.Series[float]', compute_n(M=m_mon).squeeze()) d_mix = _ef_vector_as_series(efs.D_new) n_mix = _ef_vector_as_series(efs.N_new) - table_prices = table_2_4_prices_cents_kwh(config.usa_ghg_data_year) + table_prices = electricity_end_use_retail_prices_cents_kwh( + config.usa_ghg_data_year + ) total_price = float(table_prices['Total']) equal_prices: dict[str, float] = {str(k): total_price for k in table_prices} uniform_result = compute_mixed_unit_ef_vectors(