From f7650e556311402783e7c36c265b537ea96df84e Mon Sep 17 00:00:00 2001 From: jvendries Date: Wed, 29 Jul 2026 22:16:39 -0400 Subject: [PATCH 1/3] Decouple electricity disagg from main cornerstone pipeline --- .../plan/electricity-pipeline-decoupling.md | 151 ++++++++++++++++++ bedrock/publish/cache_reset.py | 56 +++++-- .../eeio/cornerstone_disagg_pipeline.py | 141 +++++++++++++--- .../eeio/cornerstone_year_scaling.py | 14 +- 4 files changed, 321 insertions(+), 41 deletions(-) create mode 100644 .claude/plan/electricity-pipeline-decoupling.md diff --git a/.claude/plan/electricity-pipeline-decoupling.md b/.claude/plan/electricity-pipeline-decoupling.md new file mode 100644 index 00000000..f5ed833e --- /dev/null +++ b/.claude/plan/electricity-pipeline-decoupling.md @@ -0,0 +1,151 @@ +--- +name: Elec Pipeline Decoupling +overview: Structurally decouple electricity reallocation/disaggregation/mixed-units from the main Cornerstone EEIO path so canonical v0.3 (and waste-only configs) never import elec modules, while flag-on configs keep working via lazy imports. +todos: + - id: save-plan-md + content: Write plan copy to .claude/plan/electricity-pipeline-decoupling.md + status: completed + - id: lazy-cdp + content: Lazy-import elec in cornerstone_disagg_pipeline; keep waste + gates; lazy facade for end-use re-exports + status: completed + - id: lazy-year-scaling + content: Nest D7 imports inside electricity_disaggregation_enabled() in cornerstone_year_scaling + status: completed + - id: lazy-derived-cornerstone + content: Optional clarity — lazy mixed-units imports in derived_cornerstone (not required once cdp is decoupled) + status: cancelled + - id: cache-reset + content: Clear elec caches via sys.modules.get only if already loaded; never import elec under v0.3 + status: completed + - id: allocation-soft + content: Optional — demote CORNERSTONE_INDUSTRIES_ELEC hygiene (schemas only; not elec-module coupling) + status: cancelled + - id: verify + content: v0.3 import + cache_reset probe without elec modules; flag-on + existing tests green + status: completed +isProject: false +--- + +# Structural decoupling of electricity from main Cornerstone pipeline + +## Goal + +Canonical `2025_usa_cornerstone_v0_3` (all `implement_electricity_*` false; waste disagg on) must import and run without loading [`electricity_disaggregation.py`](bedrock/transform/eeio/electricity_disaggregation.py) or [`electricity_end_use_mapping.py`](bedrock/transform/eeio/electricity_end_use_mapping.py). Waste-only disagg stays. Flag-on configs keep current behavior via lazy imports. + +Also save this plan to [`.claude/plan/electricity-pipeline-decoupling.md`](.claude/plan/electricity-pipeline-decoupling.md) when implementing. + +## Current coupling (why delete-as-is fails) + +```mermaid +flowchart TD + derived["derived.py"] --> dc["derived_cornerstone.py"] + dc --> cdp["cornerstone_disagg_pipeline.py"] + cdp -->|"top-level import"| ed["electricity_disaggregation.py"] + cdp -->|"top-level import"| eum["electricity_end_use_mapping.py"] + ys["cornerstone_year_scaling.py"] -->|"import then if flag"| ed + cr["publish/cache_reset.py"] -->|"top-level import"| ed + cr --> cdp + cr --> derived +``` + +Runtime call sites are already gated. The break is **import-time**. + +## Approach + +Lazy-import electricity symbols **inside** flag-true branches (and inside mixed-units-only entry points). Keep gate helpers (`electricity_*_enabled`) and waste orchestration in [`cornerstone_disagg_pipeline.py`](bedrock/transform/eeio/cornerstone_disagg_pipeline.py). No taxonomy/schema rewrite. + +**Preferred re-export strategy:** keep `build_end_use_map` / `table_2_4_prices_cents_kwh` (and other end-use re-exports used by callers) as a **lazy facade on cdp** — defining/calling them must not load elec at `import cdp` time; the first call loads `electricity_end_use_mapping`. That preserves: + +- Production importers (e.g. [`calculate_ef_diagnostics.py`](bedrock/utils/validation/calculate_ef_diagnostics.py) ~272–276) +- Test patches on `cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh` ([`test_electricity_mixed_units.py`](bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py) ~186/272/305; [`test_calculate_ef_diagnostics.py`](bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py) ~218) +- Analysis / diagnostics imports of cdp re-exports (out of scope for retarget, but must not AttributeError) + +Do **not** retarget every caller unless the facade approach fails; if retarget is chosen instead, update every production/test `@patch` path listed above. + +## Implementation + +### 1. [`cornerstone_disagg_pipeline.py`](bedrock/transform/eeio/cornerstone_disagg_pipeline.py) — main cut + +Remove top-level imports from `electricity_disaggregation` and `electricity_end_use_mapping` (lines ~35–54 today). Keep waste imports and `electricity_*_enabled()` / `cornerstone_sector_disagg_active()` (config-only). + +**Lazy-import inventory (symbol → function):** + +| Symbol(s) | Import inside | +|---|---| +| `reallocate_electricity_coproduction` | `derive_disagg_io_bundle` — under `if electricity_reallocation_enabled()` | +| `disaggregate_electricity_make_use_va` | `derive_disagg_io_bundle` — under `if electricity_disaggregation_enabled()` | +| `get_electricity_commodity_row_weights`, `disaggregate_electricity_commodity_row_in_y` | `derive_disagg_Ytot_with_trade` — under disagg `if` | +| `distribute_electricity_aggregate_x_using_v_row_shares` | `distribute_waste_parent_x_using_v_row_shares` — under disagg `if` (~253) | +| `GENERATION_SECTOR` | `_model_year_y_row_221110` (~265) and any other user of the constant | +| `GENERATION_SECTOR`, `electricity_output_factor`, `electricity_class_row_factors` | `electricity_conversion_factors` (~281) — local-import at function entry | +| *(call only, do not local-import)* `table_2_4_prices_cents_kwh`, `build_end_use_map` | `electricity_conversion_factors` — **must** use module-level lazy facade names so `@patch('…cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh')` still works | +| `apply_electricity_unit_conversion_to_A/q` | `build_electricity_mixed_units_aq` (~315) — nest import **after** `electricity_mixed_units_enabled()` early return | +| `apply_electricity_unit_conversion_to_B` | `build_electricity_mixed_units_b` (~340) — same (after early return) | +| unit-conversion helpers (+ conversion factors path) | `compute_mixed_unit_ef_vectors` (~361) | + +**Lazy facade for end-use re-exports** (required for patch targets): keep module-level names `build_end_use_map`, `table_2_4_prices_cents_kwh`, and existing F401 re-exports (`END_USE_MAPPING_REVIEW_STATUS`, `build_end_use_map_resolved`, `classify_industry_end_use`) as **explicit thin wrappers** (prefer over `__getattr__` alone) that import from `electricity_end_use_mapping` on first call. Importing cdp must not load elec; `from cdp import table_2_4_…` / `@patch` on the cdp attribute must bind real callables. Do **not** local-import those two symbols inside `electricity_conversion_factors`. + +### 2. [`cornerstone_year_scaling.py`](bedrock/transform/eeio/cornerstone_year_scaling.py) + +In `scale_cornerstone_A` and `scale_cornerstone_q`, D7 correction imports are already function-local (~142–152, ~182–190) but run **before** the flag check. Nest the `from …electricity_disaggregation import …` **inside** `if electricity_disaggregation_enabled():`. Gate helper import from cdp stays (safe once cdp is decoupled). + +### 3. [`derived_cornerstone.py`](bedrock/transform/eeio/derived_cornerstone.py) + +**Not required for import-time decoupling** once cdp drops top-level elec imports: `from cdp import build_electricity_mixed_units_aq` no longer loads elec. Optional clarity: lazy-import mixed-units builders / `electricity_conversion_factors` inside the mixed-units derive functions (~942–950, ~1037). Keeping `electricity_mixed_units_enabled` as a top-level cdp import is fine (config-only). + +Top-level import from cdp for waste/disagg routing stays: `cornerstone_sector_disagg_active`, `derive_disagg_io_bundle`, `derive_disagg_Ytot_with_trade`, `distribute_waste_parent_x_using_v_row_shares`. + +### 4. [`publish/cache_reset.py`](bedrock/publish/cache_reset.py) + +- Remove top-level imports of elec weight builders / checkpoints from `electricity_disaggregation` (~47–52). +- Stop requiring top-level `build_end_use_map` / `table_2_4_prices_cents_kwh` from cdp for the clear list if those are not `@cache`’d (today they are listed in `UPSTREAM_CACHED_DERIVES` but clearing them is a no-op if not cached). Prefer clearing end-use mapping caches only if present on the loaded module. +- Always clear waste + core cornerstone / `derived` caches (unchanged). +- **Clear elec `@functools.cache` only if already loaded** — never `import` / try-import under v0.3: + +```python +import sys + +def _clear_cached_attrs(mod, names: tuple[str, ...]) -> None: + for name in names: + fn = getattr(mod, name, None) + if hasattr(fn, 'cache_clear'): + fn.cache_clear() + +ed = sys.modules.get('bedrock.transform.eeio.electricity_disaggregation') +if ed is not None: + _clear_cached_attrs(ed, ( + 'get_electricity_commodity_row_weights', + '_derive_post_reallocation_checkpoint_for_disagg', + 'build_electricity_disagg_use_intersection_weights', + 'build_electricity_ugo305_scaling_ratios', + # …any other cached elec derives currently in UPSTREAM_CACHED_DERIVES + )) +eum = sys.modules.get('bedrock.transform.eeio.electricity_end_use_mapping') +if eum is not None: + _clear_cached_attrs(eum, (...)) # only if any become cached +``` + +Flag-gated import is wrong for multi-config processes: after an elec run, leftover caches must still clear when switching to v0.3; `sys.modules.get` handles that without reloading. + +### 5. Soft cleanup — [`allocation/derived.py`](bedrock/transform/allocation/derived.py) (optional / demoted) + +`CORNERSTONE_INDUSTRIES_ELEC` is imported from [`cornerstone_schemas`](bedrock/utils/schemas/cornerstone_schemas.py) (~16), **not** from elec modules — this does **not** load `electricity_disaggregation.py`. Only ~486 uses the name; ~243/~391 only check the flag. Optional hygiene (`active_cornerstone_industries()` or defer import); **not** part of the decoupling acceptance probe. + +### Out of scope + +- Analysis / diagnostics packages (may keep hard imports of elec or cdp facade). +- Schema constants `ELECTRICITY_DISAGG_SECTORS` / `CORNERSTONE_*_ELEC` (harmless when unused; schemas ≠ elec module load). +- Behavior changes when any electricity flag is True. +- Retargeting analysis callers away from cdp re-exports (facade preserves them). + +## Acceptance / testing + +1. **Import + cache_reset probe (v0.3):** with elec modules blocked (rename or `sys.modules` sentinel so import fails): + - `from bedrock.transform.eeio.derived import derive_Aq_usa` + - `from bedrock.publish.cache_reset import clear_all_publish_caches` then call it + - install `2025_usa_cornerstone_v0_3` and call `derive_cornerstone_Aq_scaled()` (or `derive_Aq_usa`) + - Assert `electricity_disaggregation` and `electricity_end_use_mapping` are **absent** from `sys.modules` +2. **Multi-config cache clear:** under a process that previously loaded elec (flag-on), `clear_all_publish_caches()` still clears elec `@cache`s via `sys.modules` even when current config is v0.3 / flags false. +3. **Flag-on smoke:** `test_usa_config_waste_disagg_electricity_disaggregation` and mixed-units config still produce 407 / mixed paths; existing patches on `cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh` still work (facade). +4. **Regression:** `test_electricity_disaggregation.py`, `test_electricity_reallocation.py`, `test_electricity_mixed_units.py`, waste pipeline tests; publish CLI / helpers that call `clear_all_publish_caches` still succeed under v0.3. +5. No intentional behavior change when flags are off. diff --git a/bedrock/publish/cache_reset.py b/bedrock/publish/cache_reset.py index 932ec9fc..955d841c 100644 --- a/bedrock/publish/cache_reset.py +++ b/bedrock/publish/cache_reset.py @@ -2,7 +2,9 @@ from __future__ import annotations +import sys from collections.abc import Callable +from types import ModuleType from bedrock.extract.iot.io_2017 import ( load_2017_margins_after_redef_usa, @@ -10,13 +12,11 @@ ) from bedrock.publish.model_objects import clear_publish_caches from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( - build_end_use_map, cornerstone_sector_disagg_active, derive_disagg_io_bundle, derive_disagg_Ytot_with_trade, electricity_mixed_units_enabled, get_waste_disagg_weights, - table_2_4_prices_cents_kwh, ) from bedrock.transform.eeio.derived import ( derive_Aq_usa, @@ -44,12 +44,6 @@ derive_cornerstone_y_nab_mixed_units, derive_cornerstone_Ytot_matrix_set, ) -from bedrock.transform.eeio.electricity_disaggregation import ( - _derive_post_reallocation_checkpoint_for_disagg, - build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, - get_electricity_commodity_row_weights, -) from bedrock.transform.iot.derive_PRO_to_PUR_ratio import ( derive_margins_cornerstone_usa_at_year, derive_phi_cornerstone_usa_at_year, @@ -61,6 +55,17 @@ get_price_index_ratio, ) +# Cached electricity helpers cleared only if their modules are already loaded. +# Never import electricity_disaggregation / electricity_end_use_mapping here — +# that would re-couple v0.3 / waste-only publish clears. +_ELECTRICITY_DISAGG_CACHED_ATTRS: tuple[str, ...] = ( + 'get_electricity_commodity_row_weights', + '_derive_post_reallocation_checkpoint_for_disagg', + 'build_electricity_disagg_use_intersection_weights', + 'build_electricity_ugo305_scaling_ratios', + 'build_electricity_disagg_go_weights', +) + UPSTREAM_CACHED_DERIVES: list[Callable[..., object]] = [ derive_B_usa_non_finetuned, derive_C_usa, @@ -70,14 +75,8 @@ cornerstone_sector_disagg_active, electricity_mixed_units_enabled, get_waste_disagg_weights, - build_end_use_map, - table_2_4_prices_cents_kwh, derive_disagg_io_bundle, derive_disagg_Ytot_with_trade, - get_electricity_commodity_row_weights, - _derive_post_reallocation_checkpoint_for_disagg, - build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, derive_cornerstone_V, derive_cornerstone_x, derive_cornerstone_x_after_redefinition, @@ -105,9 +104,38 @@ ] +def _clear_cached_attrs(mod: ModuleType, names: tuple[str, ...]) -> None: + for name in names: + fn = getattr(mod, name, None) + if hasattr(fn, 'cache_clear'): + fn.cache_clear() + + +def _clear_electricity_caches_if_loaded() -> None: + """Flush elec ``@cache``s only when those modules are already in ``sys.modules``. + + After a flag-on run, leftover caches must still clear when switching to v0.3; + under a never-loaded v0.3 process, this is a no-op and never imports elec. + """ + ed = sys.modules.get('bedrock.transform.eeio.electricity_disaggregation') + if ed is not None: + _clear_cached_attrs(ed, _ELECTRICITY_DISAGG_CACHED_ATTRS) + eum = sys.modules.get('bedrock.transform.eeio.electricity_end_use_mapping') + if eum is not None: + # No @cache today; keep for future-proofing if helpers become cached. + _clear_cached_attrs( + eum, + ( + 'build_end_use_map', + 'table_2_4_prices_cents_kwh', + ), + ) + + def clear_all_publish_caches() -> None: clear_cornerstone_inflation_caches() for fn in UPSTREAM_CACHED_DERIVES: if hasattr(fn, 'cache_clear'): fn.cache_clear() + _clear_electricity_caches_if_loaded() clear_publish_caches() diff --git a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py index c8704c8d..05148206 100644 --- a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py +++ b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py @@ -3,6 +3,11 @@ Returns uninflated 2017-chain-dollar IO matrices only. Public entry points in ``derived_cornerstone`` apply inflation and year-scaling after routing here. + +Electricity modules are loaded only when electricity flags are on (or when +callers invoke the lazy end-use facade). Importing this module under +waste-only / v0.3 configs does not load ``electricity_disaggregation`` or +``electricity_end_use_mapping``. """ from __future__ import annotations @@ -11,7 +16,7 @@ import pathlib from collections.abc import Mapping from dataclasses import dataclass -from typing import cast +from typing import Any, cast import pandas as pd import pandera.typing as pt @@ -32,26 +37,6 @@ commodity_corresp, industry_corresp, ) -from bedrock.transform.eeio.electricity_disaggregation import ( - GENERATION_SECTOR, - apply_electricity_unit_conversion_to_A, - apply_electricity_unit_conversion_to_B, - apply_electricity_unit_conversion_to_q, - disaggregate_electricity_commodity_row_in_y, - disaggregate_electricity_make_use_va, - distribute_electricity_aggregate_x_using_v_row_shares, - electricity_class_row_factors, - electricity_output_factor, - get_electricity_commodity_row_weights, - reallocate_electricity_coproduction, -) -from bedrock.transform.eeio.electricity_end_use_mapping import ( - END_USE_MAPPING_REVIEW_STATUS, # noqa: F401 — re-export - build_end_use_map, - build_end_use_map_resolved, # noqa: F401 — re-export - classify_industry_end_use, # noqa: F401 — re-export - table_2_4_prices_cents_kwh, -) from bedrock.transform.eeio.waste_disaggregation import ( apply_waste_disagg_to_U, apply_waste_disagg_to_V, @@ -205,8 +190,16 @@ def derive_disagg_io_bundle() -> CornerstoneDisaggIOBundle: Udom, Uimp = derive_cornerstone_U_after_waste() VA = derive_cornerstone_VA_after_waste() if electricity_reallocation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + reallocate_electricity_coproduction, + ) + V, Udom, Uimp, VA = reallocate_electricity_coproduction(V, Udom, Uimp, VA) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + disaggregate_electricity_make_use_va, + ) + V, Udom, Uimp, VA = disaggregate_electricity_make_use_va(V, Udom, Uimp, VA) return CornerstoneDisaggIOBundle(V=V, Udom=Udom, Uimp=Uimp, VA=VA) @@ -222,6 +215,11 @@ def derive_disagg_Ytot_with_trade() -> pd.DataFrame: Ytot = apply_waste_disagg_to_Ytot(Ytot, weights) Ytot.index.name = 'sector' if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + disaggregate_electricity_commodity_row_in_y, + get_electricity_commodity_row_weights, + ) + w_row = get_electricity_commodity_row_weights() Ytot = disaggregate_electricity_commodity_row_in_y(Ytot, w_row) Ytot.index.name = 'sector' @@ -251,6 +249,10 @@ def distribute_waste_parent_x_using_v_row_shares( for code in present: x.loc[code] = parent_go * float(shares.loc[code]) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + distribute_electricity_aggregate_x_using_v_row_shares, + ) + return distribute_electricity_aggregate_x_using_v_row_shares( x, derive_disagg_io_bundle().V ) @@ -262,8 +264,85 @@ def electricity_mixed_units_enabled() -> bool: return get_usa_config().implement_electricity_mixed_units +# --- Lazy end-use facade (importing this module must not load elec) ------------- + + +def build_end_use_map() -> dict[str, str]: + """Lazy re-export of ``electricity_end_use_mapping.build_end_use_map``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + build_end_use_map as _impl, + ) + + return _impl() + + +def table_2_4_prices_cents_kwh( + year: int, + provider: str | None = None, + *, + fba: pd.DataFrame | None = None, +) -> dict[str, float]: + """Lazy re-export of ``electricity_end_use_mapping.table_2_4_prices_cents_kwh``. + + Kept as a module-level callable so + ``@patch('…cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh')`` continues + to intercept calls from ``electricity_conversion_factors``. + """ + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + TABLE_2_4_PROVIDER, + table_2_4_prices_cents_kwh as _impl, + ) + + return cast( + dict[str, float], + _impl( + year, + TABLE_2_4_PROVIDER if provider is None else provider, + fba=fba, + ), + ) + + +def classify_industry_end_use(industry_code: str) -> tuple[str, str]: + """Lazy re-export of ``electricity_end_use_mapping.classify_industry_end_use``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + classify_industry_end_use as _impl, + ) + + return _impl(industry_code) + + +def build_end_use_map_resolved( + prices_by_class: dict[str, float] | None = None, + *, + c_col: float | None = None, + c_row: pd.Series[float] | None = None, +) -> pd.DataFrame: + """Lazy re-export of ``electricity_end_use_mapping.build_end_use_map_resolved``.""" + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + build_end_use_map_resolved as _impl, + ) + + return _impl(prices_by_class, c_col=c_col, c_row=c_row) + + +def __getattr__(name: str) -> Any: + """Lazy attribute access for end-use constants re-exported from this module.""" + if name == 'END_USE_MAPPING_REVIEW_STATUS': + from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 + END_USE_MAPPING_REVIEW_STATUS, + ) + + return END_USE_MAPPING_REVIEW_STATUS + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + def _model_year_y_row_221110(aq_scaled: SingleRegionAqMatrixSet) -> pd.Series[float]: """Model-year 221110 FD row from backcompute total × 2017 share split.""" + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + GENERATION_SECTOR, + ) + y_2017 = derive_disagg_Ytot_with_trade().loc[GENERATION_SECTOR] y_total = float( backcompute_y_from_A_and_q(A=aq_scaled.Adom, q=aq_scaled.scaled_q).loc[ @@ -287,7 +366,14 @@ def electricity_conversion_factors( from bedrock.extract.disaggregation.egrid_generation import ( # noqa: PLC0415 us_total_net_generation_mwh, ) + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + GENERATION_SECTOR, + electricity_class_row_factors, + electricity_output_factor, + ) + # Call module-level facade for end-use helpers so unittest patches on + # ``cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh`` still apply. cfg = get_usa_config() q_usd = float(aq_scaled.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) @@ -320,6 +406,11 @@ def build_electricity_mixed_units_aq( """Return mixed-unit A/q when gate is on; else pass-through.""" if not electricity_mixed_units_enabled(): return aq_scaled + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_A, + apply_electricity_unit_conversion_to_q, + ) + c_col, c_row = electricity_conversion_factors( aq_scaled, prices_by_class=prices_by_class ) @@ -344,6 +435,10 @@ def build_electricity_mixed_units_b( """Return mixed-unit B when gate is on; else pass-through.""" if not electricity_mixed_units_enabled(): return b + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_B, + ) + return apply_electricity_unit_conversion_to_B(b, c_col) @@ -365,6 +460,10 @@ def compute_mixed_unit_ef_vectors( prices_by_class: Mapping[str, float] | None = None, ) -> MixedUnitEfResult: """Apply mixed conversion to monetary scaled A/q/B; never use cached mixed derives.""" + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_unit_conversion_to_A, + apply_electricity_unit_conversion_to_B, + ) from bedrock.utils.math.formulas import ( # noqa: PLC0415 compute_d, compute_L_matrix, diff --git a/bedrock/transform/eeio/cornerstone_year_scaling.py b/bedrock/transform/eeio/cornerstone_year_scaling.py index c5e9efa5..204e9a69 100644 --- a/bedrock/transform/eeio/cornerstone_year_scaling.py +++ b/bedrock/transform/eeio/cornerstone_year_scaling.py @@ -142,11 +142,12 @@ def scale_cornerstone_A( from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, ) - from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_A, - ) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_d7_scaling_correction_to_A, + ) + A_scaled = apply_electricity_d7_scaling_correction_to_A( A_scaled, original_year, target_year ) @@ -182,11 +183,12 @@ def scale_cornerstone_q( from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, ) - from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_q, - ) if electricity_disaggregation_enabled(): + from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 + apply_electricity_d7_scaling_correction_to_q, + ) + q_scaled = apply_electricity_d7_scaling_correction_to_q( q_scaled, original_year, target_year ) From 8571e092f010ab3c4628260046922ca6f461d17b Mon Sep 17 00:00:00 2001 From: jvendries Date: Thu, 30 Jul 2026 15:45:44 -0400 Subject: [PATCH 2/3] Rename electricity specific functions with clearer names --- .../plan/electricity-pipeline-decoupling.md | 151 ------------------ .../electricity/class_prices/compare_paths.py | 4 +- .../d_85/__tests__/test_disagg_scenarios.py | 6 +- .../d_85/__tests__/test_eia_inputs.py | 6 +- .../electricity/d_85/disagg_scenarios.py | 4 +- .../analysis/electricity/d_85/eia_inputs.py | 4 +- .../electricity/d_85/end_use_mapping.py | 4 +- .../monetary_disagg/monetary_disagg_report.md | 7 +- .../eia_anchored_td_markup_counterfactual.py | 4 +- .../full_trace/decompose_d_n_step.py | 4 +- .../hh_mwh_driver_decomposition.py | 4 +- ...able_2_2_unit_conversion_counterfactual.py | 4 +- bedrock/publish/cache_reset.py | 9 +- .../test_electricity_disaggregation.py | 14 +- .../__tests__/test_electricity_mixed_units.py | 6 +- .../eeio/cornerstone_disagg_pipeline.py | 21 ++- .../eeio/cornerstone_year_scaling.py | 14 +- .../eeio/electricity_disaggregation.py | 29 ++-- .../eeio/electricity_end_use_mapping.py | 4 +- .../test_calculate_ef_diagnostics.py | 2 +- .../validation/calculate_ef_diagnostics.py | 4 +- 21 files changed, 78 insertions(+), 227 deletions(-) delete mode 100644 .claude/plan/electricity-pipeline-decoupling.md diff --git a/.claude/plan/electricity-pipeline-decoupling.md b/.claude/plan/electricity-pipeline-decoupling.md deleted file mode 100644 index f5ed833e..00000000 --- a/.claude/plan/electricity-pipeline-decoupling.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -name: Elec Pipeline Decoupling -overview: Structurally decouple electricity reallocation/disaggregation/mixed-units from the main Cornerstone EEIO path so canonical v0.3 (and waste-only configs) never import elec modules, while flag-on configs keep working via lazy imports. -todos: - - id: save-plan-md - content: Write plan copy to .claude/plan/electricity-pipeline-decoupling.md - status: completed - - id: lazy-cdp - content: Lazy-import elec in cornerstone_disagg_pipeline; keep waste + gates; lazy facade for end-use re-exports - status: completed - - id: lazy-year-scaling - content: Nest D7 imports inside electricity_disaggregation_enabled() in cornerstone_year_scaling - status: completed - - id: lazy-derived-cornerstone - content: Optional clarity — lazy mixed-units imports in derived_cornerstone (not required once cdp is decoupled) - status: cancelled - - id: cache-reset - content: Clear elec caches via sys.modules.get only if already loaded; never import elec under v0.3 - status: completed - - id: allocation-soft - content: Optional — demote CORNERSTONE_INDUSTRIES_ELEC hygiene (schemas only; not elec-module coupling) - status: cancelled - - id: verify - content: v0.3 import + cache_reset probe without elec modules; flag-on + existing tests green - status: completed -isProject: false ---- - -# Structural decoupling of electricity from main Cornerstone pipeline - -## Goal - -Canonical `2025_usa_cornerstone_v0_3` (all `implement_electricity_*` false; waste disagg on) must import and run without loading [`electricity_disaggregation.py`](bedrock/transform/eeio/electricity_disaggregation.py) or [`electricity_end_use_mapping.py`](bedrock/transform/eeio/electricity_end_use_mapping.py). Waste-only disagg stays. Flag-on configs keep current behavior via lazy imports. - -Also save this plan to [`.claude/plan/electricity-pipeline-decoupling.md`](.claude/plan/electricity-pipeline-decoupling.md) when implementing. - -## Current coupling (why delete-as-is fails) - -```mermaid -flowchart TD - derived["derived.py"] --> dc["derived_cornerstone.py"] - dc --> cdp["cornerstone_disagg_pipeline.py"] - cdp -->|"top-level import"| ed["electricity_disaggregation.py"] - cdp -->|"top-level import"| eum["electricity_end_use_mapping.py"] - ys["cornerstone_year_scaling.py"] -->|"import then if flag"| ed - cr["publish/cache_reset.py"] -->|"top-level import"| ed - cr --> cdp - cr --> derived -``` - -Runtime call sites are already gated. The break is **import-time**. - -## Approach - -Lazy-import electricity symbols **inside** flag-true branches (and inside mixed-units-only entry points). Keep gate helpers (`electricity_*_enabled`) and waste orchestration in [`cornerstone_disagg_pipeline.py`](bedrock/transform/eeio/cornerstone_disagg_pipeline.py). No taxonomy/schema rewrite. - -**Preferred re-export strategy:** keep `build_end_use_map` / `table_2_4_prices_cents_kwh` (and other end-use re-exports used by callers) as a **lazy facade on cdp** — defining/calling them must not load elec at `import cdp` time; the first call loads `electricity_end_use_mapping`. That preserves: - -- Production importers (e.g. [`calculate_ef_diagnostics.py`](bedrock/utils/validation/calculate_ef_diagnostics.py) ~272–276) -- Test patches on `cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh` ([`test_electricity_mixed_units.py`](bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py) ~186/272/305; [`test_calculate_ef_diagnostics.py`](bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py) ~218) -- Analysis / diagnostics imports of cdp re-exports (out of scope for retarget, but must not AttributeError) - -Do **not** retarget every caller unless the facade approach fails; if retarget is chosen instead, update every production/test `@patch` path listed above. - -## Implementation - -### 1. [`cornerstone_disagg_pipeline.py`](bedrock/transform/eeio/cornerstone_disagg_pipeline.py) — main cut - -Remove top-level imports from `electricity_disaggregation` and `electricity_end_use_mapping` (lines ~35–54 today). Keep waste imports and `electricity_*_enabled()` / `cornerstone_sector_disagg_active()` (config-only). - -**Lazy-import inventory (symbol → function):** - -| Symbol(s) | Import inside | -|---|---| -| `reallocate_electricity_coproduction` | `derive_disagg_io_bundle` — under `if electricity_reallocation_enabled()` | -| `disaggregate_electricity_make_use_va` | `derive_disagg_io_bundle` — under `if electricity_disaggregation_enabled()` | -| `get_electricity_commodity_row_weights`, `disaggregate_electricity_commodity_row_in_y` | `derive_disagg_Ytot_with_trade` — under disagg `if` | -| `distribute_electricity_aggregate_x_using_v_row_shares` | `distribute_waste_parent_x_using_v_row_shares` — under disagg `if` (~253) | -| `GENERATION_SECTOR` | `_model_year_y_row_221110` (~265) and any other user of the constant | -| `GENERATION_SECTOR`, `electricity_output_factor`, `electricity_class_row_factors` | `electricity_conversion_factors` (~281) — local-import at function entry | -| *(call only, do not local-import)* `table_2_4_prices_cents_kwh`, `build_end_use_map` | `electricity_conversion_factors` — **must** use module-level lazy facade names so `@patch('…cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh')` still works | -| `apply_electricity_unit_conversion_to_A/q` | `build_electricity_mixed_units_aq` (~315) — nest import **after** `electricity_mixed_units_enabled()` early return | -| `apply_electricity_unit_conversion_to_B` | `build_electricity_mixed_units_b` (~340) — same (after early return) | -| unit-conversion helpers (+ conversion factors path) | `compute_mixed_unit_ef_vectors` (~361) | - -**Lazy facade for end-use re-exports** (required for patch targets): keep module-level names `build_end_use_map`, `table_2_4_prices_cents_kwh`, and existing F401 re-exports (`END_USE_MAPPING_REVIEW_STATUS`, `build_end_use_map_resolved`, `classify_industry_end_use`) as **explicit thin wrappers** (prefer over `__getattr__` alone) that import from `electricity_end_use_mapping` on first call. Importing cdp must not load elec; `from cdp import table_2_4_…` / `@patch` on the cdp attribute must bind real callables. Do **not** local-import those two symbols inside `electricity_conversion_factors`. - -### 2. [`cornerstone_year_scaling.py`](bedrock/transform/eeio/cornerstone_year_scaling.py) - -In `scale_cornerstone_A` and `scale_cornerstone_q`, D7 correction imports are already function-local (~142–152, ~182–190) but run **before** the flag check. Nest the `from …electricity_disaggregation import …` **inside** `if electricity_disaggregation_enabled():`. Gate helper import from cdp stays (safe once cdp is decoupled). - -### 3. [`derived_cornerstone.py`](bedrock/transform/eeio/derived_cornerstone.py) - -**Not required for import-time decoupling** once cdp drops top-level elec imports: `from cdp import build_electricity_mixed_units_aq` no longer loads elec. Optional clarity: lazy-import mixed-units builders / `electricity_conversion_factors` inside the mixed-units derive functions (~942–950, ~1037). Keeping `electricity_mixed_units_enabled` as a top-level cdp import is fine (config-only). - -Top-level import from cdp for waste/disagg routing stays: `cornerstone_sector_disagg_active`, `derive_disagg_io_bundle`, `derive_disagg_Ytot_with_trade`, `distribute_waste_parent_x_using_v_row_shares`. - -### 4. [`publish/cache_reset.py`](bedrock/publish/cache_reset.py) - -- Remove top-level imports of elec weight builders / checkpoints from `electricity_disaggregation` (~47–52). -- Stop requiring top-level `build_end_use_map` / `table_2_4_prices_cents_kwh` from cdp for the clear list if those are not `@cache`’d (today they are listed in `UPSTREAM_CACHED_DERIVES` but clearing them is a no-op if not cached). Prefer clearing end-use mapping caches only if present on the loaded module. -- Always clear waste + core cornerstone / `derived` caches (unchanged). -- **Clear elec `@functools.cache` only if already loaded** — never `import` / try-import under v0.3: - -```python -import sys - -def _clear_cached_attrs(mod, names: tuple[str, ...]) -> None: - for name in names: - fn = getattr(mod, name, None) - if hasattr(fn, 'cache_clear'): - fn.cache_clear() - -ed = sys.modules.get('bedrock.transform.eeio.electricity_disaggregation') -if ed is not None: - _clear_cached_attrs(ed, ( - 'get_electricity_commodity_row_weights', - '_derive_post_reallocation_checkpoint_for_disagg', - 'build_electricity_disagg_use_intersection_weights', - 'build_electricity_ugo305_scaling_ratios', - # …any other cached elec derives currently in UPSTREAM_CACHED_DERIVES - )) -eum = sys.modules.get('bedrock.transform.eeio.electricity_end_use_mapping') -if eum is not None: - _clear_cached_attrs(eum, (...)) # only if any become cached -``` - -Flag-gated import is wrong for multi-config processes: after an elec run, leftover caches must still clear when switching to v0.3; `sys.modules.get` handles that without reloading. - -### 5. Soft cleanup — [`allocation/derived.py`](bedrock/transform/allocation/derived.py) (optional / demoted) - -`CORNERSTONE_INDUSTRIES_ELEC` is imported from [`cornerstone_schemas`](bedrock/utils/schemas/cornerstone_schemas.py) (~16), **not** from elec modules — this does **not** load `electricity_disaggregation.py`. Only ~486 uses the name; ~243/~391 only check the flag. Optional hygiene (`active_cornerstone_industries()` or defer import); **not** part of the decoupling acceptance probe. - -### Out of scope - -- Analysis / diagnostics packages (may keep hard imports of elec or cdp facade). -- Schema constants `ELECTRICITY_DISAGG_SECTORS` / `CORNERSTONE_*_ELEC` (harmless when unused; schemas ≠ elec module load). -- Behavior changes when any electricity flag is True. -- Retargeting analysis callers away from cdp re-exports (facade preserves them). - -## Acceptance / testing - -1. **Import + cache_reset probe (v0.3):** with elec modules blocked (rename or `sys.modules` sentinel so import fails): - - `from bedrock.transform.eeio.derived import derive_Aq_usa` - - `from bedrock.publish.cache_reset import clear_all_publish_caches` then call it - - install `2025_usa_cornerstone_v0_3` and call `derive_cornerstone_Aq_scaled()` (or `derive_Aq_usa`) - - Assert `electricity_disaggregation` and `electricity_end_use_mapping` are **absent** from `sys.modules` -2. **Multi-config cache clear:** under a process that previously loaded elec (flag-on), `clear_all_publish_caches()` still clears elec `@cache`s via `sys.modules` even when current config is v0.3 / flags false. -3. **Flag-on smoke:** `test_usa_config_waste_disagg_electricity_disaggregation` and mixed-units config still produce 407 / mixed paths; existing patches on `cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh` still work (facade). -4. **Regression:** `test_electricity_disaggregation.py`, `test_electricity_reallocation.py`, `test_electricity_mixed_units.py`, waste pipeline tests; publish CLI / helpers that call `clear_all_publish_caches` still succeed under v0.3. -5. No intentional behavior change when flags are off. diff --git a/bedrock/analysis/electricity/class_prices/compare_paths.py b/bedrock/analysis/electricity/class_prices/compare_paths.py index b103344d..9787c539 100644 --- a/bedrock/analysis/electricity/class_prices/compare_paths.py +++ b/bedrock/analysis/electricity/class_prices/compare_paths.py @@ -9,7 +9,7 @@ from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( MixedUnitEfResult, compute_mixed_unit_ef_vectors, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.transform.eeio.derived_cornerstone import ( derive_cornerstone_Aq_scaled, @@ -31,7 +31,7 @@ class ClassPricesComparison: def _equal_prices_from_table() -> dict[str, float]: cfg = get_usa_config() - prices = table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year) + prices = electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year) total = float(prices['Total']) return {k: total for k in prices} diff --git a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py index 151d57c9..48f0924c 100644 --- a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py @@ -19,7 +19,7 @@ from bedrock.analysis.electricity.d_85.disagg_weights import ( build_ugo_col_table83_row_intersection_matrix, ) -from bedrock.analysis.electricity.d_85.eia_inputs import table_2_4_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import electricity_end_use_retail_prices_cents_kwh from bedrock.transform.eeio.electricity_disaggregation import ELECTRICITY_AGGREGATE from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS @@ -92,7 +92,7 @@ def test_t83_production_diag_step2_uses_table83_weights( @mock.patch( - 'bedrock.analysis.electricity.d_85.disagg_scenarios.table_2_4_prices_cents_kwh' + 'bedrock.analysis.electricity.d_85.disagg_scenarios.electricity_end_use_retail_prices_cents_kwh' ) @mock.patch('bedrock.analysis.electricity.d_85.disagg_scenarios.build_end_use_map') @mock.patch('bedrock.analysis.electricity.d_85.disagg_scenarios.ugo305_go_weights') @@ -109,7 +109,7 @@ def test_p24_weights_sum_to_one_per_column( checkpoint_mock.return_value = (V, Udom, Uimp, VA, Y) ugo_mock.return_value = pd.Series({'221110': 0.34, '221121': 0.04, '221122': 0.62}) map_mock.return_value = {'541000': 'Commercial', 'F01000': 'Residential'} - prices_mock.return_value = table_2_4_prices_cents_kwh( + prices_mock.return_value = electricity_end_use_retail_prices_cents_kwh( 2017, fba=mock_fba_table83_table24() ) diff --git a/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py b/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py index 345c6f4e..1c50592d 100644 --- a/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py +++ b/bedrock/analysis/electricity/d_85/__tests__/test_eia_inputs.py @@ -6,7 +6,7 @@ import pytest from bedrock.analysis.electricity.d_85.eia_inputs import ( - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, table_8_3_gtd_expenses_musd, table_8_3_purchased_power_gtd_expenses_musd, ) @@ -28,8 +28,8 @@ def test_table_8_3_purchased_power_gtd_expenses_musd(mock_fba: pd.DataFrame) -> assert out['Distribution'] == pytest.approx(4358.0) -def test_table_2_4_prices_cents_kwh(mock_fba: pd.DataFrame) -> None: - prices = table_2_4_prices_cents_kwh(2017, fba=mock_fba) +def test_electricity_end_use_retail_prices_cents_kwh(mock_fba: pd.DataFrame) -> None: + prices = electricity_end_use_retail_prices_cents_kwh(2017, fba=mock_fba) assert prices['Residential'] == pytest.approx(12.89) assert prices['Total'] == pytest.approx(10.54) diff --git a/bedrock/analysis/electricity/d_85/disagg_scenarios.py b/bedrock/analysis/electricity/d_85/disagg_scenarios.py index 49ebe11a..0347bc19 100644 --- a/bedrock/analysis/electricity/d_85/disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/disagg_scenarios.py @@ -12,7 +12,7 @@ table83_purchased_power_weights, ugo305_go_weights, ) -from bedrock.analysis.electricity.d_85.eia_inputs import table_2_4_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import electricity_end_use_retail_prices_cents_kwh from bedrock.analysis.electricity.d_85.end_use_mapping import ( build_end_use_map, build_price_tilt_weights_by_column, @@ -326,7 +326,7 @@ def _weights_t83_purchased_power_diag_compensated() -> ScenarioWeights: def _weights_p24(price_year: int) -> ScenarioWeights: w_ugo = ugo305_go_weights() - raw_prices = table_2_4_prices_cents_kwh(price_year) + raw_prices = electricity_end_use_retail_prices_cents_kwh(price_year) prices: dict[str, float] = {str(k): float(v) for k, v in raw_prices.items()} end_use_map = build_end_use_map() _, Udom, Uimp, _, Y = derive_post_reallocation_checkpoint() diff --git a/bedrock/analysis/electricity/d_85/eia_inputs.py b/bedrock/analysis/electricity/d_85/eia_inputs.py index 32b99025..60a243da 100644 --- a/bedrock/analysis/electricity/d_85/eia_inputs.py +++ b/bedrock/analysis/electricity/d_85/eia_inputs.py @@ -13,7 +13,7 @@ TABLE_2_4_DESCRIPTION, TABLE_2_4_PROVIDER, EPAEndUse, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) TABLE_8_3_DESCRIPTION = ( @@ -108,7 +108,7 @@ def table_8_3_purchased_power_gtd_expenses_musd( 'TABLE_8_3_GENERATION_FLOWNAMES', 'TABLE_8_3_PRODUCER', 'TABLE_8_3_SHARED_FLOWNAMES', - 'table_2_4_prices_cents_kwh', + 'electricity_end_use_retail_prices_cents_kwh', 'table_8_3_gtd_expenses_musd', 'table_8_3_purchased_power_gtd_expenses_musd', ] diff --git a/bedrock/analysis/electricity/d_85/end_use_mapping.py b/bedrock/analysis/electricity/d_85/end_use_mapping.py index 3193507b..a70b48d2 100644 --- a/bedrock/analysis/electricity/d_85/end_use_mapping.py +++ b/bedrock/analysis/electricity/d_85/end_use_mapping.py @@ -14,7 +14,7 @@ build_end_use_map, build_end_use_map_resolved, classify_industry_end_use, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS @@ -26,7 +26,7 @@ 'build_end_use_map_resolved', 'build_price_tilt_weights_by_column', 'classify_industry_end_use', - 'table_2_4_prices_cents_kwh', + 'electricity_end_use_retail_prices_cents_kwh', 'write_default_overrides_csv', ] diff --git a/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md b/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md index 69a238f9..88899765 100644 --- a/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md +++ b/bedrock/analysis/electricity/monetary_disagg/monetary_disagg_report.md @@ -175,8 +175,9 @@ Indirect fuel-chain emissions remain upstream (fuels assigned to generation in s After disaggregation, BEA summary sector `"22"` (Utilities) would apply one price index to all three children. Production applies the standard summary scaling, then a **D7 correction** using UGO305 detail gross-output ratios between `original_year` and -`target_year` per child (`build_electricity_ugo305_scaling_ratios` / -`apply_electricity_d7_scaling_correction_to_A` and `_q` in +`target_year` per child (`build_electricity_detail_GO_growth_ratios` / +`rescale_electricity_children_to_detail_GO_growth_A` / +`rescale_electricity_children_to_detail_GO_growth_q` in `cornerstone_year_scaling.py`). For 2017 → 2022, detail GO ratios differentiate G/T/D (generation ~1.62×, transmission @@ -287,6 +288,6 @@ Runtime ~1–2 minutes on a typical developer machine. | 4 | `disaggregate_use_industry_columns`, `_enforce_go_identity_precondition` | | 5 | `get_electricity_commodity_row_weights`, `disaggregate_use_commodity_rows`, Y split in `cornerstone_disagg_pipeline.py` | | 6 | `_apply_electricity_disagg_cornerstone_mapping` in `derived.py` | -| 7 | `build_electricity_ugo305_scaling_ratios`, `apply_electricity_d7_scaling_correction_*` in `cornerstone_year_scaling.py` | +| 7 | `build_electricity_detail_GO_growth_ratios`, `rescale_electricity_children_to_detail_GO_growth_*` in `cornerstone_year_scaling.py` | All in `bedrock/transform/eeio/electricity_disaggregation.py` unless noted. diff --git a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py index dc17ae6f..8b4266a0 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py @@ -59,7 +59,7 @@ ) from bedrock.transform.eeio.electricity_end_use_mapping import ( build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.config.usa_config import reset_usa_config, set_global_usa_config from bedrock.utils.schemas.cornerstone_schemas import ( @@ -337,7 +337,7 @@ def analyze() -> dict[str, Any]: egrid_mwh = float(us_total_net_generation_mwh(model_year)) end_use_map = build_end_use_map() - prices = cast(dict[str, float], table_2_4_prices_cents_kwh(ghg_year)) + prices = cast(dict[str, float], electricity_end_use_retail_prices_cents_kwh(ghg_year)) eia = _eia_table_2_2_sales_mwh(model_year) w_go = build_electricity_disagg_go_weights() w_trans, w_dist = _td_national_shares() diff --git a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py index 19d12c11..69dc06fc 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py @@ -753,7 +753,7 @@ def _conversion_factor_detail(config: str) -> dict[str, Any]: ) from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) reset_usa_config() @@ -764,7 +764,7 @@ def _conversion_factor_detail(config: str) -> dict[str, Any]: q_usd = float(aq.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) c_col = electricity_output_factor(q_usd, mwh) - prices = cast(dict[str, float], table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year)) + prices = cast(dict[str, float], electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year)) end_use_map = build_end_use_map() y_row = _model_year_y_row_221110(aq) adom_row = cast(pd.Series, aq.Adom.loc[GENERATION_SECTOR]) diff --git a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py index 0b245103..086f9d20 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/hh_mwh_driver_decomposition.py @@ -39,7 +39,7 @@ from bedrock.transform.eeio.electricity_end_use_mapping import ( EPA_END_USES, build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS from bedrock.utils.taxonomy.cornerstone.commodities import COMMODITY_DESC @@ -504,7 +504,7 @@ def analyze() -> dict[str, Any]: eia = _eia_values(int(cfg.model_base_year)) prices = cast( Mapping[str, float], - table_2_4_prices_cents_kwh(int(cfg.usa_ghg_data_year)), + electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)), ) return { diff --git a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py index de45f17c..9c5354c9 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py @@ -50,7 +50,7 @@ from bedrock.transform.eeio.electricity_end_use_mapping import ( EPA_END_USES, build_end_use_map, - table_2_4_prices_cents_kwh, + electricity_end_use_retail_prices_cents_kwh, ) from bedrock.utils.math.formulas import ( backcompute_y_from_A_and_q, @@ -232,7 +232,7 @@ def analyze() -> dict[str, Any]: intermediate_usd, y_row_usd, end_use_map, eia_classes ) prices_2_4 = cast( - dict[str, float], table_2_4_prices_cents_kwh(int(cfg.usa_ghg_data_year)) + dict[str, float], electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)) ) # Production path (Table 2.4 + eGRID total) diff --git a/bedrock/publish/cache_reset.py b/bedrock/publish/cache_reset.py index 955d841c..6f6d8b1a 100644 --- a/bedrock/publish/cache_reset.py +++ b/bedrock/publish/cache_reset.py @@ -62,7 +62,7 @@ 'get_electricity_commodity_row_weights', '_derive_post_reallocation_checkpoint_for_disagg', 'build_electricity_disagg_use_intersection_weights', - 'build_electricity_ugo305_scaling_ratios', + 'build_electricity_detail_GO_growth_ratios', 'build_electricity_disagg_go_weights', ) @@ -107,8 +107,9 @@ def _clear_cached_attrs(mod: ModuleType, names: tuple[str, ...]) -> None: for name in names: fn = getattr(mod, name, None) - if hasattr(fn, 'cache_clear'): - fn.cache_clear() + cache_clear = getattr(fn, 'cache_clear', None) + if callable(cache_clear): + cache_clear() def _clear_electricity_caches_if_loaded() -> None: @@ -127,7 +128,7 @@ def _clear_electricity_caches_if_loaded() -> None: eum, ( 'build_end_use_map', - 'table_2_4_prices_cents_kwh', + 'electricity_end_use_retail_prices_cents_kwh', ), ) diff --git a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py index 3cff672d..9fab73dc 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_disaggregation.py @@ -35,11 +35,11 @@ _compute_w_row, _derive_post_reallocation_checkpoint_for_disagg, _float_ndarray, - _table83_purchased_power_expenses, - _weights_from_table83_expenses, + _iou_utility_gtd_operating_expenses, + _normalize_gtd_expense_weights, + build_electricity_detail_GO_growth_ratios, build_electricity_disagg_go_weights, build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, disaggregate_use_industry_columns, get_electricity_commodity_row_weights, ) @@ -63,7 +63,7 @@ derive_disagg_Ytot_with_trade, build_electricity_disagg_go_weights, build_electricity_disagg_use_intersection_weights, - build_electricity_ugo305_scaling_ratios, + build_electricity_detail_GO_growth_ratios, get_electricity_commodity_row_weights, _derive_post_reallocation_checkpoint_for_disagg, derive_cornerstone_V, @@ -219,10 +219,10 @@ def test_purchased_power_weights_sum_to_one(self) -> None: mock_fba_table83_table24, ) - expenses = _table83_purchased_power_expenses( + expenses = _iou_utility_gtd_operating_expenses( 2017, fba=mock_fba_table83_table24() ) - w = _weights_from_table83_expenses(expenses) + w = _normalize_gtd_expense_weights(expenses) assert set(w.index) == set(ELECTRICITY_DISAGG_SECTORS) np.testing.assert_allclose(float(w.sum()), 1.0, rtol=1e-9, atol=1e-12) assert w['221110'] == pytest.approx(49030.0 / (49030.0 + 10804.0 + 4358.0)) @@ -274,7 +274,7 @@ def test_differentiated_child_q_scaling( _setup_config(electricity_disagg_config) try: aq = derive_cornerstone_Aq_scaled() - ratios = build_electricity_ugo305_scaling_ratios(2017, 2022) + ratios = build_electricity_detail_GO_growth_ratios(2017, 2022) q_vals = [float(aq.scaled_q[c]) for c in ELECTRICITY_DISAGG_SECTORS] assert len(set(round(v, 6) for v in q_vals)) == 3 assert ratios['221110'] != pytest.approx(ratios['221121']) diff --git a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py index e25e6635..97290229 100644 --- a/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py +++ b/bedrock/transform/eeio/__tests__/test_electricity_mixed_units.py @@ -183,7 +183,7 @@ def test_mixed_units_flag_off_is_noop() -> None: return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_output_mwh_anchor( mock_prices: Mock, @@ -269,7 +269,7 @@ def test_y_nab_stays_monetary_under_mixed_gate(mixed_units_config: str) -> None: return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_y_nab_mixed_differs_from_monetary_under_gate( mock_prices: Mock, @@ -302,7 +302,7 @@ def test_y_nab_mixed_differs_from_monetary_under_gate( return_value=4_000_000_000.0, ) @patch( - 'bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh', + 'bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh', ) def test_d_scalar_bridge_under_gate( mock_prices: Mock, diff --git a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py index 05148206..db5714bd 100644 --- a/bedrock/transform/eeio/cornerstone_disagg_pipeline.py +++ b/bedrock/transform/eeio/cornerstone_disagg_pipeline.py @@ -276,28 +276,27 @@ def build_end_use_map() -> dict[str, str]: return _impl() -def table_2_4_prices_cents_kwh( +def electricity_end_use_retail_prices_cents_kwh( year: int, provider: str | None = None, *, fba: pd.DataFrame | None = None, ) -> dict[str, float]: - """Lazy re-export of ``electricity_end_use_mapping.table_2_4_prices_cents_kwh``. + """Lazy re-export of ``electricity_end_use_mapping.electricity_end_use_retail_prices_cents_kwh``. Kept as a module-level callable so - ``@patch('…cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh')`` continues + ``@patch('…cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh')`` continues to intercept calls from ``electricity_conversion_factors``. """ - from bedrock.transform.eeio.electricity_end_use_mapping import ( # noqa: PLC0415 - TABLE_2_4_PROVIDER, - table_2_4_prices_cents_kwh as _impl, + from bedrock.transform.eeio import ( # noqa: PLC0415 + electricity_end_use_mapping as eum, ) return cast( dict[str, float], - _impl( + eum.electricity_end_use_retail_prices_cents_kwh( year, - TABLE_2_4_PROVIDER if provider is None else provider, + eum.TABLE_2_4_PROVIDER if provider is None else provider, fba=fba, ), ) @@ -373,15 +372,13 @@ def electricity_conversion_factors( ) # Call module-level facade for end-use helpers so unittest patches on - # ``cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh`` still apply. + # ``cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh`` still apply. cfg = get_usa_config() q_usd = float(aq_scaled.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) c_col = electricity_output_factor(q_usd, mwh) if prices_by_class is None: - prices = cast( - dict[str, float], table_2_4_prices_cents_kwh(cfg.usa_ghg_data_year) - ) + prices = electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year) else: prices = dict(prices_by_class) end_use_map = build_end_use_map() diff --git a/bedrock/transform/eeio/cornerstone_year_scaling.py b/bedrock/transform/eeio/cornerstone_year_scaling.py index 204e9a69..360598da 100644 --- a/bedrock/transform/eeio/cornerstone_year_scaling.py +++ b/bedrock/transform/eeio/cornerstone_year_scaling.py @@ -135,9 +135,9 @@ def scale_cornerstone_A( for col in oob_idx: A_scaled[col] *= 0.98 / total_industry_inputs[col] - assert ( - compute_total_industry_inputs(A=A_scaled) <= 1 - ).all(), 'A column sums exceed 1 after scaling.' + assert (compute_total_industry_inputs(A=A_scaled) <= 1).all(), ( + 'A column sums exceed 1 after scaling.' + ) from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, @@ -145,10 +145,10 @@ def scale_cornerstone_A( if electricity_disaggregation_enabled(): from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_A, + rescale_electricity_children_to_detail_GO_growth_A, ) - A_scaled = apply_electricity_d7_scaling_correction_to_A( + A_scaled = rescale_electricity_children_to_detail_GO_growth_A( A_scaled, original_year, target_year ) @@ -186,10 +186,10 @@ def scale_cornerstone_q( if electricity_disaggregation_enabled(): from bedrock.transform.eeio.electricity_disaggregation import ( # noqa: PLC0415 - apply_electricity_d7_scaling_correction_to_q, + rescale_electricity_children_to_detail_GO_growth_q, ) - q_scaled = apply_electricity_d7_scaling_correction_to_q( + q_scaled = rescale_electricity_children_to_detail_GO_growth_q( q_scaled, original_year, target_year ) diff --git a/bedrock/transform/eeio/electricity_disaggregation.py b/bedrock/transform/eeio/electricity_disaggregation.py index 9691b176..e9bcdebe 100644 --- a/bedrock/transform/eeio/electricity_disaggregation.py +++ b/bedrock/transform/eeio/electricity_disaggregation.py @@ -316,7 +316,7 @@ def build_electricity_disagg_go_weights() -> pd.Series[float]: ) -def _table83_purchased_power_expenses( +def _iou_utility_gtd_operating_expenses( year: int, *, fba: pd.DataFrame | None = None, @@ -348,7 +348,7 @@ def _table83_purchased_power_expenses( return out -def _weights_from_table83_expenses(expenses: dict[str, float]) -> pd.Series[float]: +def _normalize_gtd_expense_weights(expenses: dict[str, float]) -> pd.Series[float]: keys = list(expenses.keys()) if len(keys) != 3: raise ValueError(f'expected 3 expense buckets, got {keys!r}') @@ -369,8 +369,8 @@ def _weights_from_table83_expenses(expenses: dict[str, float]) -> pd.Series[floa @functools.cache def build_electricity_disagg_use_intersection_weights() -> pd.Series[float]: """Return Table 8.3 Purchased Power + T/D shares for step 2 intersection.""" - expenses = _table83_purchased_power_expenses(IO_ACCOUNT_YEAR) - return _weights_from_table83_expenses(expenses) + expenses = _iou_utility_gtd_operating_expenses(IO_ACCOUNT_YEAR) + return _normalize_gtd_expense_weights(expenses) def _diagonal_intersection_weights(w: pd.Series[float]) -> pd.DataFrame: @@ -913,7 +913,7 @@ def _go_levels_for_year(year: int) -> pd.Series[float]: @functools.cache -def build_electricity_ugo305_scaling_ratios( +def build_electricity_detail_GO_growth_ratios( original_year: int, target_year: int, ) -> pd.Series[float]: @@ -924,8 +924,11 @@ def build_electricity_ugo305_scaling_ratios( return ratios.fillna(1.0).reindex(ELECTRICITY_DISAGG_SECTORS) -def utilities_summary_ratio_22(original_year: int, target_year: int) -> float: - """Summary Utilities sector-22 q ratio used as D7 base scaling factor.""" +def utilities_summary_q_growth_ratio(original_year: int, target_year: int) -> float: + """BEA summary Make-Use-Trade (MUT) Utilities sector-22 q growth ratio. + + Used as the denominator when rescaling electricity children to detail GO growth. + """ orig = cast(USA_SUMMARY_MUT_YEARS, original_year) tgt = cast(USA_SUMMARY_MUT_YEARS, target_year) ratio = (derive_summary_q_usa(tgt) / derive_summary_q_usa(orig)).fillna(1.0) @@ -933,14 +936,14 @@ def utilities_summary_ratio_22(original_year: int, target_year: int) -> float: return val if np.isfinite(val) else 1.0 -def apply_electricity_d7_scaling_correction_to_A( +def rescale_electricity_children_to_detail_GO_growth_A( a: pd.DataFrame, original_year: int, target_year: int, ) -> pd.DataFrame: """Rescale electricity child rows after summary-ratio A scaling (D7 pure).""" - ratios = build_electricity_ugo305_scaling_ratios(original_year, target_year) - base = utilities_summary_ratio_22(original_year, target_year) + ratios = build_electricity_detail_GO_growth_ratios(original_year, target_year) + base = utilities_summary_q_growth_ratio(original_year, target_year) out = a.copy() for code in ELECTRICITY_DISAGG_SECTORS: if code not in out.index: @@ -950,14 +953,14 @@ def apply_electricity_d7_scaling_correction_to_A( return out -def apply_electricity_d7_scaling_correction_to_q( +def rescale_electricity_children_to_detail_GO_growth_q( q: pd.Series[float], original_year: int, target_year: int, ) -> pd.Series[float]: """Rescale electricity child q rows after summary-ratio q scaling (D7 pure).""" - ratios = build_electricity_ugo305_scaling_ratios(original_year, target_year) - base = utilities_summary_ratio_22(original_year, target_year) + ratios = build_electricity_detail_GO_growth_ratios(original_year, target_year) + base = utilities_summary_q_growth_ratio(original_year, target_year) out = q.copy() for code in ELECTRICITY_DISAGG_SECTORS: if code not in out.index: diff --git a/bedrock/transform/eeio/electricity_end_use_mapping.py b/bedrock/transform/eeio/electricity_end_use_mapping.py index cf78c40b..49fb1a65 100644 --- a/bedrock/transform/eeio/electricity_end_use_mapping.py +++ b/bedrock/transform/eeio/electricity_end_use_mapping.py @@ -170,13 +170,13 @@ def _load_eia_fba(year: int) -> pd.DataFrame: return getFlowByActivity('EIA_ElectricPowerAnnual', year) -def table_2_4_prices_cents_kwh( +def electricity_end_use_retail_prices_cents_kwh( year: int, provider: str = TABLE_2_4_PROVIDER, *, fba: pd.DataFrame | None = None, ) -> dict[EPAEndUse, float]: - """Return end-use retail prices (cents/kWh) from Table 2.4.""" + """source is EIA Electric Power Annual Table 2.4 (Average price to ultimate customers); used to build class-specific MWh/$ factors.""" df = fba if fba is not None else _load_eia_fba(year) mask = ( (df['Year'] == year) diff --git a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py index 51997d02..da13f992 100644 --- a/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/__tests__/test_calculate_ef_diagnostics.py @@ -215,7 +215,7 @@ def mock_load(name: str) -> pd.DataFrame: "bedrock.transform.eeio.cornerstone_disagg_pipeline.compute_mixed_unit_ef_vectors", ) as mock_uniform, patch( - "bedrock.transform.eeio.cornerstone_disagg_pipeline.table_2_4_prices_cents_kwh", + "bedrock.transform.eeio.cornerstone_disagg_pipeline.electricity_end_use_retail_prices_cents_kwh", return_value={"Total": 10.0}, ), patch( diff --git a/bedrock/utils/validation/calculate_ef_diagnostics.py b/bedrock/utils/validation/calculate_ef_diagnostics.py index f3d4233c..229eaae9 100644 --- a/bedrock/utils/validation/calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/calculate_ef_diagnostics.py @@ -271,8 +271,8 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 compute_mixed_unit_ef_vectors, + electricity_end_use_retail_prices_cents_kwh, electricity_mixed_units_enabled, - table_2_4_prices_cents_kwh, ) from bedrock.transform.eeio.derived import ( # noqa: PLC0415 derive_B_usa_non_finetuned, @@ -301,7 +301,7 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: n_mon = ta.cast('pd.Series[float]', compute_n(M=m_mon).squeeze()) d_mix = _ef_vector_as_series(efs.D_new) n_mix = _ef_vector_as_series(efs.N_new) - table_prices = table_2_4_prices_cents_kwh(config.usa_ghg_data_year) + table_prices = electricity_end_use_retail_prices_cents_kwh(config.usa_ghg_data_year) total_price = float(table_prices['Total']) equal_prices: dict[str, float] = {str(k): total_price for k in table_prices} uniform_result = compute_mixed_unit_ef_vectors( From f2b60503e431e1e59bd1da09412fffadac5a0bce Mon Sep 17 00:00:00 2001 From: jvendries Date: Thu, 30 Jul 2026 15:49:40 -0400 Subject: [PATCH 3/3] Linting --- .../electricity/d_85/__tests__/test_disagg_scenarios.py | 4 +++- bedrock/analysis/electricity/d_85/disagg_scenarios.py | 4 +++- .../eia_anchored_td_markup_counterfactual.py | 4 +++- .../full_trace/decompose_d_n_step.py | 5 ++++- .../table_2_2_unit_conversion_counterfactual.py | 3 ++- bedrock/transform/eeio/cornerstone_year_scaling.py | 6 +++--- bedrock/utils/validation/calculate_ef_diagnostics.py | 4 +++- 7 files changed, 21 insertions(+), 9 deletions(-) diff --git a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py index 48f0924c..dbef3121 100644 --- a/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/__tests__/test_disagg_scenarios.py @@ -19,7 +19,9 @@ from bedrock.analysis.electricity.d_85.disagg_weights import ( build_ugo_col_table83_row_intersection_matrix, ) -from bedrock.analysis.electricity.d_85.eia_inputs import electricity_end_use_retail_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import ( + electricity_end_use_retail_prices_cents_kwh, +) from bedrock.transform.eeio.electricity_disaggregation import ELECTRICITY_AGGREGATE from bedrock.utils.schemas.cornerstone_schemas import ELECTRICITY_DISAGG_SECTORS diff --git a/bedrock/analysis/electricity/d_85/disagg_scenarios.py b/bedrock/analysis/electricity/d_85/disagg_scenarios.py index 0347bc19..880790d4 100644 --- a/bedrock/analysis/electricity/d_85/disagg_scenarios.py +++ b/bedrock/analysis/electricity/d_85/disagg_scenarios.py @@ -12,7 +12,9 @@ table83_purchased_power_weights, ugo305_go_weights, ) -from bedrock.analysis.electricity.d_85.eia_inputs import electricity_end_use_retail_prices_cents_kwh +from bedrock.analysis.electricity.d_85.eia_inputs import ( + electricity_end_use_retail_prices_cents_kwh, +) from bedrock.analysis.electricity.d_85.end_use_mapping import ( build_end_use_map, build_price_tilt_weights_by_column, diff --git a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py index 8b4266a0..ac357867 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/alternate_eia_anchored_split/eia_anchored_td_markup_counterfactual.py @@ -337,7 +337,9 @@ def analyze() -> dict[str, Any]: egrid_mwh = float(us_total_net_generation_mwh(model_year)) end_use_map = build_end_use_map() - prices = cast(dict[str, float], electricity_end_use_retail_prices_cents_kwh(ghg_year)) + prices = cast( + dict[str, float], electricity_end_use_retail_prices_cents_kwh(ghg_year) + ) eia = _eia_table_2_2_sales_mwh(model_year) w_go = build_electricity_disagg_go_weights() w_trans, w_dist = _td_national_shares() diff --git a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py index 69dc06fc..38916b52 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/full_trace/decompose_d_n_step.py @@ -764,7 +764,10 @@ def _conversion_factor_detail(config: str) -> dict[str, Any]: q_usd = float(aq.scaled_q[GENERATION_SECTOR]) mwh = float(us_total_net_generation_mwh(cfg.model_base_year)) c_col = electricity_output_factor(q_usd, mwh) - prices = cast(dict[str, float], electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year)) + prices = cast( + dict[str, float], + electricity_end_use_retail_prices_cents_kwh(cfg.usa_ghg_data_year), + ) end_use_map = build_end_use_map() y_row = _model_year_y_row_221110(aq) adom_row = cast(pd.Series, aq.Adom.loc[GENERATION_SECTOR]) diff --git a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py index 9c5354c9..9d372ec1 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/hh_vs_interindustry/table_2_2_unit_conversion_counterfactual.py @@ -232,7 +232,8 @@ def analyze() -> dict[str, Any]: intermediate_usd, y_row_usd, end_use_map, eia_classes ) prices_2_4 = cast( - dict[str, float], electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)) + dict[str, float], + electricity_end_use_retail_prices_cents_kwh(int(cfg.usa_ghg_data_year)), ) # Production path (Table 2.4 + eGRID total) diff --git a/bedrock/transform/eeio/cornerstone_year_scaling.py b/bedrock/transform/eeio/cornerstone_year_scaling.py index 360598da..e09bb87c 100644 --- a/bedrock/transform/eeio/cornerstone_year_scaling.py +++ b/bedrock/transform/eeio/cornerstone_year_scaling.py @@ -135,9 +135,9 @@ def scale_cornerstone_A( for col in oob_idx: A_scaled[col] *= 0.98 / total_industry_inputs[col] - assert (compute_total_industry_inputs(A=A_scaled) <= 1).all(), ( - 'A column sums exceed 1 after scaling.' - ) + assert ( + compute_total_industry_inputs(A=A_scaled) <= 1 + ).all(), 'A column sums exceed 1 after scaling.' from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 electricity_disaggregation_enabled, diff --git a/bedrock/utils/validation/calculate_ef_diagnostics.py b/bedrock/utils/validation/calculate_ef_diagnostics.py index 229eaae9..acdfeb84 100644 --- a/bedrock/utils/validation/calculate_ef_diagnostics.py +++ b/bedrock/utils/validation/calculate_ef_diagnostics.py @@ -301,7 +301,9 @@ def calculate_ef_diagnostics(sheet_id: str) -> None: n_mon = ta.cast('pd.Series[float]', compute_n(M=m_mon).squeeze()) d_mix = _ef_vector_as_series(efs.D_new) n_mix = _ef_vector_as_series(efs.N_new) - table_prices = electricity_end_use_retail_prices_cents_kwh(config.usa_ghg_data_year) + table_prices = electricity_end_use_retail_prices_cents_kwh( + config.usa_ghg_data_year + ) total_price = float(table_prices['Total']) equal_prices: dict[str, float] = {str(k): total_price for k in table_prices} uniform_result = compute_mixed_unit_ef_vectors(