Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ These flags are grouped into themes:

| Theme | Example flags |
|---|---|
| Schema / taxonomy | `use_cornerstone_2026_model_schema`, `implement_waste_disaggregation`, `implement_electricity_reallocation` |
| Schema / taxonomy | `implement_waste_disaggregation`, `implement_electricity_reallocation` |
| Economic IOT (input-output tables) | `apply_io_year_adjustments`, `iot_before_or_after_redefinition` |
| GHG attribution | `use_cornerstone_ghg_model` |
| Data vintage | `model_base_year`, `usa_base_io_data_year`, `ipcc_ar_version` |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@
_weighted_ef,
)
from bedrock.publish.model_objects import get_B, get_D, get_L, get_N, get_q
from bedrock.transform.allocation.derived import derive_E_usa
from bedrock.transform.eeio.cornerstone_disagg_pipeline import (
electricity_conversion_factors,
electricity_mixed_units_enabled,
)
from bedrock.transform.eeio.derived import derive_E_usa
from bedrock.transform.eeio.derived_cornerstone import (
derive_cornerstone_Aq_mixed_units,
derive_cornerstone_Aq_scaled,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@

from bedrock.analysis.electricity_disagg_diagnostics.paths import OUT_DIR
from bedrock.publish.model_objects import get_B, get_D, get_L, get_N, get_q
from bedrock.transform.allocation.derived import derive_E_usa
from bedrock.transform.eeio.cornerstone_disagg_pipeline import (
derive_disagg_Ytot_with_trade,
electricity_conversion_factors,
electricity_mixed_units_enabled,
)
from bedrock.transform.eeio.derived import derive_E_usa
from bedrock.transform.eeio.derived_cornerstone import (
derive_cornerstone_Aq_scaled,
derive_cornerstone_U_set,
Expand Down
54 changes: 5 additions & 49 deletions bedrock/extract/allocation/__tests__/test_bea_use_table_schema.py
Original file line number Diff line number Diff line change
@@ -1,64 +1,20 @@
"""Tests for schema-aligned BEA use table (load_bea_use_table)."""
"""Tests for the Cornerstone-frame BEA use table (load_bea_use_table)."""

from __future__ import annotations

from unittest.mock import patch

import pytest

from bedrock.extract.allocation.bea import (
_load_bea_use_table_cached,
load_bea_use_table,
)
from bedrock.utils.config.usa_config import USAConfig
from bedrock.utils.taxonomy.bea.ceda_v7 import CEDA_V7_SECTORS
from bedrock.extract.allocation.bea import load_bea_use_table
from bedrock.utils.taxonomy.cornerstone.industries import INDUSTRIES


def _clear_use_table_cache() -> None:
_load_bea_use_table_cached.cache_clear()


@pytest.mark.eeio_integration
def test_load_bea_use_table_ceda_shape() -> None:
"""With CEDA config, table has CEDA v7 industry rows + PCE row."""
_clear_use_table_cache()
config = USAConfig(use_cornerstone_2026_model_schema=False)
with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config):
table = load_bea_use_table()
# Rows = CEDA industries + one PCE row
industry_rows = [i for i in table.index if i in CEDA_V7_SECTORS]
assert len(industry_rows) == len(CEDA_V7_SECTORS)
assert "221200" in table.columns
assert table.shape[0] >= len(CEDA_V7_SECTORS)
assert table.shape[1] > 0


@pytest.mark.eeio_integration
def test_load_bea_use_table_cornerstone_shape() -> None:
"""With Cornerstone config, table has Cornerstone industry rows + PCE row."""
_clear_use_table_cache()
config = USAConfig(use_cornerstone_2026_model_schema=True)
with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config):
table = load_bea_use_table()
"""Table has Cornerstone industry rows + PCE row."""
load_bea_use_table.cache_clear()
table = load_bea_use_table()
industry_rows = [i for i in table.index if i in INDUSTRIES]
assert len(industry_rows) == len(INDUSTRIES)
assert "221200" in table.columns
assert table.shape[0] >= len(INDUSTRIES)
assert table.shape[1] > 0


@pytest.mark.eeio_integration
def test_load_bea_use_table_cache_per_schema() -> None:
"""CEDA and Cornerstone calls return different shapes (cache keyed by schema)."""
_clear_use_table_cache()
config_ceda = USAConfig(use_cornerstone_2026_model_schema=False)
config_cs = USAConfig(use_cornerstone_2026_model_schema=True)
with patch(
"bedrock.extract.allocation.bea.get_usa_config", return_value=config_ceda
):
table_ceda = load_bea_use_table()
with patch("bedrock.extract.allocation.bea.get_usa_config", return_value=config_cs):
table_cs = load_bea_use_table()
assert table_ceda.shape[0] != table_cs.shape[0]
assert set(table_ceda.index) != set(table_cs.index)
53 changes: 18 additions & 35 deletions bedrock/extract/allocation/bea.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
import pandas as pd

from bedrock.transform.eeio.derived_2017 import (
derive_2017_U_set_usa,
derive_2017_V_usa,
derive_2017_Y_personal_consumption_expenditure_usa,
)
from bedrock.transform.eeio.derived_cornerstone import (
derive_cornerstone_U_set,
derive_cornerstone_Y_personal_consumption_expenditure,
)
from bedrock.utils.config.usa_config import get_usa_config
from bedrock.utils.io.gcp import load_from_gcs
Expand Down Expand Up @@ -45,34 +47,16 @@ def load_bea_make_table() -> pd.DataFrame:


@functools.cache
def _load_bea_use_table_cached(use_cornerstone: bool) -> pd.DataFrame:
"""Inner loader keyed by schema so both CEDA and Cornerstone can be cached."""
if use_cornerstone:
from bedrock.transform.eeio import derived_cornerstone # noqa: PLC0415

uset = derived_cornerstone.derive_cornerstone_U_set()
U_combined = (uset.Udom + uset.Uimp).T
Y_cs = (
derived_cornerstone.derive_cornerstone_Y_personal_consumption_expenditure()
.to_frame()
.T
)
return pd.concat([U_combined, Y_cs])
U_set = derive_2017_U_set_usa()
Y_usa = derive_2017_Y_personal_consumption_expenditure_usa().to_frame()
return pd.concat([(U_set.Udom + U_set.Uimp).T, Y_usa.T])


def load_bea_use_table() -> pd.DataFrame:
"""
Load BEA Use and Final Demand tables aligned to the model schema.
Load BEA Use and Final Demand tables in the Cornerstone frame.

When use_cornerstone_2026_model_schema is False, returns CEDA v7 industry rows.
When True, returns Cornerstone industry rows (from derive_cornerstone_U_set and Y).
Rows = industries + one PCE row; columns = commodities. Result is cached per schema.
Rows = Cornerstone industries + one PCE row; columns = commodities.
"""
use_cornerstone = get_usa_config().use_cornerstone_2026_model_schema
return _load_bea_use_table_cached(use_cornerstone)
uset = derive_cornerstone_U_set()
U_combined = (uset.Udom + uset.Uimp).T
Y_cs = derive_cornerstone_Y_personal_consumption_expenditure().to_frame().T
return pd.concat([U_combined, Y_cs])


def _use_table_value_ceda_sector_cornerstone_aligned(
Expand All @@ -81,12 +65,11 @@ def _use_table_value_ceda_sector_cornerstone_aligned(
ceda_sector: str,
) -> float:
"""
Value for one CEDA allocator sector from a use table (CEDA or Cornerstone shaped).
Value for one CEDA-vocabulary allocator sector from the Cornerstone use table.

When the table is CEDA-shaped (Cornerstone schema not active), the sector is
in the index and we return it directly; alignment rules are skipped. When the
table is Cornerstone-shaped, we apply alignment: 562* → 562000, 335220 ↔ 4
appliance sectors, 331313 → 331313+33131B.
Sectors present in the index are returned directly; otherwise alignment
rules apply: 562* → 562000, 335220 ↔ 4 appliance sectors,
331313 → 331313+33131B.
"""
if ceda_sector in table_idx:
return float(col.loc[ceda_sector])
Expand Down Expand Up @@ -120,11 +103,11 @@ def use_table_series_ceda_allocator_to_cornerstone_schema(
commodity: str,
) -> pd.Series:
"""
Use-table series for CEDA allocator sectors, aligned to Cornerstone schema.
Use-table series for CEDA-vocabulary allocator sectors, aligned to the
Cornerstone-shaped use table.

When the use table is CEDA-shaped (Cornerstone schema not active), sectors
are looked up directly; alignment is skipped. When Cornerstone-shaped,
alignment applies: 562* → 562000, 335220 ↔ 4 appliance sectors, 331313 →
Sectors present in the table are looked up directly; otherwise alignment
applies: 562* → 562000, 335220 ↔ 4 appliance sectors, 331313 →
331313+33131B. Missing sectors get 0. Safe to normalize (e.g. pct = s / s.sum()).
"""
table_idx = use_table.index
Expand Down
3 changes: 1 addition & 2 deletions bedrock/publish/emission_factors/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
build_purchaser_matrices,
finalize_cornerstone_ef_table,
)
from bedrock.publish.model_objects import apply_loc_suffix, require_cornerstone_config
from bedrock.publish.model_objects import apply_loc_suffix

logger = logging.getLogger(__name__)

Expand All @@ -24,7 +24,6 @@ def write_emission_factors(
write_matrices: bool = False,
) -> dict[str, str]:
"""Write CO2e SEF CSV (and optional M/N purchaser matrices) under ``output_dir``."""
require_cornerstone_config()
os.makedirs(output_dir, exist_ok=True)

table = finalize_cornerstone_ef_table(
Expand Down
5 changes: 0 additions & 5 deletions bedrock/publish/excel/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@
get_Udom,
get_V,
get_x,
require_cornerstone_config,
)
from bedrock.utils.config.settings import GIT_HASH_LONG
from bedrock.utils.config.usa_config import get_usa_config
Expand Down Expand Up @@ -370,11 +369,7 @@ def write_model_to_xlsx(out_path: str, *, config_name: str) -> None:
Sheets are produced from `_build_matrix_registry(config_name)`. Any
registry entry whose getter returns `None` is omitted from the
workbook (useeior-style "skip if NULL").

Raises `NotImplementedError` on legacy (non-cornerstone) configs --
publish is wired only for cornerstone-schema configs today.
"""
require_cornerstone_config()
registry = _build_matrix_registry(config_name)
materialized = _materialize(registry)
os.makedirs(os.path.dirname(out_path) or '.', exist_ok=True)
Expand Down
9 changes: 0 additions & 9 deletions bedrock/publish/model_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import pandas as pd

from bedrock.utils.config.usa_config import get_usa_config
from bedrock.utils.math.formulas import compute_L_matrix, compute_M_matrix

PUBLISH_LOCATION: str = 'US'
Expand Down Expand Up @@ -67,14 +66,6 @@ def assemble_extended_U(
return out


def require_cornerstone_config() -> None:
if not get_usa_config().use_cornerstone_2026_model_schema:
raise NotImplementedError(
'bedrock.publish only supports cornerstone-schema configs '
'(use_cornerstone_2026_model_schema=True).'
)


@functools.cache
def get_V() -> pd.DataFrame:
from bedrock.transform.eeio.derived_cornerstone import derive_cornerstone_V
Expand Down
14 changes: 3 additions & 11 deletions bedrock/transform/allocation/co2/industrial_coal.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,9 @@
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)
from bedrock.transform.allocation.mappings.v7.ceda_mecs import (
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
NON_MECS_INDUSTRIES,
)
from bedrock.transform.allocation.utils import get_allocation_sectors
from bedrock.utils.config.usa_config import get_usa_config
from bedrock.utils.economic.units import COAL_MMBTU_PER_SHORT_TONNE, MEGATONNE_TO_KG

load_table_a17_tbtu = functools.cache(_load_table_a17_tbtu)
Expand All @@ -38,15 +35,10 @@ def _get_mecs_3_1_naics_mappings() -> tuple[
dict[tuple[str, ...], tuple[str, ...]],
dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]],
]:
"""Return (mapping, subtraction_mapping) for MECS 3.1 NAICS; use CORNERSTONE when schema flag is on."""
if get_usa_config().use_cornerstone_2026_model_schema:
return (
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)
"""Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 3.1 NAICS."""
return (
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)


Expand Down
14 changes: 3 additions & 11 deletions bedrock/transform/allocation/co2/industrial_natural_gas.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,9 @@
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)
from bedrock.transform.allocation.mappings.v7.ceda_mecs import (
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
NON_MECS_INDUSTRIES,
)
from bedrock.transform.allocation.utils import get_allocation_sectors
from bedrock.utils.config.usa_config import get_usa_config
from bedrock.utils.economic.units import MEGATONNE_TO_KG, NAT_GAS_BCF_TO_TRILLION_BTU

load_table_a17_tbtu = functools.cache(_load_table_a17_tbtu)
Expand All @@ -47,15 +44,10 @@ def _get_mecs_3_1_naics_mappings() -> tuple[
dict[tuple[str, ...], tuple[str, ...]],
dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]],
]:
"""Return (mapping, subtraction_mapping) for MECS 3.1 NAICS; use CORNERSTONE when schema flag is on."""
if get_usa_config().use_cornerstone_2026_model_schema:
return (
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)
"""Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 3.1 NAICS."""
return (
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_3_1_NAICS_SUBTRACTION_MAPPING,
)


Expand Down
16 changes: 3 additions & 13 deletions bedrock/transform/allocation/co2/non_energy_fuels_natural_gas.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,7 @@
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING,
)
from bedrock.transform.allocation.mappings.v7.ceda_mecs import (
CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING,
)
from bedrock.transform.allocation.utils import get_allocation_sectors
from bedrock.utils.config.usa_config import get_usa_config
from bedrock.utils.economic.units import MEGATONNE_TO_KG

logger = logging.getLogger(__name__)
Expand All @@ -32,15 +27,10 @@ def _get_mecs_2_1_naics_mappings() -> tuple[
dict[tuple[str, ...], tuple[str, ...]],
dict[tuple[str, ...], tuple[tuple[str, ...], tuple[str, ...]]],
]:
"""Return (mapping, subtraction_mapping) for MECS 2.1 NAICS; use CORNERSTONE when schema flag is on."""
if get_usa_config().use_cornerstone_2026_model_schema:
return (
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING,
)
"""Return the Cornerstone-frame (mapping, subtraction_mapping) for MECS 2.1 NAICS."""
return (
CEDA_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING,
CEDA_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_MAPPING,
CORNERSTONE_INDUSTRY_TO_MECS_2_1_NAICS_SUBTRACTION_MAPPING,
)


Expand Down
Loading
Loading