From f3a4fa2f61302a39f5424da9f5e0bb3892e3eb67 Mon Sep 17 00:00:00 2001 From: briantobin-99 Date: Thu, 30 Jul 2026 04:38:29 -0700 Subject: [PATCH] chore(config): delete A-scaling alternatives, nowcast, and time-series analysis Co-Authored-By: Claude Fable 5 --- .../skills/dispatch-ef-diagnostics/SKILL.md | 20 +- .claude/skills/plot-ef-diagnostics/SKILL.md | 3 - .github/workflows/run_a_matrix_analysis.yml | 138 ---- README.md | 2 +- .../analysis/a_matrix_time_series/README.md | 209 ------ .../analysis/a_matrix_time_series/__init__.py | 25 - .../analysis/a_matrix_time_series/_loaders.py | 20 - .../a_matrix_time_series/_run_report.py | 48 -- .../compare_approaches.py | 273 -------- .../compare_key_sectors.py | 522 --------------- .../compare_method_stability.py | 543 --------------- .../compare_price_ratios.py | 151 ----- .../compare_summary_a_errors.py | 311 --------- .../compile_ef_diagnostics.py | 338 ---------- .../a_matrix_time_series/constants.py | 92 --- .../derive_A_cells_long.py | 632 ------------------ .../derive_A_cells_stability.py | 617 ----------------- .../derive_A_time_series.py | 294 -------- .../derive_useeio_nowcast_A.py | 84 --- .../dispatch_ef_time_series.py | 422 ------------ .../docs/analysis_plan.md | 406 ----------- .../docs/implement_useeio_nowcast_plan.md | 245 ------- .../plot_ef_diagnostics.py | 327 --------- .../plot_v0_3_n_pct_hist.py | 180 ----- .../recover_ef_run_index.py | 221 ------ .../summary_a_and_q_errors.py | 498 -------------- .../useeio_nowcast_ef_runbook.md | 98 --- .../view_vnorm_and_price_ratios.py | 71 -- .../electricity/d_85/scenario_ef_pipeline.py | 8 +- .../year_alignment/year_alignment_bly_e.py | 2 - bedrock/extract/iot/useeio_nowcast.py | 101 --- .../__tests__/test_summary_a_realigned.py | 78 --- bedrock/transform/eeio/derived_cornerstone.py | 89 +-- .../transform/eeio/derived_useeio_nowcast.py | 212 ------ ...a_cornerstone_A_commodity_price_index.yaml | 6 - ...2025_usa_cornerstone_A_summary_tables.yaml | 4 - .../2025_usa_cornerstone_A_useeio.yaml | 4 - ...2025_usa_cornerstone_A_useeio_nowcast.yaml | 12 - .../2025_usa_cornerstone_taxonomy.yaml | 19 - ...nerstone_v0_2_A_commodity_price_index.yaml | 10 - ...usa_cornerstone_v0_2_A_summary_tables.yaml | 7 - .../2025_usa_cornerstone_v0_2_A_useeio.yaml | 7 - ...usa_cornerstone_v0_2_A_useeio_nowcast.yaml | 16 - .../config/configs/useeio_phoebe_23.yaml | 1 - bedrock/utils/config/usa_config.py | 4 - .../economic/inflation_helpers_cornerstone.py | 10 +- 46 files changed, 10 insertions(+), 7370 deletions(-) delete mode 100644 .github/workflows/run_a_matrix_analysis.yml delete mode 100644 bedrock/analysis/a_matrix_time_series/README.md delete mode 100644 bedrock/analysis/a_matrix_time_series/__init__.py delete mode 100644 bedrock/analysis/a_matrix_time_series/_loaders.py delete mode 100644 bedrock/analysis/a_matrix_time_series/_run_report.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compare_approaches.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compare_key_sectors.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compare_method_stability.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compare_price_ratios.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compare_summary_a_errors.py delete mode 100644 bedrock/analysis/a_matrix_time_series/compile_ef_diagnostics.py delete mode 100644 bedrock/analysis/a_matrix_time_series/constants.py delete mode 100644 bedrock/analysis/a_matrix_time_series/derive_A_cells_long.py delete mode 100644 bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py delete mode 100644 bedrock/analysis/a_matrix_time_series/derive_A_time_series.py delete mode 100644 bedrock/analysis/a_matrix_time_series/derive_useeio_nowcast_A.py delete mode 100644 bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py delete mode 100644 bedrock/analysis/a_matrix_time_series/docs/analysis_plan.md delete mode 100644 bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md delete mode 100644 bedrock/analysis/a_matrix_time_series/plot_ef_diagnostics.py delete mode 100644 bedrock/analysis/a_matrix_time_series/plot_v0_3_n_pct_hist.py delete mode 100644 bedrock/analysis/a_matrix_time_series/recover_ef_run_index.py delete mode 100644 bedrock/analysis/a_matrix_time_series/summary_a_and_q_errors.py delete mode 100644 bedrock/analysis/a_matrix_time_series/useeio_nowcast_ef_runbook.md delete mode 100644 bedrock/analysis/a_matrix_time_series/view_vnorm_and_price_ratios.py delete mode 100644 bedrock/extract/iot/useeio_nowcast.py delete mode 100644 bedrock/transform/eeio/__tests__/test_summary_a_realigned.py delete mode 100644 bedrock/transform/eeio/derived_useeio_nowcast.py delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_A_commodity_price_index.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_A_summary_tables.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_commodity_price_index.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_summary_tables.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio.yaml delete mode 100644 bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml diff --git a/.claude/skills/dispatch-ef-diagnostics/SKILL.md b/.claude/skills/dispatch-ef-diagnostics/SKILL.md index b4bd67d1..37f224ff 100644 --- a/.claude/skills/dispatch-ef-diagnostics/SKILL.md +++ b/.claude/skills/dispatch-ef-diagnostics/SKILL.md @@ -1,8 +1,7 @@ --- name: dispatch-ef-diagnostics -description: Use this skill to dispatch EF (emission-factor) diagnostics runs — create and name empty diagnostics Google Sheets, trigger generate_diagnostics, and record a run index. Covers the a_matrix_time_series epic matrix and generic feature-config lists via bedrock.utils.validation.dispatch_diagnostics. Trigger when the user says "dispatch the EF diagnostics", "kick off the diagnostics runs", "create the diagnostics sheets and run them", "trigger generate_diagnostics", "re-dispatch the failed cells", or asks to run the model under several configs and produce per-run diagnostics sheets. +description: Use this skill to dispatch EF (emission-factor) diagnostics runs — create and name empty diagnostics Google Sheets, trigger generate_diagnostics, and record a run index. Covers feature-config lists via bedrock.utils.validation.dispatch_diagnostics. Trigger when the user says "dispatch the EF diagnostics", "kick off the diagnostics runs", "create the diagnostics sheets and run them", "trigger generate_diagnostics", "re-dispatch the failed cells", or asks to run the model under several configs and produce per-run diagnostics sheets. disable-model-invocation: false -argument-hint: [scenarios] [years] [approaches] (e.g. "bundle_v0_3 2019-2023" or "isolate_a_matrix useeio_nowcast 2023") --- # Dispatch EF diagnostics runs @@ -11,7 +10,6 @@ argument-hint: [scenarios] [years] [approaches] (e.g. "bundle_v0_3 2019-2023" or Fan out the `generate_diagnostics` GitHub Actions workflow to produce **one diagnostics Google Sheet per cell**. Each sheet gets the `N_and_diffs` / `D_and_diffs` / `D_and_N_significant_sectors` / `config_summary` tabs that the `plot-ef-diagnostics` skill consumes. -For the **A-matrix time-series epic**, the driver is `bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py` (one sheet per `(scenario, approach, year)`). Per cell it: (1) **creates** a Sheet in the epic Drive folder with a deterministic title, (2) **triggers** `gh workflow run generate_diagnostics.yml`, (3) **records** a row in `output/results/ef_run_index.csv`. It is **idempotent** — cells already in the index are skipped, so re-running only fills gaps. Default baseline for that epic is **CEDA-US (v0)**. ## Prerequisites — verify, don't assume @@ -32,7 +30,6 @@ Do **not** create any sheet or dispatch until these are confirmed. Ask via `AskU | Detail | Default | |---|---| | Drive folder | `1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s` (`EF_TIME_SERIES_DRIVE_FOLDER_ID`) | -| Scenarios | `bundle_v0_3` (or `isolate_a_matrix`, or both) | | Approaches | all in the scenario (e.g. restrict to `useeio_nowcast`) | | Years | `2019,2020,2021,2022,2023` — sets `model_base_year` + `usa_ghg_data_year` | | Baseline | CEDA-only (`ceda-v0`); ask whether to use `useeio` or `v0.3` instead | @@ -45,7 +42,6 @@ Echo the resolved plan back (folder, cell count, baseline, git-ref) and get a go ## Bespoke config lists (the common real case) -Most real requests come as a **config-spec Google Sheet**, not the two canned scenarios. `SCENARIO_YAMLS` only knows `isolate_a_matrix` / `bundle_v0_3` (4 A-matrix YAMLs each) — an arbitrary release-progression list (`useeio_phoebe_23*`, `2025_usa_cornerstone_*`, `…_v0_3_*`) **can't be expressed via `--scenarios/--approaches`**. Don't force-fit it. Instead drive the dispatcher's **helper functions** with the custom list (reuse, don't reinvent): @@ -55,7 +51,6 @@ from bedrock.utils.validation.dispatch_diagnostics import ( V04_DIAGNOSTICS_DRIVE_FOLDER_ID, ) # A-matrix / release-progression folder stays on the epic dispatcher: -# from bedrock.analysis.a_matrix_time_series.dispatch_ef_time_series import ( # EF_TIME_SERIES_DRIVE_FOLDER_ID, # ) # per config: wait_for_capacity(...) → create_sheet(folder, title) → trigger_workflow(...) → persist a row @@ -99,22 +94,16 @@ These surface as a **failed GH run after a successful dispatch** — the sheet i 1. **Dry-run first** — prints the plan (titles + configs), creates/triggers nothing: ```bash - python -m bedrock.analysis.a_matrix_time_series.dispatch_ef_time_series \ - --git-ref main --scenarios bundle_v0_3 --years 2019,2020,2021,2022,2023 --dry-run ``` 2. **Dispatch for real** (drop `--dry-run`): ```bash - python -m bedrock.analysis.a_matrix_time_series.dispatch_ef_time_series \ --git-ref main \ - --scenarios isolate_a_matrix,bundle_v0_3 \ --years 2019,2020,2021,2022,2023 \ [--approaches useeio_nowcast] [--use-useeio-baseline] [--throttle poll|sleep:N|none] ``` 3. **Wait** for GH Actions (~2–5 min per run, serial). Watch with `gh run list --workflow generate_diagnostics.yml`. 4. **Compile + plot** (reviewer path — see the `plot-ef-diagnostics` skill): ```bash - python -m bedrock.analysis.a_matrix_time_series.compile_ef_diagnostics - python -m bedrock.analysis.a_matrix_time_series.plot_ef_diagnostics ``` ## Flags @@ -122,7 +111,6 @@ These surface as a **failed GH run after a successful dispatch** — the sheet i | Flag | Meaning | |---|---| | `--git-ref` | **Required.** Branch/tag the workflow runs against (usually `main`). | -| `--scenarios` | Comma list: `isolate_a_matrix`, `bundle_v0_3` (default `bundle_v0_3`). | | `--years` | Comma list (default `2019,2020,2021,2022,2023`). Sets `model_base_year` and `usa_ghg_data_year`. | | `--approaches` | Optional filter, e.g. `useeio_nowcast`. Default = all approaches in the scenario. | | `--use-useeio-baseline` | Epic dispatcher alias for `--baseline useeio`. | @@ -136,13 +124,10 @@ These surface as a **failed GH run after a successful dispatch** — the sheet i - **Sheet title:** `[{YYYY-MM-DD}, {year}, {baseline} based, {approach label}, {scenario}] EFs diagnostics`. - **Run index:** `output/results/ef_run_index.csv` — columns `scenario, approach, year, baseline, config_name, sheet_id, sheet_title, useeio_box_ticked, git_ref, triggered_at`. - **Scenario → YAML** (in `dispatch_ef_time_series.py`), keyed by approach: - - `isolate_a_matrix` (A-matrix method only, else v0 defaults): `2025_usa_cornerstone_A_{useeio,summary_tables,commodity_price_index,useeio_nowcast}`. - - `bundle_v0_3` (full v0.3 stack + one A-matrix alternative): `2025_usa_cornerstone_v0_2_A_{…}`. - **Approach labels** (title text): `useeio → "A matrix with 2017 benchmark A"`, `summary_tables → "A matrix with summary tables"`, `commodity_price_index → "A matrix with commodity price index"`, `useeio_nowcast → "A matrix from USEEIO nowcast"`. ## Recovery / utilities -- **Lost the local index?** Rebuild from Drive: `python -m bedrock.analysis.a_matrix_time_series.recover_ef_run_index --folder-id 1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s`. - **Batch hit rate limits?** `--re-dispatch-from-csv` re-triggers without minting new sheets. ## Manual fallback (no driver) @@ -156,9 +141,6 @@ gh workflow run generate_diagnostics.yml --ref main \ ## Reference -- Driver (A-matrix epic): `bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py` - Shared helpers + feature CLI: `bedrock/utils/validation/dispatch_diagnostics.py` - Workflow: `.github/workflows/generate_diagnostics.yml` → `bedrock/utils/validation/generate_diagnostics.py` (single-run entry) → `calculate_ef_diagnostics.py` (writes the tabs). -- Package overview + DAG: `bedrock/analysis/a_matrix_time_series/README.md`. -- Operator checklist: `bedrock/analysis/a_matrix_time_series/useeio_nowcast_ef_runbook.md`. - Feature-flag playbook: `bedrock/utils/validation/evaluate_feature_impact.md`. diff --git a/.claude/skills/plot-ef-diagnostics/SKILL.md b/.claude/skills/plot-ef-diagnostics/SKILL.md index 2fbb7630..dd9a5db5 100644 --- a/.claude/skills/plot-ef-diagnostics/SKILL.md +++ b/.claude/skills/plot-ef-diagnostics/SKILL.md @@ -40,7 +40,6 @@ For Recipe B, surface the **dollar-year guard** (below) and ask whether to proce - **Fetch (parquet-cached):** `from bedrock.utils.validation.analysis.fetch import load_tab` → `load_tab(sheet_id, "N_and_diffs")` (pass `refresh=True` to re-pull). - **Plot primitives:** `from bedrock.utils.validation.analysis.plotting import setup_mpl, percent_histogram, apply_axis_fonts, save_and_close, DEFAULT_XLIM, TITLE_FONTSIZE`. - **Deck-panel extractor + constants:** `from bedrock.utils.validation.analysis.ef_hist_panels import pct_values, draw_per_sector_pct_hist_panel, HIST_BINS, HIST_PCT_CLIP, HIST_FONT_SCALE, HIST_STATS_EXTRA_SCALE, PANEL_STATS_FONTSIZE, PANEL_TITLE_FONTSIZE, PANEL_AXIS_LABEL_FONTSIZE, PANEL_TICK_LABEL_FONTSIZE`. -- **Approach palette** (`from bedrock.analysis.a_matrix_time_series.constants import APPROACH_COLORS`): `useeio #7f7f7f`, `ceda_default #bcbd22`, `summary_tables #1f77b4`, `industry_price_index #9467bd`, `commodity_price_index #2ca02c`, `useeio_nowcast #ff7f0e`. ## Recipe A — single sheet, N/D vs CEDA v0 (full suite) @@ -63,7 +62,6 @@ Reuse `load_tab` + `percent_histogram`: produce a log-log scatter (`x = baseline Deck panels (e.g. "[CEDA as baseline] Bundled effect in N", titled by approach) use `draw_per_sector_pct_hist_panel` from `ef_hist_panels` (also used by `plot_v0_3_n_pct_hist` and A-matrix `plot_ef_diagnostics` histogram grids): each panel is one sheet's `N_perc_diff`, clipped to ±`HIST_PCT_CLIP` (100%), `HIST_BINS` (60) bins, zero line, `PercentFormatter` x-axis "Percentage Diff (%)", y "sector count", an `n / median / p95(|·|)` white box top-left, title + bar color from the approach. -- Single sheet, exact deck style: `python -m bedrock.analysis.a_matrix_time_series.plot_v0_3_n_pct_hist `. - **Color override** (the script colors by approach, grey for unrecognized configs): replicate the `_render` body in a small figure, pulling pct via `pct_values(load_tab(sid, "N_and_diffs"), "N")`, and pass an explicit color. **Orange = `#ff7f0e`** (the `useeio_nowcast` entry). - **Side-by-side** (e.g. "v0.2 vs new"): lay panels out 1×N reusing the same constants so output matches the deck. Keep v0.2 blue (`#1f77b4`); color the new scenario as requested. @@ -105,6 +103,5 @@ uv run python -m bedrock.utils.validation.analysis.overlay_ef_hist \ ## Reference - `bedrock/utils/validation/analysis/`: `diagnostics_plots.py` (single-sheet CLI), `overlay_ef_hist.py` (version-overlay CLI), `ef_hist_panels.py` (`pct_values`, `draw_per_sector_pct_hist_panel`), `bly_plots.py` (`build_sector_stack_frame`), `plotting.py` (primitives: `percent_histogram`, `overlay_pct_diff_histogram`, `plot_stacked_net_change`), `fetch.py` (cached loader). -- `bedrock/analysis/a_matrix_time_series/`: `plot_ef_diagnostics.py` (A-matrix scatter + histogram grids), `plot_v0_3_n_pct_hist.py` (single-sheet CLI), `constants.py` (`APPROACH_COLORS`). - `bedrock/utils/config/usa_config.py`: `_load_usa_config_from_file_name(".yaml")` to resolve a config's intrinsic dollar years. - Sheets come from the `dispatch-ef-diagnostics` skill / `generate_diagnostics` workflow. diff --git a/.github/workflows/run_a_matrix_analysis.yml b/.github/workflows/run_a_matrix_analysis.yml deleted file mode 100644 index bf40179c..00000000 --- a/.github/workflows/run_a_matrix_analysis.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: run_a_matrix_analysis - -on: - workflow_dispatch: - inputs: - pr_url: - description: "Optional PR URL to associate with this run" - required: false - type: string - default: "" - -env: - FORCE_COLOR: 2 - -jobs: - analysis: - runs-on: ubuntu-latest - timeout-minutes: 60 - permissions: - id-token: write - contents: read - outputs: - sheet_id: ${{ steps.capture_sheet.outputs.sheet_id }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python and install dependencies - uses: ./.github/actions/setup-python - - - name: Authenticate with Google Cloud - uses: "google-github-actions/auth@v2" - with: - project_id: cornerstone-data - workload_identity_provider: projects/75718421862/locations/global/workloadIdentityPools/github-actions/providers/github - service_account: github-actions-cornerstone@cornerstone-data.iam.gserviceaccount.com - - - name: Step 1 — derive A matrix time series - run: uv run python -m bedrock.analysis.a_matrix_time_series.derive_A_time_series - - - name: Step 2 — cell-by-cell diagnostics + plots - run: uv run python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_long - - - name: Step 2.5 — cell-level stability + persistence - run: uv run python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_stability - - - name: Capture run-report Sheet ID - id: capture_sheet - run: | - SHEET_ID_PATH="bedrock/analysis/a_matrix_time_series/output/results/last_run_sheet_id.txt" - if [ -f "$SHEET_ID_PATH" ]; then - echo "sheet_id=$(cat $SHEET_ID_PATH)" >> "$GITHUB_OUTPUT" - else - echo "sheet_id=" >> "$GITHUB_OUTPUT" - fi - - - name: Upload plots - uses: actions/upload-artifact@v4 - with: - name: run_a_matrix_analysis_plots_${{ github.sha }} - path: bedrock/analysis/a_matrix_time_series/output/plots/ - if-no-files-found: warn - - - name: Upload results CSVs - uses: actions/upload-artifact@v4 - with: - name: run_a_matrix_analysis_results_${{ github.sha }} - path: | - bedrock/analysis/a_matrix_time_series/output/results/*.csv - bedrock/analysis/a_matrix_time_series/output/results/last_run_sheet_id.txt - if-no-files-found: warn - - notify_slack_failure: - needs: analysis - runs-on: ubuntu-latest - if: always() && needs.analysis.result == 'failure' - - steps: - - name: Notify on Slack - failure - uses: slackapi/slack-github-action@v1.24.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} - with: - channel-id: C09S03U9CH4 # #alerts-bedrock - payload: | - { - "text": "Bedrock A-matrix Analysis Failure", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":warning: *Bedrock A-matrix Analysis Failure*\nFailed on `${{ github.ref_name }}` `${{ github.sha }}`" - }, - "accessory": { - "type": "button", - "text": { - "type": "plain_text", - "text": "View logs" - }, - "url": "${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" - } - } - ] - } - - notify_slack_success: - needs: analysis - runs-on: ubuntu-latest - if: always() && needs.analysis.result == 'success' - - steps: - - name: Notify on Slack - success - uses: slackapi/slack-github-action@v1.24.0 - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} - with: - channel-id: C09S03U9CH4 # #alerts-bedrock - payload: | - { - "text": "Bedrock A-matrix Analysis Success", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: *Bedrock A-matrix Analysis Success*\nCompleted on `${{ github.ref_name }}` `${{ github.sha }}`\n\n*Run-report Sheet:* https://docs.google.com/spreadsheets/d/${{ needs.analysis.outputs.sheet_id }}\n*Plots:* download via the run's Artifacts tab" - }, - "accessory": { - "type": "button", - "text": { - "type": "plain_text", - "text": "View logs" - }, - "url": "${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" - } - } - ] - } diff --git a/README.md b/README.md index af693c90..0d45017c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ See [`USAConfig`](bedrock/utils/config/usa_config.py) for the full list. All configuration files are in [`bedrock/utils/config/configs/`](bedrock/utils/config/configs/), where: - A single *full-model* config represents a full set of methodology choices made for a data release. [`2025_usa_cornerstone_v0_3.yaml`](bedrock/utils/config/configs/2025_usa_cornerstone_v0_3.yaml) is the default config in `get_usa_config()`. [`2025_usa_cornerstone_v0_2.yaml`](bedrock/utils/config/configs/2025_usa_cornerstone_v0_2.yaml) is the v0.2 methodology stack for historical comparison. -- Several *atomic configs* each isolate a single methodological change from the baseline so the impact of each choice can be measured independently. For example, [`2025_usa_cornerstone_taxonomy.yaml`](bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy.yaml) is the config for a specific choice to use Cornerstone taxonomy. +- Several *atomic configs* each isolate a single methodological change from the baseline so the impact of each choice can be measured independently (e.g. the `v03_waterfall_*` release-assessment configs). A separate `snapshot_version_or_git_sha` field specifies the baseline SHA, so diagnostic runs can compare current output against any released baseline. diff --git a/bedrock/analysis/a_matrix_time_series/README.md b/bedrock/analysis/a_matrix_time_series/README.md deleted file mode 100644 index 55fcdf61..00000000 --- a/bedrock/analysis/a_matrix_time_series/README.md +++ /dev/null @@ -1,209 +0,0 @@ -# `a_matrix_time_series` — A-matrix methodology comparison - -This module compares **six ways of deriving the Cornerstone A matrix** for years 2017–2024 and feeds the cell-level + EF-level diagnostics that back the v0.3 recommendation for which method ships in the 2026 model. Plain-English glossary of the six approaches lives in [`docs/analysis_plan.md`](docs/analysis_plan.md); the canonical list (with stable string keys used everywhere in this folder) is in [`constants.py`](constants.py). - -| Category | Approach key | Role | -|---|---|---| -| Baseline (do-nothing) | `useeio` | 2017 A used as-is | -| Baseline (production) | `ceda_default` | Current CEDA-US two-step scale + inflate | -| Internal alternative | `summary_tables` | Scale 2017 → target via summary-A ratios | -| Internal alternative | `industry_price_index` | Industry PI applied to 2017 A | -| Internal alternative | `commodity_price_index` | V-norm-derived commodity PI applied to 2017 A | -| External reference | `useeio_nowcast` | EPA USEEIO team's GRAS-balanced detail SUTs (2017–2023) | - -`FOCUS_APPROACHES` in [`constants.py`](constants.py) is the 3-approach subset the v0.3 plots zoom in on: `commodity_price_index`, `summary_tables`, `useeio_nowcast`. `industry_price_index` is kept in the data layer but omitted from the focus plots (superseded by `commodity_price_index`). - ---- - -## Where things live - -``` -a_matrix_time_series/ -├── README.md # this file — live navigation -├── useeio_nowcast_ef_runbook.md # live operator runbook (USEEIO-nowcast EF run) -├── docs/ -│ ├── analysis_plan.md # historical: epic spec (5 approaches, DoD, key questions) -│ └── implement_useeio_nowcast_plan.md # historical: integration of the 6th approach -│ -├── constants.py # paths, approach order, colors, year coverage -├── _loaders.py # load (Adom, Aimp) parquet pairs by (approach, year) -├── _run_report.py # publish result tabs to a Google Sheets run-report -├── __init__.py # pins update_inflation_factors=True + apply_inflation_to_V=True -│ -├── derive_A_time_series.py # Step 1: cache A_{approach}_{year}.parquet -├── derive_useeio_nowcast_A.py # Step 1 (external): cache A_useeio_nowcast_{year}.parquet -├── derive_A_cells_long.py # Step 2: A_cells_long.parquet + scatter/divergence -├── derive_A_cells_stability.py # Step 2.5: Jaccard + persistence (cell-set stability) -│ -├── compare_approaches.py # Step 3: pairwise hexbins at the latest year -├── compare_key_sectors.py # Step 4: impact-weighted top cells / heatmap -├── compare_summary_a_errors.py # Step 5: weighted RMSE vs published BEA summary A -├── compare_price_ratios.py # Pre-flight: industry-PI vs commodity-PI sanity -├── compare_method_stability.py # Step 7d: YoY N stability from compiled EF sheets -│ -├── dispatch_ef_time_series.py # Step 7a: trigger generate_diagnostics workflow runs -├── compile_ef_diagnostics.py # Step 7b: aggregate per-run Sheets → workbook + parquet -├── plot_ef_diagnostics.py # Step 7c: EF scatter + histogram from compiled parquet -├── plot_v0_3_n_pct_hist.py # Ad-hoc: single-sheet N/D histogram -│ -├── recover_ef_run_index.py # Utility: reconstruct ef_run_index.csv from Drive -├── view_vnorm_and_price_ratios.py # Utility: print V-norm + industry-vs-commodity PI summaries -│ -└── output/ - ├── results/ # parquets, CSVs, last_run_sheet_id.txt, ef_run_index.csv - └── plots/ # PNGs — published artifacts -``` - -The file order above mirrors the execution DAG (data → cross-approach views → EF orchestration → utilities). Within each block, scripts share a prefix (`derive_*` for data producers, `compare_*` for cross-approach views, `dispatch_*` / `compile_*` / `plot_*` for the async EF pipeline) so `ls | sort` clusters them by role. - -All driver scripts share the `constants.py` paths and color palette, and the `__init__.py` toggles two config flags that **every** script in this folder assumes: `update_inflation_factors=True` (BEA-derived industry PI path) and `apply_inflation_to_V=True` (V inflated to `model_base_year` for V-norm computations). Scripts that swap the global config mid-run (currently only `derive_A_time_series.py`) re-set those flags after each swap. - ---- - -## How to run things — execution DAG - -``` - (optional pre-flight) - compare_price_ratios.py - │ - ┌───────────────────────────┴───────────────────────────┐ - ▼ ▼ -derive_A_time_series.py derive_useeio_nowcast_A.py - │ │ - └──────────────────────────┬────────────────────────────┘ - ▼ - A_{approach}_{year}.parquet - │ - ┌────────────────┬──────────────────┬──────────────┬─────────────┐ - ▼ ▼ ▼ ▼ ▼ - derive_A_cells_ derive_A_cells_ compare_ compare_ compare_ - long.py stability.py approaches.py key_sectors.py summary_a_errors.py - │ - └──► A_cells_long.parquet (used by stability + several plots) - - │ (Step 6 / 7 — async, via GH Actions) - ▼ - dispatch_ef_time_series.py - │ - generate_diagnostics - │ - ▼ - compile_ef_diagnostics.py - │ - ▼ - ef_scatter_coords.parquet - │ - ┌────────────────┴────────────────┐ - ▼ ▼ - plot_ef_diagnostics.py compare_method_stability.py -``` - -### 1. (Optional) Pre-flight sanity - -```bash -python -m bedrock.analysis.a_matrix_time_series.compare_price_ratios -``` - -Confirms industry-PI and commodity-PI distributions differ enough that they're worth studying separately. Skip if you've run it before — outputs are committed. - -### 2. Cache A matrices for every (approach × year) - -```bash -# Internal approaches — loops 5 YAMLs × {2017…2024} and writes parquets. -python -m bedrock.analysis.a_matrix_time_series.derive_A_time_series - -# External reference — depends on the USEEIO nowcast extract module. -python -m bedrock.analysis.a_matrix_time_series.derive_useeio_nowcast_A -``` - -Outputs land in `output/results/A_{approach}_{year}.parquet`. The first script also creates a run-report Sheet in the [Drive folder](https://drive.google.com/drive/folders/1UcPmwLnL6MwTq9pMYJw5d43FJQOFQVO_) and writes `last_run_sheet_id.txt` so downstream scripts can append tabs to the same Sheet. - -`useeio_nowcast` covers 2017–2023 only — `APPROACH_YEAR_COVERAGE` in `constants.py` is the source of truth and every script that mixes it with other approaches filters via that table. - -### 3. Cell-level diagnostics - -```bash -python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_long -python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_stability -``` - -`derive_A_cells_long.py` produces the tall `A_cells_long.parquet` used by several downstream scripts plus baseline-reference and divergence-share plots. `derive_A_cells_stability.py` adds Jaccard + persistence diagnostics on top of that parquet. - -### 4. Cross-approach + key-sector + summary-A diagnostics - -```bash -python -m bedrock.analysis.a_matrix_time_series.compare_approaches -python -m bedrock.analysis.a_matrix_time_series.compare_key_sectors -python -m bedrock.analysis.a_matrix_time_series.compare_summary_a_errors -``` - -These three are independent (all read `output/results/*.parquet`); run in any order. - -### 5. EF diagnostics — Step 6 / 7 (async via GH Actions) - -This phase fans out to the `generate_diagnostics` GitHub Actions workflow; one Sheet per `(scenario, approach, year)` cell. The runs for the v0.3 evaluation have already been dispatched and live in Drive folder [`1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s`](https://drive.google.com/drive/folders/1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s). See [`useeio_nowcast_ef_runbook.md`](useeio_nowcast_ef_runbook.md) for the operator checklist. - -For a methodology flag outside this A-matrix scenario matrix, use -[`bedrock/utils/validation/evaluate_feature_impact.md`](../../utils/validation/evaluate_feature_impact.md) -and `python -m bedrock.utils.validation.dispatch_diagnostics` instead of the -scenario dispatcher below. Shared create-sheet / trigger helpers live in -`bedrock.utils.validation.dispatch_diagnostics`; this module owns the -time-series scenario matrix and `EF_TIME_SERIES_DRIVE_FOLDER_ID`. - -**Reviewer path — skip dispatch, use the existing runs:** - -```bash -# 5a-review. Pull the run index from Drive — required if you don't already -# have output/results/ef_run_index.csv locally. Needs Google -# application-default-credentials. -python -m bedrock.analysis.a_matrix_time_series.recover_ef_run_index \ - --folder-id 1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s - -# 5b-review. Compile — reads ef_run_index.csv, pulls each Sheet's EF diff -# tab, writes ef_scatter_coords.parquet. -python -m bedrock.analysis.a_matrix_time_series.compile_ef_diagnostics - -# 5c-review. Plot. -python -m bedrock.analysis.a_matrix_time_series.plot_ef_diagnostics -python -m bedrock.analysis.a_matrix_time_series.compare_method_stability -``` - -**Engineer path — triggering new runs** (requires GitHub Actions `workflow:write` and the `gh` CLI authenticated): - -```bash -# Dispatch is idempotent — skips cells already in ef_run_index.csv. -python -m bedrock.analysis.a_matrix_time_series.dispatch_ef_time_series \ - --git-ref main \ - --scenarios isolate_a_matrix,bundle_v0_3 \ - --years 2019,2020,2021,2022,2023 - -# Then wait for GH Actions to finish (each run takes ~30–60 min), then -# proceed to compile + plot as in the reviewer path above. -``` - -### Ad-hoc - -`plot_v0_3_n_pct_hist.py` renders a single-sheet N/D histogram for any diagnostics Sheet ID — useful when staring at one approach's results without running the full compile pipeline. - ---- - -## Conventions - -- **Approach keys are strings**, not enums. `APPROACH_ORDER` in `constants.py` is the canonical ordering for any plot grid or legend. -- **Two-baseline reporting is non-negotiable**: every comparison in this folder reports against both `useeio` (do-nothing invariant) and `ceda_default` (production status quo). See [`docs/analysis_plan.md` § Comparison baselines](docs/analysis_plan.md#comparison-baselines-apply-to-every-comparison-in-this-plan) for the rationale. -- **Year coverage gaps live in `APPROACH_YEAR_COVERAGE`** — scripts that mix `useeio_nowcast` with other approaches filter on it (`useeio_nowcast` has no 2024 data upstream). -- **`output/results/` holds data + bookkeeping**, `output/plots/` holds PNGs. The Sheet-tab side-channel is opt-in: scripts publish via `_run_report.py`, which silently no-ops when the auth/Sheet isn't configured. -- **All scripts use snake_case filenames** and one of the prefixes above. Utilities that don't fit a pipeline prefix (`recover_*`, `view_*`) are grouped at the bottom of the listing. - ---- - -## Related reading - -- [`../../utils/config/feature_flag.md`](../../utils/config/feature_flag.md) / - [`../../utils/validation/evaluate_feature_impact.md`](../../utils/validation/evaluate_feature_impact.md) - — `USAConfig` flag and diagnostics playbooks. -- [`useeio_nowcast_ef_runbook.md`](useeio_nowcast_ef_runbook.md) — **live** operator runbook for running the USEEIO-nowcast EF diagnostics via GH Actions. -- [`docs/analysis_plan.md`](docs/analysis_plan.md) — **historical** epic spec: 5-approach taxonomy, Definition of Done, Checkpoints A–D, six Key Questions. -- [`docs/implement_useeio_nowcast_plan.md`](docs/implement_useeio_nowcast_plan.md) — **historical** integration plan for the 6th (external) approach (Steps N1–N4). -- [`bedrock/transform/eeio/derived_cornerstone.py`](../../transform/eeio/derived_cornerstone.py) — `derive_cornerstone_Aq_scaled()` is the gated entry point that the five internal YAMLs select between. -- [`bedrock/analysis/time_series_B_matrix/`](../time_series_B_matrix) — sibling module the time-series caching pattern was modeled on. diff --git a/bedrock/analysis/a_matrix_time_series/__init__.py b/bedrock/analysis/a_matrix_time_series/__init__.py deleted file mode 100644 index a78fc640..00000000 --- a/bedrock/analysis/a_matrix_time_series/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Pin analysis-specific config flags for every script in this package. - -- `update_inflation_factors=True` — preserves pre-#369 BEA-derived industry - PI path, which this analysis universally assumes. Production defaults to - `False` to keep the legacy parquet flow. -- `apply_inflation_to_V=True` — inflates V to `cfg.model_base_year` when - computing the V-norm-derived commodity price ratio. Only consumed by - `commodity_price_index` approach but cheap to set globally. - -`get_cornerstone_industry_price_ratio` and -`get_vnorm_adjusted_commodity_price_ratio` are `@functools.cache`'d on -`(original_year, target_year)` only, so flags must be set before any -helper call or the default-branch result becomes the cached answer for -the rest of the process. Setting them here guarantees ordering. - -Scripts that swap the global config mid-run (e.g. `derive_A_time_series` -iterating over approach YAMLs) must re-set these flags inside their swap -helper — the toggle here only covers the initial config. -""" - -from bedrock.utils.config.usa_config import get_usa_config - -_cfg = get_usa_config() -_cfg.update_inflation_factors = True -_cfg.apply_inflation_to_V = True diff --git a/bedrock/analysis/a_matrix_time_series/_loaders.py b/bedrock/analysis/a_matrix_time_series/_loaders.py deleted file mode 100644 index 1a64adbe..00000000 --- a/bedrock/analysis/a_matrix_time_series/_loaders.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Shared parquet loaders for analysis scripts in this package.""" - -from __future__ import annotations - -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import RESULTS_DIR - - -def load_a_pair(approach: str, year: int) -> dict[str, pd.DataFrame]: - """Load (Adom, Aimp) from ``A_{approach}_{year}.parquet``. - - Returns ``{"dom": Adom, "imp": Aimp}`` so callers can iterate over the - two kinds with a single key. - """ - combined = pd.read_parquet(RESULTS_DIR / f"A_{approach}_{year}.parquet") - return { - "dom": pd.DataFrame(combined.loc["dom"]), - "imp": pd.DataFrame(combined.loc["imp"]), - } diff --git a/bedrock/analysis/a_matrix_time_series/_run_report.py b/bedrock/analysis/a_matrix_time_series/_run_report.py deleted file mode 100644 index 6e9658a8..00000000 --- a/bedrock/analysis/a_matrix_time_series/_run_report.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Shared helper for appending tabs to the analysis run-report Sheet. - -The run-report Sheet ID is written by ``derive_A_time_series`` (Step 1) -to ``LAST_RUN_SHEET_ID_PATH``. Every later step appends one or more -result tabs to that Sheet. Each step previously inlined the same -guard / read / try-update / log block; this module is the single -canonical version. -""" - -from __future__ import annotations - -import logging - -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import LAST_RUN_SHEET_ID_PATH -from bedrock.utils.io.gcp import update_sheet_tab - -logger = logging.getLogger(__name__) - - -def publish_tabs(tabs: dict[str, pd.DataFrame]) -> None: - """Append ``tabs`` to the run-report Sheet, no-op if it doesn't exist. - - Skipped with a warning if ``LAST_RUN_SHEET_ID_PATH`` is missing (Step 1 - hasn't run, or it ran without Drive auth). Network errors during the - update are caught and logged so the local CSV/PNG artifacts always - succeed even when Sheet publishing fails. - """ - if not LAST_RUN_SHEET_ID_PATH.exists(): - logger.warning( - "No %s found — skipping Sheet publish. Run derive_A_time_series " - "first (with valid Drive auth) to create the run report.", - LAST_RUN_SHEET_ID_PATH, - ) - return - sheet_id = LAST_RUN_SHEET_ID_PATH.read_text().strip() - try: - for tab_name, df in tabs.items(): - update_sheet_tab(sheet_id, tab_name, df) - except Exception as e: # noqa: BLE001 - logger.warning( - "Sheet publish skipped (%s: %s). Local artifacts still complete.", - type(e).__name__, - e, - ) - return - logger.info("Updated %d tab(s) on sheet %s", len(tabs), sheet_id) diff --git a/bedrock/analysis/a_matrix_time_series/compare_approaches.py b/bedrock/analysis/a_matrix_time_series/compare_approaches.py deleted file mode 100644 index ab07feab..00000000 --- a/bedrock/analysis/a_matrix_time_series/compare_approaches.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Step 3 of epic #337: cross-approach comparison at fixed target year (2024). - -Reads the parquet caches produced by ``derive_A_time_series.py`` (Step 1) and -produces: - -- ``pairwise_hexbins_{dom,imp}.png`` — log-log hexbin - density plots between the alternative approaches at 2024 - (``summary_tables`` vs ``commodity_price_index``). Hexbin density - uses ``bins='log'`` because A cells span ~6 orders of magnitude and a - linear color scale would be dominated by the near-zero peak. Step 2 - already covers alternative-vs-baseline scatters at 2024 (linear axes); - this figure adds the alternative-vs-alternative angle that Step 2 can't - show, on log-log axes that resolve the dense low-magnitude region. - -- ``column_cap_audit.csv`` — every (year, dom_or_imp, col_sector) - where the 0.98 column-cap inside ``scale_cornerstone_A`` fired on the - ``summary_tables`` approach. Cap-fired detection: column sum within - ``CAP_TOL`` of 0.98 after scaling. Columns just below 0.98 (sum > 0.97 - but cap not reached) are also written so reviewers can see how close - the cap was to firing. - -- Sheet tab ``column_cap_audit`` appended to the run-report Sheet - (sheet ID read from ``last_run_sheet_id.txt``). - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compare_approaches -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series._loaders import load_a_pair -from bedrock.analysis.a_matrix_time_series._run_report import publish_tabs -from bedrock.analysis.a_matrix_time_series.constants import ( - PLOTS_DIR, - RESULTS_DIR, -) - -logger = logging.getLogger(__name__) - -# 2023, not LATEST_TARGET_YEAR (=2024): useeio_nowcast has no 2024 upstream, -# and we want all 6 approaches on a single shared year for apples-to-apples. -HEXBIN_TARGET_YEAR = 2023 - -BASELINE_APPROACHES: tuple[str, ...] = ("useeio", "ceda_default") - -# v0.3 focus pairs at HEXBIN_TARGET_YEAR: -# - The two top internal candidates vs each other. -# - Each candidate vs the external reference (useeio_nowcast). -# - The external reference vs each baseline. -# industry_price_index is dropped (superseded by commodity_price_index). -HEXBIN_PAIRS: tuple[tuple[str, str], ...] = ( - ("summary_tables", "commodity_price_index"), - ("summary_tables", "useeio_nowcast"), - ("commodity_price_index", "useeio_nowcast"), - ("ceda_default", "useeio_nowcast"), - ("useeio", "useeio_nowcast"), -) -KINDS: tuple[str, ...] = ("dom", "imp") - -# Column cap inside scale_cornerstone_A — re-stated here as a constant so the -# audit can test for it without importing the production helper. -COLUMN_CAP = 0.98 -CAP_TOL = 1e-9 -NEAR_CAP_THRESHOLD = 0.97 # report borderline columns too - - -def _load_approaches_at_year( - approaches: tuple[str, ...], year: int -) -> dict[str, dict[str, pd.DataFrame]]: - """Returns ``{kind: {approach: A_matrix}}`` for the given approaches at ``year``.""" - out: dict[str, dict[str, pd.DataFrame]] = {kind: {} for kind in KINDS} - for approach in approaches: - pair = load_a_pair(approach, year) - for kind in KINDS: - out[kind][approach] = pair[kind] - return out - - -def _aligned_arrays(a: pd.DataFrame, b: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]: - """Reindex ``b`` to ``a``'s row/column order and return raveled values. - - Consumed by: ``plot_pairwise_hexbins``. - """ - b_aligned = b.reindex(index=a.index, columns=a.columns) - return a.to_numpy().ravel(), b_aligned.to_numpy().ravel() - - -# --------------------------------------------------------------------------- -# Pairwise hexbins -# --------------------------------------------------------------------------- - - -def plot_pairwise_hexbins( - matrices: dict[str, dict[str, pd.DataFrame]], - kind: str, - path: Path, - *, - pairs: tuple[tuple[str, str], ...] = HEXBIN_PAIRS, - year: int = HEXBIN_TARGET_YEAR, -) -> None: - """1×N log-log hexbin density grid: pairs of A-matrix approaches. - - Cells where both values are zero (joint sparsity in A) are dropped — they - inflate the density at the origin and obscure the disagreement story. - Each panel reports n_cells, R², and the share of cells off the y=x - diagonal by more than 1× (i.e. ratio outside [0.5, 2]). - """ - fig, axes = plt.subplots(1, len(pairs), figsize=(5 * len(pairs), 5), squeeze=False) - fig.suptitle(f"Pairwise A-matrix comparison — {kind} — {year}", fontsize=12) - eps = 1e-12 - - for ax_idx, (a_name, b_name) in enumerate(pairs): - ax = axes[0][ax_idx] - x, y = _aligned_arrays(matrices[kind][a_name], matrices[kind][b_name]) - mask = ~(np.isnan(x) | np.isnan(y)) & ~((x == 0) & (y == 0)) - x_pos = x[mask] - y_pos = y[mask] - if x_pos.size == 0: - ax.text(0.5, 0.5, "no data", transform=ax.transAxes, ha="center") - continue - - # Log-log because A spans many orders of magnitude. eps shifts joint - # zero-on-one-side cells onto the plot rather than dropping them. - log_x = np.log10(x_pos + eps) - log_y = np.log10(y_pos + eps) - hb = ax.hexbin(log_x, log_y, gridsize=60, bins="log", cmap="viridis") - cbar = fig.colorbar(hb, ax=ax) - cbar.set_label("log10(count)", fontsize=9) - - lo = float(min(log_x.min(), log_y.min())) - hi = float(max(log_x.max(), log_y.max())) - ax.plot([lo, hi], [lo, hi], "r--", lw=0.8, alpha=0.8, label="y=x") - ax.set_xlim(lo, hi) - ax.set_ylim(lo, hi) - ax.set_aspect("equal", adjustable="box") - ax.set_xlabel(f"log10({a_name})") - ax.set_ylabel(f"log10({b_name})") - ax.set_title(f"{a_name} vs {b_name}", fontsize=10) - ax.grid(True, alpha=0.3) - - # Stats: R² in original (non-log) space; off-diagonal share = cells - # where ratio is outside [0.5, 2] (i.e. one side ≥2× the other). - x_f = np.asarray(x_pos, dtype=float) - y_f = np.asarray(y_pos, dtype=float) - r2 = ( - float(np.corrcoef(x_f, y_f)[0, 1] ** 2) - if x_f.std() > 0 and y_f.std() > 0 - else float("nan") - ) - denom = np.where(np.abs(x_f) > 0, np.abs(x_f), np.nan) - ratio = np.abs(y_f / denom) - valid = ~np.isnan(ratio) - off_diag_share = ( - float(((ratio[valid] < 0.5) | (ratio[valid] > 2.0)).mean()) - if valid.any() - else float("nan") - ) - stats_text = ( - f"n = {x_pos.size:,}\n" - f"R² = {r2:.4f}\n" - f"|y/x| outside [0.5,2]: {off_diag_share:.1%}" - ) - ax.text( - 0.02, - 0.98, - stats_text, - transform=ax.transAxes, - va="top", - ha="left", - fontsize=9, - family="monospace", - bbox={ - "boxstyle": "round,pad=0.3", - "facecolor": "white", - "alpha": 0.85, - "edgecolor": "gray", - }, - ) - - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -# --------------------------------------------------------------------------- -# Column-cap audit (summary_tables only, all years) -# --------------------------------------------------------------------------- - - -def column_cap_audit(years: list[int]) -> pd.DataFrame: - """For each (year, kind) of ``summary_tables``, list every column whose - sum is at the 0.98 cap or close to it. - - The cap fires inside ``scale_cornerstone_A`` whenever the post-scaling - column sum exceeds 1; the column is then rescaled to exactly 0.98. So - "cap fired" ⇔ column sum within ``CAP_TOL`` of 0.98 after scaling. - Columns just below the cap (sum > 0.97) are also reported so reviewers - can see how close the cap was to engaging. - """ - rows: list[dict[str, object]] = [] - for year in years: - pair = load_a_pair("summary_tables", year) - for kind in KINDS: - col_sum = pair[kind].sum(axis=0) - for col, val in col_sum.items(): - cap_fired = abs(val - COLUMN_CAP) <= CAP_TOL - if cap_fired or float(val) > NEAR_CAP_THRESHOLD: - rows.append( - { - "year": year, - "dom_or_imp": kind, - "col_sector": col, - "col_sum": float(val), - "cap_fired": bool(cap_fired), - } - ) - df = pd.DataFrame(rows) - if not df.empty: - df = df.sort_values( - ["year", "dom_or_imp", "cap_fired", "col_sum"], - ascending=[True, True, False, False], - ).reset_index(drop=True) - return df - - -# --------------------------------------------------------------------------- -# Sheet publish -# --------------------------------------------------------------------------- - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - # Load every approach mentioned in HEXBIN_PAIRS at HEXBIN_TARGET_YEAR (2023). One plot - # per kind covers all 5 pairs — internal-vs-internal, internal-vs-nowcast, - # nowcast-vs-baselines — on a single shared year. - pair_approaches = tuple(dict.fromkeys(a for pair in HEXBIN_PAIRS for a in pair)) - logger.info( - "Loading %d approaches at %d for pairwise hexbins", - len(pair_approaches), - HEXBIN_TARGET_YEAR, - ) - matrices = _load_approaches_at_year(pair_approaches, HEXBIN_TARGET_YEAR) - - for kind in KINDS: - plot_pairwise_hexbins( - matrices, kind, PLOTS_DIR / f"pairwise_hexbins_{kind}.png" - ) - - # Cap audit spans all available summary_tables years to give a full picture - # of how often the 0.98 cap engages, not just at the focus year. - audit_years = sorted( - int(p.stem.rsplit("_", 1)[-1]) - for p in RESULTS_DIR.glob("A_summary_tables_*.parquet") - ) - cap_audit_df = column_cap_audit(audit_years) - cap_audit_df.to_csv(RESULTS_DIR / "column_cap_audit.csv", index=False) - - publish_tabs({"column_cap_audit": cap_audit_df}) - logger.info("Step 3 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/compare_key_sectors.py b/bedrock/analysis/a_matrix_time_series/compare_key_sectors.py deleted file mode 100644 index 0efe3aee..00000000 --- a/bedrock/analysis/a_matrix_time_series/compare_key_sectors.py +++ /dev/null @@ -1,522 +0,0 @@ -"""Step 4 of epic #337: data-driven sector deep dive, impact-weighted, two-baseline. - -Reframed from a curated-shortlist deep dive into a data-driven discovery: rank -all A-matrix cells by `max_alt |A_alt − A_baseline| × |A_baseline|` for each -of the two baselines (USEEIO, CEDA-US), then surface (a) what BEA summary -parent groups dominate the ranking and (b) which specific cells matter most. - -The two-baseline rule from the analysis plan applies: every divergence -quantity is reported against both USEEIO and CEDA-US. - -Reads ``A_cells_long.parquet`` (Step 2 output). Produces: - -- ``keysector_impact_heatmap.png`` — single figure, two side-by-side panels - (vs USEEIO | vs CEDA-US). Rows = BEA summary parent groups of - ``col_sector`` (the consuming industry), filtered to top - ``HEATMAP_TOP_GROUPS`` by total impact across both baselines. Cols = the - three alternative approaches (USEEIO and CEDA-US are baselines, not - candidates). Cell value = sum of ``impact_vs_baseline`` across every - ``dom`` A cell whose ``col_sector`` rolls up to that summary group. - Color: viridis, shared scale across panels for direct cross-baseline - reading. Annotated with the numeric value. - -- ``keysector_top_cells_grid.png`` — drill-in grid. Top - ``DRILL_IN_TOP_N`` cells by ``max(impact_vs_useeio, impact_vs_ceda)``, - laid out 3 columns × N/3 rows. Each panel = one cell, all five - approaches overlaid. Title shows ``{row_sector} → {col_sector}`` plus - the ``col_sector``'s BEA summary parent. Per-panel annotation reports - both impact scores so the reader sees which baseline drove the cell - into the top-N. - -- ``keysector_top_cells_ranked.csv`` — full top-``CSV_TOP_N`` ranked cells - with both impact scores, the BEA summary group of ``col_sector`` and - ``row_sector``, and the per-approach values at ``RANK_TARGET_YEAR``. - -- ``keysector_curated_shortlist.csv`` — appendix retained from the - curated theory-driven shortlist. The headline figures are now - data-driven; the curated list is preserved here for the "did the priors - hold?" methodology check in the eventual write-up. - -- Sheet tabs ``keysector_top_cells_ranked``, ``keysector_curated_shortlist`` - appended to the run-report Sheet. - -Tunable constants: ``RANK_TARGET_YEAR``, ``FOCUS_APPROACHES``, -``BASELINES``, ``HEATMAP_TOP_GROUPS``, ``DRILL_IN_TOP_N``, ``CSV_TOP_N``. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compare_key_sectors -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series._run_report import publish_tabs -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_COLORS, - BASELINES, - FOCUS_APPROACHES, - PLOTS_DIR, - RESULTS_DIR, -) -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_ORDER as ALL_APPROACHES_PLOT_ORDER, -) -from bedrock.utils.taxonomy.bea.v2017_commodity_summary import ( - USA_2017_SUMMARY_COMMODITY_DESC, -) -from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( - load_bea_v2017_summary_to_cornerstone, -) -from bedrock.utils.taxonomy.cornerstone.commodities import COMMODITY_DESC - -logger = logging.getLogger(__name__) - -A_CELLS_LONG_PATH = RESULTS_DIR / "A_cells_long.parquet" - -# 2023, not 2024 — useeio_nowcast has no 2024 upstream data. -RANK_TARGET_YEAR = 2023 - -HEATMAP_TOP_GROUPS = 15 -DRILL_IN_TOP_N = 12 -CSV_TOP_N = 30 -DRILL_IN_NCOLS = 3 - -# Curated theory-driven shortlist — kept as appendix CSV. The headline -# figures are now data-driven. Categories from the original Step 4 plan. -CURATED_SHORTLIST: dict[str, list[tuple[str, str]]] = { - "energy": [ - ("221100", "Electric power generation/transmission/distribution"), - ("221200", "Natural gas distribution"), - ("211000", "Oil and gas extraction"), - ("324110", "Petroleum refineries"), - ], - "travel": [ - ("481000", "Air transportation"), - ("721000", "Accommodation"), - ("722110", "Full-service restaurants"), - ("722211", "Limited-service restaurants"), - ("711100", "Performing arts companies"), - ], - "waste": [ - ("562111", "Solid waste collection"), - ("562212", "Solid waste landfills"), - ("562213", "Solid waste combustors"), - ("562910", "Remediation services"), - ("562HAZ", "Hazardous waste (Cornerstone disagg)"), - ("562OTH", "Other waste (Cornerstone disagg)"), - ], - "volatility": [ - ("331110", "Iron and steel mills"), - ("331313", "Alumina refining"), - ("331200", "Steel product manufacturing"), - ("311224", "Oilseed processing"), - ("336111", "Automobile manufacturing"), - ], - "disagg": [ - ("33131B", "Other nonferrous (BEA aggregate)"), - ("31151A", "Dairy products (BEA aggregate)"), - ("31161A", "Animal slaughtering (BEA aggregate)"), - ("3118A0", "Other food (BEA aggregate)"), - ("722A00", "All other food services (BEA aggregate)"), - ("711A00", "Independent artists / spectator sports (BEA aggregate)"), - ], -} - - -def _cornerstone_to_summary() -> dict[str, str]: - """Invert ``load_bea_v2017_summary_to_cornerstone`` into a - cornerstone_code → bea_summary_code lookup.""" - summary_to_cornerstone = load_bea_v2017_summary_to_cornerstone() - return { - code: str(summary) - for summary, codes in summary_to_cornerstone.items() - for code in codes - } - - -# Widen the upstream Literal-keyed dicts to plain str → str so descriptive -# lookups work on arbitrary cell codes coming out of the parquet without -# tripping mypy's call-overload check on the Literal `.get()` signature. -_SUMMARY_DESC: dict[str, str] = { - str(k): str(v) for k, v in USA_2017_SUMMARY_COMMODITY_DESC.items() -} -_COMMODITY_DESC: dict[str, str] = {str(k): str(v) for k, v in COMMODITY_DESC.items()} - - -def _summary_desc(code: str, max_len: int = 32) -> str: - """Short BEA summary code description, truncated to ``max_len``.""" - desc = _SUMMARY_DESC.get(code, "") - if not desc: - return "" - return desc if len(desc) <= max_len else desc[: max_len - 1] + "…" - - -def _commodity_desc(code: str, max_len: int = 28) -> str: - """Short cornerstone commodity description, truncated to ``max_len``.""" - desc = _COMMODITY_DESC.get(code, "") - if not desc: - return "" - return desc if len(desc) <= max_len else desc[: max_len - 1] + "…" - - -def _heatmap_row_label(code: str) -> str: - """``CODE — short description`` for heatmap y-axis tick labels.""" - desc = _summary_desc(code, max_len=36) - return f"{code} — {desc}" if desc else code - - -def compute_impact_table(long: pd.DataFrame) -> pd.DataFrame: - """Cell-level impact metrics for both baselines at ``RANK_TARGET_YEAR``. - - For each (row_sector, col_sector) cell on the ``dom`` matrix, computes: - - ``impact_vs_useeio = max_alt |A_alt − A_useeio| × |A_useeio|`` - - ``impact_vs_ceda = max_alt |A_alt − A_ceda| × |A_ceda|`` - - per-approach values at ``RANK_TARGET_YEAR`` - - Where ``alt`` ranges over ``FOCUS_APPROACHES`` only — the two - baselines are anchors, not candidates being evaluated. - """ - snap = long.loc[(long["dom_or_imp"] == "dom") & (long["year"] == RANK_TARGET_YEAR)] - pivot = snap.pivot_table( - index=["row_sector", "col_sector"], - columns="approach", - values="A_value", - ).fillna(0.0) - - # Compute max-over-alternatives |A_alt − A_baseline| per cell. - alt_cols = [c for c in FOCUS_APPROACHES if c in pivot.columns] - out = pivot.reset_index() - - for baseline_col, _ in BASELINES: - if baseline_col not in pivot.columns: - out[f"impact_vs_{baseline_col}"] = 0.0 - continue - baseline_vals = pivot[baseline_col].to_numpy() - max_abs_diff = np.zeros_like(baseline_vals) - for alt in alt_cols: - diff = np.abs(pivot[alt].to_numpy() - baseline_vals) - max_abs_diff = np.maximum(max_abs_diff, diff) - impact = max_abs_diff * np.abs(baseline_vals) - out[f"impact_vs_{baseline_col}"] = impact - - out["impact_max_either"] = np.maximum( - out["impact_vs_useeio"].to_numpy(), - out["impact_vs_ceda_default"].to_numpy(), - ) - - cs2sum = _cornerstone_to_summary() - out["col_summary_group"] = out["col_sector"].map(cs2sum).fillna("UNMAPPED") - out["row_summary_group"] = out["row_sector"].map(cs2sum).fillna("UNMAPPED") - - return out - - -def aggregate_impact_by_group( - long: pd.DataFrame, top_n_groups: int -) -> tuple[pd.DataFrame, pd.DataFrame, list[str]]: - """Sum impact per (col_summary_group, alternative) for each baseline. - - Returns two DataFrames (vs_useeio, vs_ceda) of shape - ``(n_groups × n_alternatives)`` and the list of group codes (rows) - selected as the top-N by total impact across both baselines. Computed - fresh from the snap pivot — needs the per-alternative breakdown rather - than the per-cell ``max-over-alternatives`` from ``compute_impact_table``. - """ - cs2sum = _cornerstone_to_summary() - rows_useeio: list[dict[str, object]] = [] - rows_ceda: list[dict[str, object]] = [] - - snap = long.loc[(long["dom_or_imp"] == "dom") & (long["year"] == RANK_TARGET_YEAR)] - pivot = snap.pivot_table( - index=["row_sector", "col_sector"], - columns="approach", - values="A_value", - ).fillna(0.0) - pivot = pivot.reset_index() - pivot["col_summary_group"] = pivot["col_sector"].map(cs2sum).fillna("UNMAPPED") - - for baseline_col, _ in BASELINES: - if baseline_col not in pivot.columns: - continue - for alt in FOCUS_APPROACHES: - if alt not in pivot.columns: - continue - cell_impact = np.abs( - pivot[alt].to_numpy() - pivot[baseline_col].to_numpy() - ) * np.abs(pivot[baseline_col].to_numpy()) - tmp = pd.DataFrame( - { - "col_summary_group": pivot["col_summary_group"], - "impact": cell_impact, - } - ) - for grp, grp_sum in ( - tmp.groupby("col_summary_group")["impact"].sum().items() - ): - row = { - "col_summary_group": str(grp), - "alternative": alt, - "impact": float(grp_sum), - } - if baseline_col == "useeio": - rows_useeio.append(row) - else: - rows_ceda.append(row) - - df_u = pd.DataFrame(rows_useeio).pivot_table( - index="col_summary_group", - columns="alternative", - values="impact", - fill_value=0.0, - ) - df_c = pd.DataFrame(rows_ceda).pivot_table( - index="col_summary_group", - columns="alternative", - values="impact", - fill_value=0.0, - ) - - # Make column order stable. - cols_order = [c for c in FOCUS_APPROACHES if c in df_u.columns] - df_u = df_u.reindex(columns=cols_order, fill_value=0.0) - df_c = df_c.reindex(columns=cols_order, fill_value=0.0) - - # Align row index across the two and pick top-N by combined impact. - combined = df_u.sum(axis=1).add(df_c.sum(axis=1), fill_value=0.0) - top_groups = combined.nlargest(top_n_groups).index.tolist() - df_u = df_u.reindex(index=top_groups, fill_value=0.0) - df_c = df_c.reindex(index=top_groups, fill_value=0.0) - - return df_u, df_c, top_groups - - -def plot_impact_heatmap(df_u: pd.DataFrame, df_c: pd.DataFrame, path: Path) -> None: - """Two-panel heatmap, vs USEEIO | vs CEDA-US, shared color scale. - - Reading: each cell is the total ``|A_alt − A_baseline| × |A_baseline|`` - summed over every dom A cell whose ``col_sector`` rolls up to the row's - BEA summary parent. Dark = method choice changes a lot of A·A_baseline - in that group. Shared color scale across panels means one row darker - on the left than on the right ⇒ that group of cells diverges from - USEEIO more than from CEDA-US. - """ - vmax = float(max(df_u.to_numpy().max(), df_c.to_numpy().max()) or 1.0) - fig, axes = plt.subplots( - 1, - 2, - figsize=(6.5 * 2, 0.45 * len(df_u) + 2.0), - squeeze=False, - constrained_layout=True, - ) - fig.suptitle( - f"Impact-weighted divergence by BEA summary group — dom — {RANK_TARGET_YEAR}", - fontsize=12, - ) - - im = None - for ax, (df, baseline_label) in zip( - axes[0], [(df_u, "vs USEEIO"), (df_c, "vs CEDA-US")], strict=True - ): - im = ax.imshow(df.to_numpy(), aspect="auto", cmap="viridis", vmin=0, vmax=vmax) - ax.set_xticks(range(len(df.columns))) - ax.set_xticklabels(list(df.columns), rotation=30, ha="right", fontsize=9) - ax.set_yticks(range(len(df.index))) - ax.set_yticklabels( - [_heatmap_row_label(str(code)) for code in df.index], fontsize=9 - ) - ax.set_title(baseline_label, fontsize=10) - - arr = df.to_numpy() - for i in range(arr.shape[0]): - for j in range(arr.shape[1]): - val = arr[i, j] - if val == 0: - txt = "0" - elif val < 0.001: - txt = f"{val:.1e}" - else: - txt = f"{val:.3g}" - color = "white" if val < vmax * 0.55 else "black" - ax.text(j, i, txt, ha="center", va="center", fontsize=7, color=color) - - if im is not None: - cbar = fig.colorbar(im, ax=axes[0], shrink=0.7, pad=0.02) - cbar.set_label("Σ |A_alt − A_baseline| · |A_baseline|", fontsize=9) - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def plot_top_cells_grid( - long: pd.DataFrame, top_cells: pd.DataFrame, path: Path -) -> None: - """Drill-in grid: top-N cells by ``max(impact_vs_useeio, impact_vs_ceda)``. - - One panel per cell. All five approaches overlaid. Title shows - ``{row_sector} → {col_sector}`` and the BEA summary parent of the - consuming industry. Per-panel annotation lists both impact scores. - """ - n = len(top_cells) - n_cols = DRILL_IN_NCOLS - n_rows = (n + n_cols - 1) // n_cols - fig, axes = plt.subplots( - n_rows, - n_cols, - figsize=(4.0 * n_cols, 3.0 * n_rows), - squeeze=False, - sharex=True, - ) - fig.suptitle( - f"Top-{n} cells by max(impact_vs_useeio, impact_vs_ceda) — dom — " - f"ranked at {RANK_TARGET_YEAR}", - fontsize=12, - ) - - legend_handles: list[Any] = [] - legend_labels: list[str] = [] - - for k, (_, row) in enumerate(top_cells.iterrows()): - ax = axes[k // n_cols][k % n_cols] - row_sector = str(row["row_sector"]) - col_sector = str(row["col_sector"]) - col_summary = str(row.get("col_summary_group", "")) - imp_u = float(row["impact_vs_useeio"]) - imp_c = float(row["impact_vs_ceda_default"]) - - cell_data = long.loc[ - (long["dom_or_imp"] == "dom") - & (long["row_sector"] == row_sector) - & (long["col_sector"] == col_sector) - ] - for approach in ALL_APPROACHES_PLOT_ORDER: - ad = cell_data.loc[cell_data["approach"] == approach].sort_values("year") - if ad.empty: - continue - (line,) = ax.plot( - ad["year"].to_numpy(), - ad["A_value"].to_numpy(), - color=APPROACH_COLORS[approach], - lw=1.5, - marker="o", - markersize=3, - label=approach, - ) - if approach not in legend_labels: - legend_handles.append(line) - legend_labels.append(approach) - - col_summary_short = _summary_desc(col_summary, max_len=22) - row_short = _commodity_desc(row_sector) - col_short = _commodity_desc(col_sector) - subtitle = ( - f"{row_short} → {col_short}" - if (row_short and col_short) - else f"in {col_summary_short}" - ) - # Multi-line title: code line + plain-text descriptive line. Using - # set_title rather than separate ax.text avoids the overlap that arises - # when both claim y=1.0 in axes coordinates. - ax.set_title( - f"{row_sector} → {col_sector} [{col_summary}]\n{subtitle}", - fontsize=8, - linespacing=1.3, - ) - ax.grid(True, alpha=0.3) - ax.text( - 0.02, - 0.98, - f"vs USEEIO: {imp_u:.2e}\nvs CEDA : {imp_c:.2e}", - transform=ax.transAxes, - va="top", - ha="left", - fontsize=7, - family="monospace", - bbox={ - "boxstyle": "round,pad=0.2", - "facecolor": "white", - "alpha": 0.85, - "edgecolor": "lightgray", - }, - ) - if k % n_cols == 0: - ax.set_ylabel("A_value", fontsize=8) - if k // n_cols == n_rows - 1: - ax.set_xlabel("year", fontsize=8) - - for k in range(n, n_rows * n_cols): - axes[k // n_cols][k % n_cols].axis("off") - - if legend_handles: - fig.legend( - legend_handles, - legend_labels, - loc="lower center", - ncol=len(legend_labels), - bbox_to_anchor=(0.5, -0.01), - fontsize=9, - framealpha=0.4, - ) - fig.tight_layout(rect=(0, 0.02, 1, 1)) - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def build_curated_shortlist_df() -> pd.DataFrame: - rows: list[dict[str, object]] = [] - for category, entries in CURATED_SHORTLIST.items(): - for code, justification in entries: - rows.append( - { - "category": category, - "bea_code": code, - "justification": justification, - } - ) - return pd.DataFrame(rows) - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - logger.info("Loading %s", A_CELLS_LONG_PATH) - long = pd.read_parquet(A_CELLS_LONG_PATH) - - impact_df = compute_impact_table(long) - impact_df_sorted = impact_df.sort_values( - "impact_max_either", ascending=False - ).reset_index(drop=True) - - top_cells_df = impact_df_sorted.head(CSV_TOP_N).copy() - top_cells_df.to_csv(RESULTS_DIR / "keysector_top_cells_ranked.csv", index=False) - - curated_df = build_curated_shortlist_df() - curated_df.to_csv(RESULTS_DIR / "keysector_curated_shortlist.csv", index=False) - - df_u, df_c, _ = aggregate_impact_by_group(long, HEATMAP_TOP_GROUPS) - plot_impact_heatmap(df_u, df_c, PLOTS_DIR / "keysector_impact_heatmap.png") - - plot_top_cells_grid( - long, - impact_df_sorted.head(DRILL_IN_TOP_N), - PLOTS_DIR / "keysector_top_cells_grid.png", - ) - - publish_tabs( - { - "keysector_top_cells_ranked": top_cells_df, - "keysector_curated_shortlist": curated_df, - } - ) - logger.info("Step 4 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/compare_method_stability.py b/bedrock/analysis/a_matrix_time_series/compare_method_stability.py deleted file mode 100644 index 42b3a799..00000000 --- a/bedrock/analysis/a_matrix_time_series/compare_method_stability.py +++ /dev/null @@ -1,543 +0,0 @@ -"""Compare year-over-year stability of N across 3 A-matrix methods. - -Reads the per-pair tabs in ``ef_comparison.xlsx`` (produced by -``compile_ef_diagnostics.py`` after dispatching the time-series cells), -computes per-sector year-over-year (YoY) changes in ``N_new_ref`` (deflated -to 2023$), and rolls up to per-approach metrics. - -The ``useeio`` approach is excluded — it pins A to the 2017 detail benchmark -with no temporal scaling, so its N time series reflects only B/x drift and -does not represent a comparable method here. - -Per-approach metrics: -- ``mean_abs_yoy_pct`` — average |YoY %| across the 4 transitions - (2019→20, 20→21, 21→22, 22→23). The headline stability metric. -- ``max_abs_yoy_pct`` — biggest single-year swing (tail). -- ``total_drift_pct`` — ``N_2023 / N_2019 - 1``. End-to-end shift. - -Each metric is rolled up three ways: median (typical sector), p95 (tail), -and emissions-weighted (by ``mean_N``, the metric to optimize). - -Outputs: -- ``output/results/n_yoy_ranking.csv`` -- ``output/results/n_yoy_per_sector.csv`` -- ``output/plots/n_indexed_lines.png`` — top-K sectors, N - indexed to year-2019 = 100, - faceted by method -- ``output/plots/n_yoy_signed_violin_no_industry_pi.png`` — violin plot of signed - YoY % per method (drops - industry_price_index for - readability; useeio is - excluded as benchmark) - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compare_method_stability -""" - -from __future__ import annotations - -import logging -import re -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from matplotlib.lines import Line2D -from matplotlib.patches import Patch - -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_COLORS, - PLOTS_DIR, - RESULTS_DIR, -) - -logger = logging.getLogger(__name__) - -EF_COMPARISON_XLSX_PATH = RESULTS_DIR / "ef_comparison.xlsx" -YOY_RANKING_PATH = RESULTS_DIR / "n_yoy_ranking.csv" -YOY_PER_SECTOR_PATH = RESULTS_DIR / "n_yoy_per_sector.csv" -INDEXED_LINES_PLOT_PATH = PLOTS_DIR / "n_indexed_lines.png" -YOY_SIGNED_VIOLIN_NO_IPI_PLOT_PATH = ( - PLOTS_DIR / "n_yoy_signed_violin_no_industry_pi.png" -) - -# `compile_ef_diagnostics.py` keys per-pair tabs as -# `{scenario}_{year}.0_{approach}__vs_{baseline}` truncated to 31 chars. -TAB_RE = re.compile(r"^(?P[a-z0-9_]+?)_(?P\d{4})(?:\.0)?_(?P.+)$") -APPROACH_PREFIXES: tuple[tuple[str, str], ...] = ( - ("commodity_pr", "commodity_price_index"), - ("summary_tabl", "summary_tables"), - # useeio_nowcast must come BEFORE useeio — "useeio" is a prefix of - # "useeio_nowca" (the 31-char-truncated tab form), so the longer match - # has to be tested first. - ("useeio_nowca", "useeio_nowcast"), - ("useeio", "useeio"), -) - -EXCLUDED_APPROACHES: frozenset[str] = frozenset({"useeio"}) - -YOY_YEARS: tuple[int, ...] = (2019, 2020, 2021, 2022, 2023) -YOY_TRANSITIONS: tuple[tuple[int, int], ...] = tuple( - (YOY_YEARS[i], YOY_YEARS[i + 1]) for i in range(len(YOY_YEARS) - 1) -) - -# Drop sectors with mean N below this percentile from non-weighted metrics -# to avoid YoY % blow-up from numerical noise on near-zero means. -MIN_MEAN_PERCENTILE = 5 - -# Cumulative-share threshold for the indexed line plot: include the smallest -# set of sectors whose cumulative |mean_N| reaches this fraction of the total. -# These are the lines that actually move corporate-footprint outcomes. -LINE_PLOT_CUMULATIVE_SHARE = 0.30 -# Hard ceiling so the plot stays readable even if the head is very long-tailed. -LINE_PLOT_MAX_SECTORS = 8 - - -def _parse_tab(tab: str) -> tuple[str, int, str] | None: - """Return ``(scenario, year, approach)`` for a time-series tab, else None.""" - m = TAB_RE.match(tab) - if m is None: - return None - rest = m.group("rest") - for prefix, approach in APPROACH_PREFIXES: - if rest.startswith(prefix): - return m.group("scenario"), int(m.group("year")), approach - return None - - -def _read_panel(xlsx_path: Path) -> pd.DataFrame: - """Long-format panel: ``(scenario, approach, year, sector, N_new_ref)``. - - Excludes the approaches in ``EXCLUDED_APPROACHES``. - """ - if not xlsx_path.exists(): - raise FileNotFoundError( - f"{xlsx_path} not found — run compile_ef_diagnostics.py first." - ) - xls = pd.ExcelFile(xlsx_path) - rows: list[pd.DataFrame] = [] - for tab in xls.sheet_names: - tab_str = str(tab) - parsed = _parse_tab(tab_str) - if parsed is None: - continue - scenario, year, approach = parsed - if approach in EXCLUDED_APPROACHES: - continue - df = pd.read_excel(xls, sheet_name=tab_str) - if "N_new_ref" not in df.columns: - logger.warning( - "Tab %r missing N_new_ref — re-run compile after the deflation " - "step was added.", - tab_str, - ) - continue - sector_col = df.columns[0] - chunk = pd.DataFrame( - { - "scenario": scenario, - "approach": approach, - "year": year, - "sector": df[sector_col].astype(str), - "N_new_ref": pd.to_numeric(df["N_new_ref"], errors="coerce"), - } - ) - rows.append(chunk) - if not rows: - raise RuntimeError( - f"No time-series tabs found in {xlsx_path}; verify compile output." - ) - return pd.concat(rows, ignore_index=True) - - -def _yoy_per_sector(panel: pd.DataFrame) -> pd.DataFrame: - """Per ``(approach, sector)``: per-year N + YoY %s + aggregates.""" - wide = panel.pivot_table( - index=["approach", "sector"], - columns="year", - values="N_new_ref", - aggfunc="first", - ) - # Make sure every year-column we expect exists (NaN if a cell missing). - for y in YOY_YEARS: - if y not in wide.columns: - wide[y] = np.nan - wide = wide[list(YOY_YEARS)] - wide.columns = pd.Index([f"N_{y}" for y in YOY_YEARS]) - wide = wide.reset_index() - - # YoY % change for each transition. - yoy_cols: list[str] = [] - for y0, y1 in YOY_TRANSITIONS: - col = f"yoy_{y0}_{y1}" - prev = wide[f"N_{y0}"] - curr = wide[f"N_{y1}"] - wide[col] = (curr - prev) / prev.abs() - yoy_cols.append(col) - - abs_yoy = wide[yoy_cols].abs() - wide["mean_abs_yoy_pct"] = abs_yoy.mean(axis=1) - wide["max_abs_yoy_pct"] = abs_yoy.max(axis=1) - wide["total_drift_pct"] = ( - wide[f"N_{YOY_YEARS[-1]}"] - wide[f"N_{YOY_YEARS[0]}"] - ) / wide[f"N_{YOY_YEARS[0]}"].abs() - wide["abs_total_drift_pct"] = wide["total_drift_pct"].abs() - wide["mean_N"] = wide[[f"N_{y}" for y in YOY_YEARS]].mean(axis=1) - return wide - - -def _aggregate_yoy_per_method(per_sector: pd.DataFrame) -> pd.DataFrame: - """Per approach: median / p95 / emissions-weighted of each YoY metric.""" - metrics = ("mean_abs_yoy_pct", "max_abs_yoy_pct", "abs_total_drift_pct") - rows: list[dict[str, object]] = [] - for approach in sorted(per_sector["approach"].unique()): - grp = per_sector[per_sector["approach"] == approach] - cutoff = grp["mean_N"].abs().quantile(MIN_MEAN_PERCENTILE / 100) - big = grp[grp["mean_N"].abs() >= cutoff] - weights = big["mean_N"].abs() - row: dict[str, object] = { - "approach": approach, - "n_sectors": int(len(grp)), - } - for metric in metrics: - values = big[metric] - row[f"{metric}__median"] = float(values.median()) - row[f"{metric}__p95"] = float(values.quantile(0.95)) - row[f"{metric}__weighted"] = float( - (values * weights).sum() / weights.sum() - if weights.sum() > 0 - else np.nan - ) - rows.append(row) - cols = ["approach", "n_sectors"] + [ - f"{m}__{r}" for m in metrics for r in ("median", "p95", "weighted") - ] - ranking = pd.DataFrame(rows, columns=pd.Index(cols)) - return ranking.sort_values("mean_abs_yoy_pct__weighted").reset_index(drop=True) - - -def _select_head_sectors( - per_sector: pd.DataFrame, - cumulative_share: float = LINE_PLOT_CUMULATIVE_SHARE, - max_sectors: int = LINE_PLOT_MAX_SECTORS, -) -> list[str]: - """Return the smallest set of sectors covering ``cumulative_share`` of total |mean_N|. - - Across approaches, take the per-sector ``|mean_N|`` averaged over methods, - sort descending, and accumulate until coverage hits the threshold (capped - by ``max_sectors``). These are the lines that actually move - corporate-footprint outcomes. - """ - avg_abs_mean = ( - per_sector.groupby("sector")["mean_N"].mean().abs().sort_values(ascending=False) - ) - total = float(avg_abs_mean.sum()) - if total <= 0: - return list(avg_abs_mean.head(max_sectors).index) - cum = avg_abs_mean.cumsum() / total - head = avg_abs_mean[cum <= cumulative_share] - # Always include the next sector that pushes us across the threshold. - if len(head) < len(avg_abs_mean): - head = avg_abs_mean.iloc[: len(head) + 1] - return list(head.head(max_sectors).index) - - -def _indexed_lines_plot( - panel: pd.DataFrame, - per_sector: pd.DataFrame, - out_path: Path, -) -> None: - """Head-sector indexed (2019=100) line plot, faceted by method.""" - approaches = sorted(panel["approach"].unique()) - head_sectors = _select_head_sectors(per_sector) - head_share = ( - per_sector[per_sector["sector"].isin(head_sectors)] - .groupby("sector")["mean_N"] - .mean() - .abs() - .sum() - / per_sector.groupby("sector")["mean_N"].mean().abs().sum() - ) - cmap = plt.get_cmap("tab10") - color_by_sector = {s: cmap(i % 10) for i, s in enumerate(head_sectors)} - - n = len(approaches) - fig, axes = plt.subplots(1, n, figsize=(5 * n, 5.5), sharey=True) - if n == 1: - axes = np.array([axes]) - for ax, approach in zip(axes, approaches): - sub = panel[ - (panel["approach"] == approach) & panel["sector"].isin(head_sectors) - ] - for sector in head_sectors: - sg = sub[sub["sector"] == sector].sort_values("year") - if sg.empty or sg["year"].min() != YOY_YEARS[0]: - continue - base = sg.loc[sg["year"] == YOY_YEARS[0], "N_new_ref"].iloc[0] - if base == 0 or pd.isna(base): - continue - indexed = sg["N_new_ref"] / base * 100 - ax.plot( - sg["year"], - indexed, - marker="o", - label=sector, - color=color_by_sector[sector], - ) - ax.axhline(100, color="black", linestyle=":", linewidth=1, alpha=0.6) - ax.set_title(approach, fontsize=14) - ax.set_xlabel("Year") - ax.set_xticks(list(YOY_YEARS)) - ax.set_xticklabels([str(y) for y in YOY_YEARS]) - ax.grid(alpha=0.3) - axes[0].set_ylabel("N indexed (2019 = 100)") - handles, labels = axes[0].get_legend_handles_labels() - fig.legend( - handles, - labels, - loc="center right", - fontsize=8, - bbox_to_anchor=(1.05, 0.5), - title="sector", - framealpha=0.4, - ) - fig.suptitle( - f"Head sectors covering {head_share:.0%} of |mean_N| " - f"(n={len(head_sectors)}), N rebased to 2019=100, by A-matrix method", - fontsize=14, - ) - fig.tight_layout() - out_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(out_path, dpi=120, bbox_inches="tight") - plt.close(fig) - - -def _pooled_signed_yoy(big: pd.DataFrame, approach: str) -> pd.Series: - """All sector×transition signed YoYs for ``approach``, no per-sector averaging.""" - sub = big[big["approach"] == approach] - return pd.concat([sub[f"yoy_{y0}_{y1}"] for y0, y1 in YOY_TRANSITIONS]).dropna() - - -def _clip_to_percentile(values: np.ndarray, lo: float, hi: float) -> np.ndarray: - """Drop values outside ``[lo, hi]`` percentiles so the violin KDE shows the - distribution body, not the outlier tail. - - Outlier-driven KDEs collapse the visible mass; clipping keeps the violin - informative. The ECDF panel is computed on the unclipped data so the tail - is still visible there. - """ - if values.size == 0: - return values - p_lo, p_hi = np.percentile(values, [lo, hi]) - return values[(values >= p_lo) & (values <= p_hi)] - - -def _yoy_signed_violin_plot( - per_sector: pd.DataFrame, - out_path: Path, - exclude_approaches: frozenset[str] = frozenset(), - violin_ylim: tuple[float, float] = (-30.0, 30.0), -) -> None: - """2-panel summary of per-sector signed YoY across methods (violin form). - - Keeps the sign of YoY (no abs) so over- vs under-shoots are - distinguishable; uses violins to show the full shape of the distribution. - - Left: pooled signed YoY per method (all sector×transition values). - Right: signed YoY per transition, grouped violins per method. - - ``exclude_approaches`` drops those approach names from the panels (e.g. - to zoom on the spread between the remaining methods). Colors are assigned - from the full sorted approach list before exclusion so each method keeps - its identity across variants of this plot. - """ - # Use the shared per-approach palette so colors stay consistent with the - # histogram / line plots elsewhere in this package. - all_approaches = sorted(per_sector["approach"].unique()) - colors = {a: APPROACH_COLORS.get(a, "#7f7f7f") for a in all_approaches} - approaches = [a for a in all_approaches if a not in exclude_approaches] - - cutoff = per_sector["mean_N"].abs().quantile(MIN_MEAN_PERCENTILE / 100) - big = per_sector[per_sector["mean_N"].abs() >= cutoff] - - # Plot in percent units (YoY × 100) so axes read 0–30 = 0–30 %. - pooled_by_approach = {a: _pooled_signed_yoy(big, a) * 100 for a in approaches} - - # Symmetric tick grid matching the requested ``violin_ylim``. - lo, hi = violin_ylim - span = int(max(abs(lo), abs(hi))) - step = 10 - percent_ticks = list(range(-span, span + 1, step)) - percent_tick_labels = [f"{t}%" for t in percent_ticks] - - fig, axes = plt.subplots(1, 2, figsize=(13, 5.5)) - - # Left: pooled signed YoY per method (one violin per method). - ax = axes[0] - pooled_data = [ - _clip_to_percentile(pooled_by_approach[a].to_numpy(), 1, 99) for a in approaches - ] - positions = list(range(1, len(approaches) + 1)) - parts = ax.violinplot( - pooled_data, - positions=positions, - showmedians=True, - showextrema=True, - widths=0.75, - ) - # `violinplot` returns one merged collection for bodies; recolor per-method. - for body, approach in zip(parts["bodies"], approaches): - body.set_facecolor(colors[approach]) - body.set_edgecolor(colors[approach]) - body.set_alpha(0.55) - for key in ("cmedians", "cmaxes", "cmins", "cbars"): - if key in parts: - parts[key].set_color("black") - parts[key].set_linewidth(1.0) - parts[key].set_alpha(0.7) - ax.axhline(0, color="black", linewidth=0.8, alpha=0.6) - ax.set_xticks(positions) - ax.set_xticklabels(approaches) - ax.set_title("Pooled YoY (all sector-years)", fontsize=13) - ax.set_ylabel("YoY (signed, 0 = no change)") - ax.set_ylim(*violin_ylim) - ax.set_yticks(percent_ticks) - ax.set_yticklabels(percent_tick_labels) - legend_handles = [ - Patch(facecolor=colors[a], alpha=0.55, label=a) for a in approaches - ] + [ - Line2D([], [], color="black", linewidth=1.0, alpha=0.7, label="median"), - Line2D( - [], - [], - color="black", - linewidth=1.0, - marker="_", - markersize=13, - markeredgewidth=1.2, - alpha=0.7, - label="min / max (clipped to 1–99 pct)", - ), - ] - # Semi-transparent so the violin tails behind the legend remain visible. - ax.legend(handles=legend_handles, loc="upper left", fontsize=11, framealpha=0.4) - ax.grid(axis="y", alpha=0.3) - - # Right: per-transition signed YoY, grouped violins per method. - ax = axes[1] - transition_labels = [f"{y0}→{y1}" for y0, y1 in YOY_TRANSITIONS] - n_methods = len(approaches) - width = 0.8 / n_methods - for i, approach in enumerate(approaches): - sub = big[big["approach"] == approach] - per_transition = [ - _clip_to_percentile(sub[f"yoy_{y0}_{y1}"].dropna().to_numpy() * 100, 1, 99) - for y0, y1 in YOY_TRANSITIONS - ] - transition_positions: list[float] = [ - j + 1 + (i - (n_methods - 1) / 2) * width - for j in range(len(YOY_TRANSITIONS)) - ] - # Skip empty arrays — matplotlib violinplot errors on zero-size data. - good = [ - (p, d) for p, d in zip(transition_positions, per_transition) if d.size > 0 - ] - if not good: - continue - good_positions, good_data = zip(*good) - parts = ax.violinplot( - list(good_data), - positions=list(good_positions), - widths=width * 0.95, - showmedians=True, - showextrema=False, - ) - for body in parts["bodies"]: - body.set_facecolor(colors[approach]) - body.set_edgecolor(colors[approach]) - body.set_alpha(0.55) - if "cmedians" in parts: - parts["cmedians"].set_color("black") - parts["cmedians"].set_linewidth(1.0) - parts["cmedians"].set_alpha(0.7) - ax.axhline(0, color="black", linewidth=0.8, alpha=0.6) - ax.set_xticks([j + 1 for j in range(len(YOY_TRANSITIONS))]) - ax.set_xticklabels(transition_labels) - ax.set_title("YoY per transition, by method", fontsize=13) - ax.set_ylabel("YoY (signed)") - ax.set_ylim(*violin_ylim) - ax.set_yticks(percent_ticks) - ax.set_yticklabels(percent_tick_labels) - ax.grid(axis="y", alpha=0.3) - - fig.suptitle( - "Signed year-over-year change in N across A-matrix methods", fontsize=15 - ) - fig.tight_layout() - out_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(out_path, dpi=120, bbox_inches="tight") - plt.close(fig) - - -def _print_summary(ranking: pd.DataFrame, per_sector: pd.DataFrame) -> None: - print("\n=== YoY stability ranking (lower = more stable) ===") - cols_show = [ - "approach", - "mean_abs_yoy_pct__weighted", - "mean_abs_yoy_pct__median", - "max_abs_yoy_pct__weighted", - "abs_total_drift_pct__weighted", - ] - print(ranking[cols_show].round(4).to_string(index=False)) - - print("\n=== Top-5 most-fluctuating big-N sectors per method ===") - cutoff = per_sector["mean_N"].abs().quantile(MIN_MEAN_PERCENTILE / 100) - big = per_sector[per_sector["mean_N"].abs() >= cutoff] - for approach in sorted(big["approach"].unique()): - grp = big[big["approach"] == approach] - worst = grp.nlargest(n=5, columns="mean_abs_yoy_pct")[ - [ - "sector", - "mean_N", - "mean_abs_yoy_pct", - "max_abs_yoy_pct", - "total_drift_pct", - ] - ] - print(f"\n[{approach}]") - print(worst.round(4).to_string(index=False)) - - -def main() -> None: - panel = _read_panel(EF_COMPARISON_XLSX_PATH) - per_sector = _yoy_per_sector(panel) - ranking = _aggregate_yoy_per_method(per_sector) - - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - ranking.to_csv(YOY_RANKING_PATH, index=False) - per_sector.to_csv(YOY_PER_SECTOR_PATH, index=False) - _indexed_lines_plot(panel, per_sector, INDEXED_LINES_PLOT_PATH) - # Signed violin, dropping industry_price_index (nearly co-linear with - # commodity_pi) and zooming ylim to ±40% so the summary_tables shape is - # legible. The all-method variants (n_yoy_distribution / n_yoy_signed_violin) - # were removed — they made the figure crowded without adding signal. - _yoy_signed_violin_plot( - per_sector, - YOY_SIGNED_VIOLIN_NO_IPI_PLOT_PATH, - exclude_approaches=frozenset({"industry_price_index"}), - violin_ylim=(-40.0, 40.0), - ) - - _print_summary(ranking, per_sector) - print(f"\nWrote: {YOY_RANKING_PATH}") - print(f"Wrote: {YOY_PER_SECTOR_PATH}") - print(f"Wrote: {INDEXED_LINES_PLOT_PATH}") - print(f"Wrote: {YOY_SIGNED_VIOLIN_NO_IPI_PLOT_PATH}") - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format="%(message)s") - main() diff --git a/bedrock/analysis/a_matrix_time_series/compare_price_ratios.py b/bedrock/analysis/a_matrix_time_series/compare_price_ratios.py deleted file mode 100644 index f5a22f1b..00000000 --- a/bedrock/analysis/a_matrix_time_series/compare_price_ratios.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Compare V-norm-derived commodity price ratio vs industry price ratio. - -Step 0 sanity-check companion to epic #337. Confirms whether the two -inflation approaches produce meaningfully different price ratios; if they -collapse to the same numbers the downstream analysis is moot. - -Outputs (gitignored under `output/`): -- ratio_summary.csv : per-year distribution stats for both ratios. -- ratio_per_code.csv : long-format with `vnorm_col_sum` for diagnosing - zero-V_norm-column anomalies. -- ratio_scatter.png : per-year scatter, industry (x) vs commodity (y), - with `y=x` reference. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compare_price_ratios -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import cast - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import ( - LATEST_TARGET_YEAR, - ORIGINAL_YEAR, - PLOTS_DIR, - RESULTS_DIR, -) -from bedrock.transform.eeio.derived_cornerstone import ( - derive_cornerstone_Vnorm_scrap_corrected, -) -from bedrock.utils.config.usa_config import get_usa_config -from bedrock.utils.economic.inflation_helpers_cornerstone import ( - get_cornerstone_industry_price_ratio, - get_vnorm_adjusted_commodity_price_ratio, -) - -logger = logging.getLogger(__name__) - -# Skip ORIGINAL_YEAR (year-to-self ratio is trivially 1.0). -TARGET_YEARS: list[int] = list(range(ORIGINAL_YEAR + 1, LATEST_TARGET_YEAR + 1)) - - -def build_comparison_long( - target_years: list[int] = TARGET_YEARS, original_year: int = ORIGINAL_YEAR -) -> pd.DataFrame: - - apply_inflation_to_V = get_usa_config().apply_inflation_to_V - rows: list[pd.DataFrame] = [] - for year in target_years: - Vnorm = derive_cornerstone_Vnorm_scrap_corrected( - apply_inflation=apply_inflation_to_V, target_year=year - ) - vnorm_col_sum = Vnorm.sum(axis=0).rename("vnorm_col_sum") - industry = get_cornerstone_industry_price_ratio( - original_year, target_year=year - ).rename("industry_ratio") - commodity = get_vnorm_adjusted_commodity_price_ratio( - original_year, target_year=year - ).rename("commodity_ratio") - df = pd.concat([industry, commodity, vnorm_col_sum], axis=1).reset_index( - names="code" - ) - df["year"] = year - df["abs_delta"] = (df["commodity_ratio"] - df["industry_ratio"]).abs() - df["rel_delta"] = df["abs_delta"] / df["industry_ratio"].replace(0, np.nan) - rows.append(df) - return pd.concat(rows, ignore_index=True)[ - [ - "year", - "code", - "industry_ratio", - "commodity_ratio", - "abs_delta", - "rel_delta", - "vnorm_col_sum", - ] - ] - - -def summarize(long: pd.DataFrame) -> pd.DataFrame: - def stats(s: pd.Series) -> dict[str, float]: - return { - "mean": float(s.mean()), - "median": float(s.median()), - "p05": float(s.quantile(0.05)), - "p95": float(s.quantile(0.95)), - "min": float(s.min()), - "max": float(s.max()), - } - - rows = [] - for year, group in long.groupby("year"): - rows.append( - { - "year": cast("int", year), - **{f"ind_{k}": v for k, v in stats(group["industry_ratio"]).items()}, - **{f"com_{k}": v for k, v in stats(group["commodity_ratio"]).items()}, - "rel_delta_p95": float(group["rel_delta"].quantile(0.95)), - "n_rel_delta_gt_1pct": int((group["rel_delta"] > 0.01).sum()), - "n_zero_vnorm_col": int((group["vnorm_col_sum"] < 1e-6).sum()), - } - ) - return pd.DataFrame(rows) - - -def plot_scatter(long: pd.DataFrame, path: Path) -> None: - years = sorted(long["year"].unique()) - n = len(years) - cols = 3 - rows = (n + cols - 1) // cols - fig, axes = plt.subplots(rows, cols, figsize=(4 * cols, 4 * rows), squeeze=False) - for i, year in enumerate(years): - ax = axes[i // cols][i % cols] - sub = long[long["year"] == year] - ax.scatter(sub["industry_ratio"], sub["commodity_ratio"], s=8, alpha=0.4) - lo = float(min(sub["industry_ratio"].min(), sub["commodity_ratio"].min())) - hi = float(max(sub["industry_ratio"].max(), sub["commodity_ratio"].max())) - ax.plot([lo, hi], [lo, hi], "k--", lw=0.5, label="y=x") - ax.set_xlabel("industry price ratio") - ax.set_ylabel("V-norm commodity price ratio") - ax.set_title(f"2017 → {year}") - ax.legend(loc="lower right", fontsize=8, framealpha=0.4) - for j in range(n, rows * cols): - axes[j // cols][j % cols].axis("off") - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -def main() -> None: - addon = "_V_inflated" if get_usa_config().apply_inflation_to_V else "" - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - long = build_comparison_long(TARGET_YEARS) - long.to_csv(RESULTS_DIR / ("ratio_per_code" + addon + ".csv"), index=False) - summary = summarize(long) - summary.to_csv(RESULTS_DIR / ("ratio_summary" + addon + ".csv"), index=False) - plot_scatter(long, PLOTS_DIR / ("ratio_scatter" + addon + ".png")) - logger.info("Wrote outputs to %s and %s", RESULTS_DIR, PLOTS_DIR) - print(summary.to_string(index=False)) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/compare_summary_a_errors.py b/bedrock/analysis/a_matrix_time_series/compare_summary_a_errors.py deleted file mode 100644 index 58086c43..00000000 --- a/bedrock/analysis/a_matrix_time_series/compare_summary_a_errors.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Step 5 of epic #337 (Option 2 refined): cell-level A comparison at BEA -summary aggregation, ranked by weighted RMSE per (approach, year). - -The original Step 5 spec — comparing the model's saved ``q`` vector to BEA -gross output — turned out not to test A: ``q`` is produced by -``scale_cornerstone_q`` + ``inflate_cornerstone_q_or_y``, helpers that run -in parallel to A and never depend on it. - -The first reframing (Option 2) tried to compare ``col_sum(A)`` to BEA -observed II/GO share per industry. That ran into a data wall (annual II -isn't in the codebase at detail level) AND was column-sum-only — a cell -with the wrong row distribution but right column total would pass. - -This module ships **Option 2 refined**: it uses BEA's *summary-level* A -matrix, available annually via ``derive_summary_Adom_usa(year)`` + -``derive_summary_Aimp_usa(year)``, as ground truth. For each -(approach, year) it aggregates the Cornerstone detail A to summary level -via observed dollar-flow weighting (``Z = A · diag(q)`` then groupby -summary parent, then divide by aggregated q) and compares to the BEA -observed summary A cell by cell. Tests A's row distribution AND column -structure at summary granularity — column-sum caveat resolved. - -See [issue #344 reframing comment](https://github.com/cornerstone-data/bedrock/issues/344#issuecomment-4357342817). - -Reads the ``A_{approach}_{year}.parquet`` and ``q_{approach}_{year}.parquet`` -caches from Step 1 and the BEA summary A from -``bedrock.transform.eeio.derived_2017``. Produces: - -- ``summary_a_errors.csv`` — one row per (approach, year, dom_or_imp) with - ``rmse_vs_bea_summary_a`` (Z-magnitude weighted), ``mean_abs_diff``, - ``top_5_worst_cells`` (semicolon-joined ``ROW->COL:diff`` pairs). - -- ``summary_a_rmse_ranking.png`` — grouped bar chart, x = year, five bars - per group (one per approach), y = weighted RMSE for the combined - (dom + imp) A matrix. - -- Sheet tab ``summary_a_errors`` appended to the run-report Sheet. - -**Caveat:** weighting and aggregation use each approach's own ``q`` -vector (from the parquet cache). Approaches with poorer q-scaling get a -slightly biased weight. A future refinement would use BEA observed -detail GO as the common weight — pending a Cornerstone-detail-to-BEA- -detail GO split for sectors where Cornerstone disaggregates. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compare_summary_a_errors -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series._run_report import publish_tabs -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_COLORS, - APPROACH_ORDER, - APPROACH_YEAR_COVERAGE, - PLOTS_DIR, - RESULTS_DIR, -) -from bedrock.transform.eeio.derived_2017 import ( - derive_summary_Adom_usa, - derive_summary_Aimp_usa, -) -from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( - load_bea_v2017_summary_to_cornerstone, -) - -logger = logging.getLogger(__name__) - -SUMMARY_A_ERROR_YEARS: tuple[int, ...] = ( - 2017, - 2018, - 2019, - 2020, - 2021, - 2022, - 2023, - 2024, -) - -TOP_K_WORST_CELLS_REPORTED = 5 - - -def _cornerstone_to_summary() -> dict[str, str]: - """Invert ``load_bea_v2017_summary_to_cornerstone`` into a - cornerstone_code → bea_summary_code lookup.""" - summary_to_cornerstone = load_bea_v2017_summary_to_cornerstone() - return { - code: str(summary) - for summary, codes in summary_to_cornerstone.items() - for code in codes - } - - -def _load_a_total_dom_plus_imp(approach: str, year: int) -> pd.DataFrame: - """A_total = A_dom + A_imp, str-typed indices.""" - combined = pd.read_parquet(RESULTS_DIR / f"A_{approach}_{year}.parquet") - adom = pd.DataFrame(combined.loc["dom"]) - aimp = pd.DataFrame(combined.loc["imp"]) - a = adom.add(aimp, fill_value=0.0) - a.index = a.index.astype(str) - a.columns = a.columns.astype(str) - return a - - -def _load_q_detail(approach: str, year: int) -> pd.Series: - df = pd.read_parquet(RESULTS_DIR / f"q_{approach}_{year}.parquet") - series = pd.Series(df["q"].astype(float)) - series.index = series.index.astype(str) - return series - - -def aggregate_detail_a_to_summary( - a_detail: pd.DataFrame, q_detail: pd.Series, cs_to_summary: dict[str, str] -) -> pd.DataFrame: - """Aggregate detail-level A to summary level via dollar-flow weighting. - - Method: ``Z_detail[i,j] = A_detail[i,j] · q_detail[j]`` (commodity i - used to make commodity j, in $). Group rows and columns by their BEA - summary parent, sum, and divide by aggregated ``q`` to get summary A - coefficients. This is the same construction BEA uses to derive - summary A from a detail Use table. - """ - common_idx = a_detail.index.intersection(q_detail.index) - a_aligned = a_detail.loc[common_idx, common_idx] - q_aligned = q_detail.reindex(common_idx).fillna(0.0) - - z = a_aligned.multiply(q_aligned.to_numpy(), axis=1) - cs_to_sum = {k: v for k, v in cs_to_summary.items() if k in common_idx} - detail_to_summary_index = pd.Index( - [cs_to_sum.get(c, "UNMAPPED") for c in common_idx] - ) - - z.index = detail_to_summary_index - z.columns = detail_to_summary_index - z_summary = z.groupby(level=0).sum().T.groupby(level=0).sum().T - - q_summary = pd.Series(q_aligned.to_numpy(), index=detail_to_summary_index) - q_summary = q_summary.groupby(level=0).sum() - - safe_q = q_summary.replace(0, np.nan) - a_summary = z_summary.divide(safe_q, axis=1) - a_summary = a_summary.fillna(0.0) - a_summary = a_summary.drop(index="UNMAPPED", errors="ignore").drop( - columns="UNMAPPED", errors="ignore" - ) - return a_summary - - -def _bea_observed_summary_a(year: int) -> pd.DataFrame: - """BEA observed summary A = ``A_dom_summary + A_imp_summary`` at year.""" - return derive_summary_Adom_usa(year).add( - derive_summary_Aimp_usa(year), fill_value=0.0 - ) - - -def _cell_errors_one_pair( - a_pred: pd.DataFrame, a_obs: pd.DataFrame -) -> tuple[float, float, list[tuple[str, str, float]]]: - """Per (approach, year) cell-level summary stats. - - Both inputs are summary-level A. Returns: - - Z-magnitude-weighted RMSE (cells with bigger flows dominate) - - mean absolute diff (uniform-weighted) - - top-K worst cells by signed difference - """ - common_rows = a_pred.index.intersection(a_obs.index) - common_cols = a_pred.columns.intersection(a_obs.columns) - pred = a_pred.loc[common_rows, common_cols].to_numpy() - obs = a_obs.loc[common_rows, common_cols].to_numpy() - diff = pred - obs - - abs_obs = np.abs(obs) - weight_sum = abs_obs.sum() - weights = ( - abs_obs / weight_sum if weight_sum > 0 else np.ones_like(abs_obs) / abs_obs.size - ) - rmse = float(np.sqrt(np.sum(weights * diff**2))) - mean_abs = float(np.abs(diff).mean()) - - flat = diff.flatten() - flat_abs = np.abs(flat) - worst = np.argsort(flat_abs)[::-1][:TOP_K_WORST_CELLS_REPORTED] - rows_arr = np.asarray(common_rows) - cols_arr = np.asarray(common_cols) - n_cols = len(common_cols) - worst_cells = [ - (str(rows_arr[i // n_cols]), str(cols_arr[i % n_cols]), float(flat[i])) - for i in worst - ] - return rmse, mean_abs, worst_cells - - -def compute_errors_table() -> pd.DataFrame: - """Per (approach, year) cell-level errors at summary aggregation.""" - cs_to_summary = _cornerstone_to_summary() - rows: list[dict[str, object]] = [] - for year in SUMMARY_A_ERROR_YEARS: - try: - a_summary_obs = _bea_observed_summary_a(year) - except Exception as e: # noqa: BLE001 - logger.warning("BEA summary A unavailable for year=%d (%s)", year, e) - continue - for approach in APPROACH_ORDER: - # Skip year/approach combos the approach doesn't cover. Avoids - # log noise for known gaps (e.g. useeio_nowcast has no 2024). - coverage = APPROACH_YEAR_COVERAGE.get(approach) - if coverage is not None and year not in coverage: - continue - a_path = RESULTS_DIR / f"A_{approach}_{year}.parquet" - q_path = RESULTS_DIR / f"q_{approach}_{year}.parquet" - if not (a_path.exists() and q_path.exists()): - logger.warning("Missing %s or %s — skipping", a_path, q_path) - continue - a_detail = _load_a_total_dom_plus_imp(approach, year) - q_detail = _load_q_detail(approach, year) - a_summary_pred = aggregate_detail_a_to_summary( - a_detail, q_detail, cs_to_summary - ) - rmse, mean_abs, worst_cells = _cell_errors_one_pair( - a_summary_pred, a_summary_obs - ) - rows.append( - { - "approach": approach, - "year": year, - "rmse_vs_bea_summary_a": rmse, - "mean_abs_diff": mean_abs, - "n_summary_cells": int( - a_summary_pred.shape[0] * a_summary_pred.shape[1] - ), - "top_5_worst_cells": "; ".join( - f"{r}->{c}:{d:+.3f}" for r, c, d in worst_cells - ), - } - ) - return pd.DataFrame(rows) - - -def plot_rmse_ranking(errors_df: pd.DataFrame, path: Path) -> None: - """Grouped bar chart: x = year, 5 bars per group (one per approach). - - y = Z-magnitude-weighted cell-level RMSE between predicted summary A - (Cornerstone aggregated) and observed summary A (BEA). Tests A's full - cell shape at summary aggregation, not just column sums. - """ - pivot = errors_df.pivot_table( - index="year", columns="approach", values="rmse_vs_bea_summary_a" - ).reindex(columns=APPROACH_ORDER) - - n_years = len(pivot.index) - n_approaches = len(APPROACH_ORDER) - bar_w = 0.8 / n_approaches - x = np.arange(n_years) - - fig, ax = plt.subplots(figsize=(1.4 * n_years + 2, 5)) - fig.suptitle( - "Cell-level RMSE of Cornerstone A vs BEA observed summary A — by approach", - fontsize=11, - ) - - for i, approach in enumerate(APPROACH_ORDER): - if approach not in pivot.columns: - continue - offset = (i - (n_approaches - 1) / 2) * bar_w - vals = pivot[approach].to_numpy() - ax.bar( - x + offset, - vals, - width=bar_w, - color=APPROACH_COLORS[approach], - label=approach, - edgecolor="white", - linewidth=0.4, - ) - - ax.set_xticks(x) - ax.set_xticklabels([str(y) for y in pivot.index]) - ax.set_xlabel("year") - ax.set_ylabel("Z-weighted RMSE") - ax.grid(True, axis="y", alpha=0.3) - ax.legend(loc="upper left", fontsize=9, framealpha=0.4) - - fig.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - logger.info("Computing summary-A cell-level errors across approaches × years") - errors_df = compute_errors_table() - errors_df.to_csv(RESULTS_DIR / "summary_a_errors.csv", index=False) - - plot_rmse_ranking(errors_df, PLOTS_DIR / "summary_a_rmse_ranking.png") - - publish_tabs({"summary_a_errors": errors_df}) - logger.info("Step 5 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/compile_ef_diagnostics.py b/bedrock/analysis/a_matrix_time_series/compile_ef_diagnostics.py deleted file mode 100644 index bd00d535..00000000 --- a/bedrock/analysis/a_matrix_time_series/compile_ef_diagnostics.py +++ /dev/null @@ -1,338 +0,0 @@ -"""Compile per-(approach, baseline) EF diagnostics Sheets into one workbook. - -Step 6 / Phase 2 of epic #337. Reads ``output/results/ef_run_index.csv`` for -the ``(approach, baseline, sheet_id)`` triples, pulls the EF diff tabs from -each Sheet, and produces: - -- ``output/results/ef_comparison.xlsx``: - - One tab per ``(approach, baseline)`` carrying the full ``N_and_diffs`` - joined with ``D_and_diffs``. - - ``summary_vs_useeio`` and ``summary_vs_ceda``: per-approach roll-up - with p50 / p95 / max of ``|N_perc_diff|`` and ``|D_perc_diff|``, plus - ``n_significant`` (sectors where the percent diff exceeds - ``SIGNIFICANT_PCT_THRESHOLD``). -- ``output/results/ef_scatter_coords.parquet``: long-format coordinates - ``(scenario, year, approach, baseline, ef_kind, sector, x_baseline, - y_approach)`` for the Phase 3 scatter plots. ``x_baseline`` is - ``*_old_inflated`` (the baseline's EF, inflation-adjusted to the - candidate's base year) and ``y_approach`` is ``*_new``. ``scenario`` - distinguishes runs where only the A-matrix derivation differs - (``isolate_a_matrix``) from runs that bundle the A-matrix change with - the full v0.3 stack (``bundle_v0_3``). -- ``ef_summary_vs_useeio`` and ``ef_summary_vs_ceda`` tabs appended to the - run-report Sheet (sheet ID from ``last_run_sheet_id.txt``, written by - Step 1). Skipped with a warning if that file is missing. - -The compile script reads only ``sheet_id`` from ``ef_run_index.csv``; -``run_id``, ``useeio_box_ticked`` and ``triggered_at`` are audit-only. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.compile_ef_diagnostics -""" - -from __future__ import annotations - -import logging - -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import ( - LAST_RUN_SHEET_ID_PATH, - RESULTS_DIR, -) -from bedrock.utils.io.gcp import read_sheet_tab, update_sheet_tab -from bedrock.utils.validation.diagnostics_helpers import ( - inflation_adjust_ef_denom_to_new_base_year, -) - -logger = logging.getLogger(__name__) - -EF_RUN_INDEX_PATH = RESULTS_DIR / "ef_run_index.csv" -EF_COMPARISON_XLSX_PATH = RESULTS_DIR / "ef_comparison.xlsx" -EF_SCATTER_COORDS_PATH = RESULTS_DIR / "ef_scatter_coords.parquet" - -# A row counts as "significantly different" when |perc_diff| exceeds this. -SIGNIFICANT_PCT_THRESHOLD = 0.10 - -# Legacy single-year EF diagnostics runs (Step 6, pre-dispatch) lack a -# ``scenario`` value in the run index. They were generated from the -# `2025_usa_cornerstone_A_*` YAMLs, which flip only the A-matrix flag, so -# retag them with this label. -ISOLATE_A_MATRIX_SCENARIO = "isolate_a_matrix" - -# Time-series cells differ in `model_base_year`, so each cell's `D_new` / -# `N_new` lives in its own dollar year. Deflate them to this common reference -# so values are commensurable across years. -REFERENCE_DOLLAR_YEAR = 2023 - -# Tab names produced by `calculate_ef_diagnostics.py` per run. Both -# baseline modes (CEDA-only and USEEIO-checked) emit identical column -# headers — confirmed via spot-check. -TAB_N = "N_and_diffs" -TAB_D = "D_and_diffs" - -# Numeric columns to coerce after `read_sheet_tab` (which returns all str). -_N_NUMERIC_COLS = ("N_new", "N_old", "N_old_inflated", "N_perc_diff") -_D_NUMERIC_COLS = ("D_new", "D_old", "D_old_inflated", "D_perc_diff") - - -def _coerce_numeric(df: pd.DataFrame, cols: tuple[str, ...]) -> pd.DataFrame: - """Coerce string columns to numeric, handling percent-formatted cells. - - Sheets returns formatted display values by default, so a column with - percent cell formatting comes back like ``"0.23%"`` even though the - underlying value is the fraction ``0.0023``. Strip the suffix and - divide by 100; raw-float columns pass through unchanged. - """ - for col in cols: - if col not in df.columns: - continue - s = df[col].astype(str).str.strip() - is_pct = s.str.endswith("%") - cleaned = s.str.rstrip("%").str.replace(",", "", regex=False) - numeric = pd.to_numeric(cleaned, errors="coerce") - df[col] = numeric.mask(is_pct, numeric / 100) - return df - - -def _read_pair(sheet_id: str) -> pd.DataFrame: - """Return ``N_and_diffs`` joined with ``D_and_diffs`` on the sector index.""" - n = _coerce_numeric(read_sheet_tab(sheet_id, TAB_N), _N_NUMERIC_COLS) - d = _coerce_numeric(read_sheet_tab(sheet_id, TAB_D), _D_NUMERIC_COLS) - # Both tabs come back with the sector code as the first column; align on it. - sector_col = n.columns[0] - n = n.set_index(sector_col) - d = d.set_index(sector_col) - # `sector_name` and `comparison_type` are duplicated across N and D — keep - # them from N only. - drop_from_d = [c for c in ("sector_name", "comparison_type") if c in d.columns] - return n.join(d.drop(columns=drop_from_d), how="outer") - - -def _deflate_new_to_ref( - joined: pd.DataFrame, source_year: int, ref_year: int -) -> pd.DataFrame: - """Add ``D_new_ref`` / ``N_new_ref`` columns deflated to ``ref_year`` dollars. - - Each diagnostics cell is run with ``model_base_year=source_year``, so - ``D_new`` / ``N_new`` are denominated in ``source_year`` dollars. Time-series - plots over years require a common dollar reference; this multiplies by the - same per-sector price ratio used for baseline alignment. - """ - if source_year == ref_year: - if "D_new" in joined.columns: - joined["D_new_ref"] = joined["D_new"] - if "N_new" in joined.columns: - joined["N_new_ref"] = joined["N_new"] - return joined - for new_col, ref_col in (("D_new", "D_new_ref"), ("N_new", "N_new_ref")): - if new_col not in joined.columns: - continue - joined[ref_col] = inflation_adjust_ef_denom_to_new_base_year( - old_ef_vector=joined[new_col].astype(float), - new_base_year=ref_year, - old_base_year=source_year, - ) - return joined - - -def _summarize(joined: pd.DataFrame, approach: str) -> pd.Series: - n_perc = joined["N_perc_diff"].abs() - d_perc = joined["D_perc_diff"].abs() - return pd.Series( - { - "approach": approach, - "n_sectors": int(joined.shape[0]), - "N_p50": float(n_perc.quantile(0.50)), - "N_p95": float(n_perc.quantile(0.95)), - "N_max": float(n_perc.max()), - "N_n_significant": int((n_perc > SIGNIFICANT_PCT_THRESHOLD).sum()), - "D_p50": float(d_perc.quantile(0.50)), - "D_p95": float(d_perc.quantile(0.95)), - "D_max": float(d_perc.max()), - "D_n_significant": int((d_perc > SIGNIFICANT_PCT_THRESHOLD).sum()), - } - ) - - -def _scatter_coords( - joined: pd.DataFrame, - approach: str, - baseline: str, - scenario: str, - year: str, -) -> pd.DataFrame: - """Long-format scatter coords with run dimensions stamped on each row.""" - rows: list[pd.DataFrame] = [] - for kind, new_col, old_col in ( - ("N", "N_new", "N_old_inflated"), - ("D", "D_new", "D_old_inflated"), - ): - if new_col not in joined.columns or old_col not in joined.columns: - continue - chunk = joined[[new_col, old_col]].dropna().copy() - chunk.columns = pd.Index(["y_approach", "x_baseline"]) - chunk = chunk.reset_index().rename(columns={chunk.index.name: "sector"}) - chunk["scenario"] = scenario - chunk["year"] = year - chunk["approach"] = approach - chunk["baseline"] = baseline - chunk["ef_kind"] = kind - rows.append(chunk) - if not rows: - return pd.DataFrame() - combined = pd.concat(rows, ignore_index=True) - return combined.loc[ - :, - [ - "scenario", - "year", - "approach", - "baseline", - "ef_kind", - "sector", - "x_baseline", - "y_approach", - ], - ] - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - - if not EF_RUN_INDEX_PATH.exists(): - raise FileNotFoundError( - f"{EF_RUN_INDEX_PATH} not found.\n" - "Two ways to populate it:\n" - " (a) Auto-rebuild from existing Sheets in the diagnostics " - "Drive folder:\n" - " python -m bedrock.analysis.a_matrix_time_series." - "recover_ef_run_index --folder-id \n" - " (b) Hand-write a CSV with at least 3 columns " - "(approach, baseline, sheet_id), one row per Sheet." - ) - index_df = pd.read_csv(EF_RUN_INDEX_PATH) - required = {"approach", "baseline", "sheet_id"} - missing = required - set(index_df.columns) - if missing: - raise ValueError(f"{EF_RUN_INDEX_PATH} missing columns: {sorted(missing)}") - - # `scenario` and `year` are optional. Step 6 runs lack them; Step 7 - # time-series dispatch populates both. Default to empty so the same - # script handles both schemas. - if "scenario" not in index_df.columns: - index_df["scenario"] = "" - if "year" not in index_df.columns: - index_df["year"] = "" - - summaries_by_baseline: dict[str, list[pd.Series]] = {} - scatter_chunks: list[pd.DataFrame] = [] - per_pair_tables: dict[str, pd.DataFrame] = {} - - for _, row in index_df.iterrows(): - approach = str(row["approach"]) - baseline = str(row["baseline"]) - sheet_id = str(row["sheet_id"]) - scenario = str(row["scenario"]) if pd.notna(row["scenario"]) else "" - year = str(row["year"]) if pd.notna(row["year"]) else "" - # Retag legacy single-year rows (Step 6 era) so the scatter parquet - # carries a consistent scenario column for downstream plotting. - if not scenario: - scenario = ISOLATE_A_MATRIX_SCENARIO - cell_label = ", ".join( - f"{k}={v}" - for k, v in ( - ("scenario", scenario), - ("approach", approach), - ("year", year), - ("baseline", baseline), - ) - if v - ) - logger.info("Pulling tabs for %s", cell_label) - try: - joined = _read_pair(sheet_id) - except ( - Exception - ) as exc: # noqa: BLE001 — surface but don't crash on one bad sheet - logger.warning( - "%s — failed to read Sheet %s (%s); skipping", - cell_label, - sheet_id, - exc, - ) - continue - if joined.empty: - logger.warning("%s returned empty data; skipping", cell_label) - continue - if year: - joined = _deflate_new_to_ref( - joined, - source_year=int(float(year)), - ref_year=REFERENCE_DOLLAR_YEAR, - ) - # Build a deterministic 31-char-bounded tab name including any - # populated scenario/year prefix. - prefix = "_".join(p for p in (scenario, year) if p) - pair_key = ( - f"{prefix}_{approach}__vs_{baseline}" - if prefix - else f"{approach}__vs_{baseline}" - ) - per_pair_tables[pair_key[:31]] = joined.reset_index() - - summary_row = _summarize(joined, approach) - # Stamp the optional dimensions onto the row so the summary tab is - # navigable in time-series mode. - if scenario: - summary_row["scenario"] = scenario - if year: - summary_row["year"] = year - summaries_by_baseline.setdefault(baseline, []).append(summary_row) - scatter_chunks.append( - _scatter_coords(joined, approach, baseline, scenario, year) - ) - - summaries: dict[str, pd.DataFrame] = { - baseline: pd.DataFrame(rows).set_index("approach") - for baseline, rows in summaries_by_baseline.items() - } - - with pd.ExcelWriter(EF_COMPARISON_XLSX_PATH, engine="openpyxl") as writer: - for baseline, summary in summaries.items(): - summary.reset_index().to_excel( - writer, sheet_name=f"summary_vs_{baseline}", index=False - ) - for tab, df in per_pair_tables.items(): - df.to_excel(writer, sheet_name=tab[:31], index=False) - logger.info("Wrote %s", EF_COMPARISON_XLSX_PATH) - - if scatter_chunks: - all_coords = pd.concat(scatter_chunks, ignore_index=True) - all_coords.to_parquet(EF_SCATTER_COORDS_PATH) - logger.info("Wrote %s (%d rows)", EF_SCATTER_COORDS_PATH, len(all_coords)) - - if LAST_RUN_SHEET_ID_PATH.exists(): - run_sheet_id = LAST_RUN_SHEET_ID_PATH.read_text().strip() - for baseline, summary in summaries.items(): - update_sheet_tab( - run_sheet_id, - f"ef_summary_vs_{baseline}", - summary.reset_index(), - clean_nans=True, - ) - logger.info("Appended ef_summary_vs_* tabs to run Sheet %s", run_sheet_id) - else: - logger.warning( - "%s not found — skipping run-report Sheet upload", - LAST_RUN_SHEET_ID_PATH, - ) - - for baseline, summary in summaries.items(): - print(f"\n=== summary_vs_{baseline} ===") - print(summary.round(4).to_string()) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format="%(message)s") - main() diff --git a/bedrock/analysis/a_matrix_time_series/constants.py b/bedrock/analysis/a_matrix_time_series/constants.py deleted file mode 100644 index d9c7c292..00000000 --- a/bedrock/analysis/a_matrix_time_series/constants.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Shared constants for the a_matrix_time_series analysis package. - -Centralizes paths, the BEA detail base year, the latest target year all -approaches have data for, and the Drive folder for run-report Sheets. -""" - -from __future__ import annotations - -from pathlib import Path - -from bedrock.utils.config.usa_config import get_usa_config - -OUTPUT_DIR = Path(__file__).parent / "output" -RESULTS_DIR = OUTPUT_DIR / "results" -PLOTS_DIR = OUTPUT_DIR / "plots" -LAST_RUN_SHEET_ID_PATH = RESULTS_DIR / "last_run_sheet_id.txt" - -ANALYSIS_DRIVE_FOLDER_ID = "1UcPmwLnL6MwTq9pMYJw5d43FJQOFQVO_" - -# BEA detail IO base year — the inflation `original_year` for every script. -ORIGINAL_YEAR: int = get_usa_config().usa_base_io_data_year - -# Latest year for which all approaches have data (USEEIO, industry_pi, -# commodity_pi at 2024; summary_tables falls back to 2023 internally). -LATEST_TARGET_YEAR: int = 2024 - -# Canonical 6-approach plot order (useeio + ceda_default baselines first, -# then the three internal alternatives, then the external reference). -# Top-left → bottom-right when laid out in a grid. -APPROACH_ORDER: tuple[str, ...] = ( - "useeio", - "ceda_default", - "summary_tables", - "industry_price_index", - "commodity_price_index", - "useeio_nowcast", -) - -# Subset of APPROACH_ORDER that drops baselines AND external references — -# the three A-matrix-derivation alternatives we evaluate as recommendation -# candidates. -ALTERNATIVE_APPROACHES: tuple[str, ...] = ( - "summary_tables", - "industry_price_index", - "commodity_price_index", -) - -# Approaches that are external references — included in figures for context -# but never proposed as the production method. Plot helpers should style -# these distinctly (e.g. dashed/dotted line) so reviewers don't mistake them -# for candidates. -EXTERNAL_REFERENCES: tuple[str, ...] = ("useeio_nowcast",) - -# The 3 approaches the v0.3 Cornerstone recommendation focuses on for the -# multi-path comparison plots. ``summary_tables`` and ``commodity_price_index`` -# are the two top internal candidates; ``useeio_nowcast`` is the external -# reference. ``industry_price_index`` is excluded — superseded by -# ``commodity_price_index`` in the recommendation. Use this in plot scripts -# when ``ALTERNATIVE_APPROACHES`` (all 3 internal alts) is too broad. -FOCUS_APPROACHES: tuple[str, ...] = ( - "commodity_price_index", - "summary_tables", - "useeio_nowcast", -) - -# Per-approach colors used by every plot in this package. Keys match -# APPROACH_ORDER. Missing keys (e.g. `ceda` vs `ceda_default`) should -# fall back via `.get(approach, default)` at the call site. -APPROACH_COLORS: dict[str, str] = { - "useeio": "#7f7f7f", - "ceda_default": "#bcbd22", - "summary_tables": "#1f77b4", - "industry_price_index": "#9467bd", - "commodity_price_index": "#2ca02c", - "useeio_nowcast": "#ff7f0e", -} - -# (approach_key, display_label) for the two baselines that the alternatives -# are compared against. Order matches APPROACH_ORDER. -BASELINES: tuple[tuple[str, str], ...] = ( - ("useeio", "USEEIO"), - ("ceda_default", "CEDA-US"), -) - -# Year coverage gaps per approach. ``useeio_nowcast`` upstream pipeline has -# not been run for 2024 — drop that year from any plot/comparison that -# includes useeio_nowcast. Source of truth lives in the extract module. -from bedrock.extract.iot.useeio_nowcast import USEEIO_NOWCAST_YEARS # noqa: E402 - -APPROACH_YEAR_COVERAGE: dict[str, tuple[int, ...]] = { - "useeio_nowcast": USEEIO_NOWCAST_YEARS, -} diff --git a/bedrock/analysis/a_matrix_time_series/derive_A_cells_long.py b/bedrock/analysis/a_matrix_time_series/derive_A_cells_long.py deleted file mode 100644 index 1cfbaef3..00000000 --- a/bedrock/analysis/a_matrix_time_series/derive_A_cells_long.py +++ /dev/null @@ -1,632 +0,0 @@ -"""Step 2 of epic #337: cell-by-cell time-series diagnostics for the A matrix. - -Reads the parquet caches produced by ``derive_A_time_series.py`` (Step 1, #340) -and produces: - -- ``A_cells_long.parquet`` — long-format - ``(row_sector, col_sector, year, approach, dom_or_imp, A_value, - delta_from_2017, delta_yoy, delta_vs_useeio, delta_vs_ceda)``. ~12.8M rows. - Two baseline-divergence columns honor the **two-baseline convention** from - the analysis plan: every comparison is reported against both USEEIO (the - unchanged BEA-2017 base) and CEDA-US (the production-default approach at - the same year). - -- ``scatter_vs_baselines_{dom,imp}.png`` — 2×2 grid (alternative approach × - baseline) of element-wise scatter plots. Per-row target year is the latest - year where the approach and both baselines all have data: - ``commodity_price_index`` rows resolve to 2024; - ``summary_tables`` falls back to 2023 (BEA Excel hasn't published 2024 yet). - All 4 panels share the same ``xlim``/``ylim`` so the ``y=x`` line is a true - 45° reference. Each panel has a top-left summary box with ``n``, mean / p95 - / max ``|Δ|``, and ``R²``. - -- ``divergence_share_{dom,imp}.png`` — 2×2 grid (alternative × baseline) of - the share of cells whose ``|A_approach − A_baseline|`` exceeds each - threshold (1e-4, 1e-3, 1e-2, 1e-1) plotted over years. Answers "how - widespread is the divergence and how does it spread?" — complements the - scatter (which shows magnitude at a single year). - -- ``baseline_reference_{dom,imp}.png`` — 1×2 reference plot comparing the - two baselines directly: USEEIO-vs-CEDA scatter at the latest common year - plus share-of-cells over thresholds across years. Provides a "reference - floor" for interpreting alternative-vs-baseline divergence — readers can - read the alternative-vs-USEEIO and alternative-vs-CEDA panels against - the gap between the baselines themselves. - -- ``divergence_vs_useeio`` and ``divergence_vs_ceda`` tabs appended to the - run-report Sheet. Each row is per (approach, year, dom_or_imp), with - ``max``, ``p99``, ``p95``, ``p75``, ``p50``, ``mean``, and - ``n_above_1pct`` of ``|delta_vs_|``. Sheet ID is read from - ``last_run_sheet_id.txt`` (written by Step 1); if missing, Sheet publish - is skipped with a warning. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_long -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series._loaders import load_a_pair -from bedrock.analysis.a_matrix_time_series._run_report import publish_tabs -from bedrock.analysis.a_matrix_time_series.constants import ( - BASELINES, - FOCUS_APPROACHES, - PLOTS_DIR, - RESULTS_DIR, -) - -logger = logging.getLogger(__name__) - -A_CELLS_LONG_PATH = RESULTS_DIR / "A_cells_long.parquet" - -# v0.3 focus: scatter rows are the 3 focus approaches (summary_tables, -# commodity_price_index, useeio_nowcast). Reversed so the matrix shows -# useeio_nowcast on top, commodity_price_index middle, summary_tables bottom — -# matches "external reference at top, candidates below" reading order. -SCATTER_APPROACHES: tuple[str, ...] = tuple(reversed(FOCUS_APPROACHES)) - - -def _list_pairs() -> list[tuple[str, int]]: - """Discover (approach, year) pairs from the parquet cache. - - Skips files that don't match ``A_{approach}_{4-digit-year}.parquet`` so - other artifacts in the same dir (e.g. ``A_cells_long.parquet``) are - ignored. - - Consumed by: ``build_a_cells_long``. - """ - pairs: list[tuple[str, int]] = [] - for path in sorted(RESULTS_DIR.glob("A_*.parquet")): - stem = path.stem # e.g. A_useeio_2017 - body = stem[2:] # strip leading "A_" - approach, _, year_str = body.rpartition("_") - if not (year_str.isdigit() and len(year_str) == 4): - continue - pairs.append((approach, int(year_str))) - return pairs - - -def _melt(df: pd.DataFrame, approach: str, year: int, kind: str) -> pd.DataFrame: - """Wide A matrix → long (row_sector, col_sector, A_value) + metadata. - - Direct numpy-based melt: faster than ``df.stack().reset_index(...)`` and - avoids the duplicate-column collision when both axes share the source - parquet's ``sector`` name. - - Consumed by: ``build_a_cells_long``. - """ - rows = df.index.to_numpy() - cols = df.columns.to_numpy() - return pd.DataFrame( - { - "row_sector": np.repeat(rows, len(cols)), - "col_sector": np.tile(cols, len(rows)), - "A_value": df.to_numpy().ravel(), - "approach": approach, - "year": year, - "dom_or_imp": kind, - } - ) - - -def build_a_cells_long() -> pd.DataFrame: - """Concat all cells; attach intra-approach drift (``delta_from_2017``, - ``delta_yoy``) and dual-baseline divergence (``delta_vs_useeio``, - ``delta_vs_ceda``) per (approach, dom_or_imp, row_sector, col_sector, - year).""" - chunks: list[pd.DataFrame] = [] - for approach, year in _list_pairs(): - matrices = load_a_pair(approach, year) - for kind, mat in matrices.items(): - chunks.append(_melt(mat, approach, year, kind)) - - long = pd.concat(chunks, ignore_index=True) - - cell_keys = ["approach", "dom_or_imp", "row_sector", "col_sector"] - long = long.sort_values(cell_keys + ["year"]).reset_index(drop=True) - - # Intra-approach drift: this approach's value at year y, minus this - # approach's value at 2017. - base_2017 = ( - long[long["year"] == 2017] - .set_index(cell_keys)["A_value"] - .rename("A_value_2017") - ) - long = long.join(base_2017, on=cell_keys) - long["delta_from_2017"] = long["A_value"] - long["A_value_2017"] - long.drop(columns="A_value_2017", inplace=True) - - long["delta_yoy"] = long.groupby(cell_keys)["A_value"].diff() - - # Cross-approach divergence vs each baseline at the same year. - same_year_keys = ["year", "dom_or_imp", "row_sector", "col_sector"] - for baseline_approach, col_name in ( - ("useeio", "delta_vs_useeio"), - ("ceda_default", "delta_vs_ceda"), - ): - baseline = ( - long[long["approach"] == baseline_approach] - .set_index(same_year_keys)["A_value"] - .rename(f"A_value_{baseline_approach}") - ) - long = long.join(baseline, on=same_year_keys) - long[col_name] = long["A_value"] - long[f"A_value_{baseline_approach}"] - long.drop(columns=f"A_value_{baseline_approach}", inplace=True) - - return long[ - [ - "approach", - "dom_or_imp", - "year", - "row_sector", - "col_sector", - "A_value", - "delta_from_2017", - "delta_yoy", - "delta_vs_useeio", - "delta_vs_ceda", - ] - ] - - -def compute_divergence_quantiles(long: pd.DataFrame, baseline: str) -> pd.DataFrame: - """Per (approach, year, dom_or_imp): quantile stats of ``|delta_vs_|``. - - ``baseline`` is the suffix on the column, i.e. ``"useeio"`` or ``"ceda"``. - """ - col = f"delta_vs_{baseline}" - rows: list[dict[str, object]] = [] - for (approach, year, kind), group in long.groupby( - ["approach", "year", "dom_or_imp"] - ): - abs_delta = group[col].dropna().abs() - if abs_delta.empty: - continue - rows.append( - { - "approach": approach, - "year": int(year), - "dom_or_imp": kind, - "n_cells": int(abs_delta.size), - "max": float(abs_delta.max()), - "p99": float(abs_delta.quantile(0.99)), - "p95": float(abs_delta.quantile(0.95)), - "p75": float(abs_delta.quantile(0.75)), - "p50": float(abs_delta.quantile(0.50)), - "mean": float(abs_delta.mean()), - "n_above_1pct": int((abs_delta > 0.01).sum()), - } - ) - return pd.DataFrame(rows) - - -def _latest_common_year(long: pd.DataFrame, approaches: list[str]) -> int | None: - """Latest year for which every approach in ``approaches`` has data. - - Consumed by: ``plot_scatter_vs_baselines``, ``plot_baseline_reference``. - """ - years_per_approach = [ - set(long.loc[long["approach"] == a, "year"].unique()) for a in approaches - ] - if not years_per_approach: - return None - common = set.intersection(*years_per_approach) - return max(common) if common else None - - -def plot_scatter_vs_baselines(long: pd.DataFrame, kind: str, path: Path) -> None: - """2×2 grid of element-wise scatters; rows = alternative approach, - cols = baseline (USEEIO | CEDA-US). - - Per-row target year: the latest year where the approach AND both - baselines all have data. ``commodity_price_index`` rows resolve to - 2024; ``summary_tables`` falls back to 2023 (BEA Excel hasn't - published 2024). The actual year used - appears in each panel title. - - All 6 panels share the same ``xlim``/``ylim`` (global min/max across - every panel's data) so the ``y=x`` line is a true 45° reference and the - panels are visually comparable. Each panel has a top-left summary box - with ``n``, mean / p95 / max ``|Δ|`` and ``R²``. - """ - sub = long[long["dom_or_imp"] == kind] - - panel_data: list[ - tuple[int, int, str, str, int, "np.ndarray[Any, Any]", "np.ndarray[Any, Any]"] - ] = [] - global_min = np.inf - global_max = -np.inf - - for i, approach in enumerate(SCATTER_APPROACHES): - target_year = _latest_common_year(sub, [approach, *(b[0] for b in BASELINES)]) - if target_year is None: - logger.warning( - "No common year between %s and baselines for kind=%s; " "skipping row.", - approach, - kind, - ) - continue - - year_pivot = sub[sub["year"] == target_year].pivot_table( - index=["row_sector", "col_sector"], - columns="approach", - values="A_value", - ) - for j, (baseline_col, baseline_label) in enumerate(BASELINES): - if ( - approach not in year_pivot.columns - or baseline_col not in year_pivot.columns - ): - continue - x = year_pivot[baseline_col].to_numpy() - y = year_pivot[approach].to_numpy() - # Filter NaN, then drop (0, 0) pairs: A is sparse, ~50% of cells - # are jointly zero and trivially correlate, inflating R² and pulling - # mean |Δ| toward 0. Stats reported here are over cells where at - # least one of (baseline, approach) is non-zero. - mask = ~(np.isnan(x) | np.isnan(y)) & ~((x == 0) & (y == 0)) - x = x[mask] - y = y[mask] - if x.size == 0: - continue - global_min = min(global_min, float(x.min()), float(y.min())) - global_max = max(global_max, float(x.max()), float(y.max())) - panel_data.append((i, j, approach, baseline_label, target_year, x, y)) - - if not panel_data: - logger.warning("No scatter panels could be drawn for kind=%s.", kind) - return - - n_rows = len(SCATTER_APPROACHES) - n_cols = len(BASELINES) - fig, axes = plt.subplots( - n_rows, n_cols, figsize=(4 * n_cols, 4 * n_rows), squeeze=False - ) - fig.suptitle(f"A_approach vs A_baseline — {kind}", fontsize=12) - - for ax_row in axes: - for ax in ax_row: - ax.axis("off") - - for i, j, approach, baseline_label, target_year, x, y in panel_data: - ax = axes[i][j] - ax.axis("on") - ax.scatter(x, y, s=8, alpha=0.35, color="steelblue", edgecolor="none") - ax.plot( - [global_min, global_max], - [global_min, global_max], - "r--", - lw=0.6, - alpha=0.7, - label="y=x", - ) - ax.set_xlim(global_min, global_max) - ax.set_ylim(global_min, global_max) - ax.set_aspect("equal", adjustable="box") - ax.set_xlabel(f"{baseline_label} A_value") - ax.set_ylabel(f"{approach} A_value") - ax.set_title(f"{approach} vs {baseline_label} ({target_year})", fontsize=9) - ax.grid(True, alpha=0.3) - - x_f = np.asarray(x, dtype=float) - y_f = np.asarray(y, dtype=float) - abs_delta = np.abs(y_f - x_f) - r2 = ( - float(np.corrcoef(x_f, y_f)[0, 1] ** 2) - if x_f.std() > 0 and y_f.std() > 0 - else float("nan") - ) - stats_text = ( - f"n = {x.size:,}\n" - f"mean |Δ| = {abs_delta.mean():.4f}\n" - f"p95 |Δ| = {np.quantile(abs_delta, 0.95):.4f}\n" - f"max |Δ| = {abs_delta.max():.4f}\n" - f"R² = {r2:.4f}" - ) - ax.text( - 0.02, - 0.98, - stats_text, - transform=ax.transAxes, - va="top", - ha="left", - fontsize=11, - family="monospace", - bbox={ - "boxstyle": "round,pad=0.3", - "facecolor": "white", - "alpha": 0.85, - "edgecolor": "gray", - }, - ) - - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -DIVERGENCE_THRESHOLDS: tuple[float, ...] = (1e-6, 1e-5, 1e-4, 1e-3, 1e-2) - - -def plot_divergence_share(long: pd.DataFrame, kind: str, path: Path) -> None: - """2×2 grid: share of cells whose ``|delta_vs_baseline|`` exceeds each - threshold, plotted over years. - - Rows are the three alternative approaches; columns are baselines - (USEEIO | CEDA-US). Each panel has one line per threshold in - ``DIVERGENCE_THRESHOLDS`` (1e-4, 1e-3, 1e-2, 1e-1); y-axis is the - fraction of cells whose absolute deviation from the baseline at that - year exceeds the threshold. Answers "how widespread is the divergence, - and how does it spread over time?" — a complement to - ``scatter_vs_baselines`` (which shows magnitude at one snapshot). - """ - sub = long[long["dom_or_imp"] == kind] - n_rows = len(SCATTER_APPROACHES) - n_cols = len(BASELINES) - fig, axes = plt.subplots( - n_rows, n_cols, figsize=(6 * n_cols, 4 * n_rows), squeeze=False - ) - fig.suptitle( - f"Share of cells with |A_approach − A_baseline| above threshold — {kind}", - fontsize=12, - ) - - cmap = plt.get_cmap("viridis") - threshold_colors = [ - cmap(i / max(len(DIVERGENCE_THRESHOLDS) - 1, 1)) - for i in range(len(DIVERGENCE_THRESHOLDS)) - ] - - for i, approach in enumerate(SCATTER_APPROACHES): - for j, (baseline_col, baseline_label) in enumerate(BASELINES): - ax = axes[i][j] - delta_col = ( - "delta_vs_useeio" if baseline_col == "useeio" else "delta_vs_ceda" - ) - approach_sub = sub[sub["approach"] == approach] - pivot = approach_sub.pivot_table( - index=["row_sector", "col_sector"], - columns="year", - values=delta_col, - ).abs() - years_arr = np.array(sorted(pivot.columns), dtype=float) - values_arr = pivot.to_numpy() - keep = ~np.isnan(values_arr).any(axis=1) - kept_values = values_arr[keep] - if kept_values.size == 0: - ax.text( - 0.5, - 0.5, - "no data", - transform=ax.transAxes, - ha="center", - va="center", - ) - continue - - panel_max_share = 0.0 - for thr, color in zip(DIVERGENCE_THRESHOLDS, threshold_colors, strict=True): - share = (kept_values > thr).mean(axis=0) - panel_max_share = max(panel_max_share, float(share.max())) - ax.plot( - years_arr, - share, - color=color, - lw=1.8, - marker="o", - markersize=3, - label=f"|Δ| > {thr:g}", - ) - - ax.set_xlim(years_arr.min(), years_arr.max()) - ax.set_ylim(0, max(panel_max_share * 1.1, 0.01)) - ax.set_xlabel("year") - ax.set_ylabel("share of cells") - ax.set_title( - f"{approach} vs {baseline_label} (n={int(keep.sum())} cells)", - fontsize=10, - ) - ax.grid(True, alpha=0.3) - ax.legend(loc="upper left", fontsize=8, framealpha=0.4) - - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -def plot_baseline_reference(long: pd.DataFrame, kind: str, path: Path) -> None: - """1×2 reference figure comparing the two baselines (USEEIO vs CEDA-US). - - Left: scatter of A_useeio (x) vs A_ceda_default (y) at the latest year - both baselines have data, square aspect, shared limits, with summary - stats (n, mean / p95 / max ``|Δ|``, R²). - - Right: share of cells with ``|A_useeio − A_ceda_default|`` above each - threshold in ``DIVERGENCE_THRESHOLDS``, plotted over years. - - Provides a "reference floor": divergence between alternative approaches - and a baseline can be read against the baseline-vs-baseline divergence - from the same data. - """ - sub = long[long["dom_or_imp"] == kind] - target_year = _latest_common_year(sub, ["useeio", "ceda_default"]) - if target_year is None: - logger.warning( - "No common year for useeio + ceda_default, kind=%s; skipping baseline ref.", - kind, - ) - return - - fig, axes = plt.subplots(1, 2, figsize=(12, 5), squeeze=False) - fig.suptitle(f"Baseline reference: USEEIO vs CEDA-US — {kind}", fontsize=12) - - # --- Left: scatter at latest year -------------------------------------- - ax_scatter = axes[0][0] - year_pivot = sub[sub["year"] == target_year].pivot_table( - index=["row_sector", "col_sector"], - columns="approach", - values="A_value", - ) - if "useeio" in year_pivot.columns and "ceda_default" in year_pivot.columns: - x = year_pivot["useeio"].to_numpy() - y = year_pivot["ceda_default"].to_numpy() - # Drop jointly-zero (sparse-A) cells so R² and mean |Δ| reflect the - # active matrix (matches the filter in plot_scatter_vs_baselines). - mask = ~(np.isnan(x) | np.isnan(y)) & ~((x == 0) & (y == 0)) - x_f = np.asarray(x[mask], dtype=float) - y_f = np.asarray(y[mask], dtype=float) - ax_scatter.scatter( - x_f, y_f, s=8, alpha=0.35, color="darkorange", edgecolor="none" - ) - lo = float(min(x_f.min(), y_f.min())) - hi = float(max(x_f.max(), y_f.max())) - ax_scatter.plot([lo, hi], [lo, hi], "r--", lw=0.6, alpha=0.7, label="y=x") - ax_scatter.set_xlim(lo, hi) - ax_scatter.set_ylim(lo, hi) - ax_scatter.set_aspect("equal", adjustable="box") - abs_delta = np.abs(y_f - x_f) - r2 = ( - float(np.corrcoef(x_f, y_f)[0, 1] ** 2) - if x_f.std() > 0 and y_f.std() > 0 - else float("nan") - ) - stats_text = ( - f"n = {x_f.size:,}\n" - f"mean |Δ| = {abs_delta.mean():.4f}\n" - f"p95 |Δ| = {np.quantile(abs_delta, 0.95):.4f}\n" - f"max |Δ| = {abs_delta.max():.4f}\n" - f"R² = {r2:.4f}" - ) - ax_scatter.text( - 0.02, - 0.98, - stats_text, - transform=ax_scatter.transAxes, - va="top", - ha="left", - fontsize=11, - family="monospace", - bbox={ - "boxstyle": "round,pad=0.3", - "facecolor": "white", - "alpha": 0.85, - "edgecolor": "gray", - }, - ) - ax_scatter.set_xlabel("USEEIO A_value") - ax_scatter.set_ylabel("CEDA-US A_value") - ax_scatter.set_title(f"USEEIO vs CEDA-US ({target_year})", fontsize=10) - ax_scatter.grid(True, alpha=0.3) - - # --- Right: share over thresholds, by year ---------------------------- - ax_share = axes[0][1] - pivot_baseline = sub.pivot_table( - index=["row_sector", "col_sector"], - columns=["approach", "year"], - values="A_value", - ) - if "useeio" not in pivot_baseline.columns.get_level_values( - 0 - ) or "ceda_default" not in pivot_baseline.columns.get_level_values(0): - ax_share.text( - 0.5, - 0.5, - "no baseline data", - transform=ax_share.transAxes, - ha="center", - va="center", - ) - else: - useeio_wide = pivot_baseline["useeio"] - ceda_wide = pivot_baseline["ceda_default"] - common_years = sorted(set(useeio_wide.columns) & set(ceda_wide.columns)) - if not common_years: - ax_share.text( - 0.5, - 0.5, - "no common years", - transform=ax_share.transAxes, - ha="center", - va="center", - ) - else: - useeio_arr = useeio_wide[common_years].to_numpy() - ceda_arr = ceda_wide[common_years].to_numpy() - abs_delta_arr = np.abs(useeio_arr - ceda_arr) - keep = ~np.isnan(abs_delta_arr).any(axis=1) - kept = abs_delta_arr[keep] - years_arr = np.array(common_years, dtype=float) - cmap = plt.get_cmap("viridis") - colors = [ - cmap(i / max(len(DIVERGENCE_THRESHOLDS) - 1, 1)) - for i in range(len(DIVERGENCE_THRESHOLDS)) - ] - panel_max = 0.0 - for thr, color in zip(DIVERGENCE_THRESHOLDS, colors, strict=True): - share = (kept > thr).mean(axis=0) - panel_max = max(panel_max, float(share.max())) - ax_share.plot( - years_arr, - share, - color=color, - lw=1.8, - marker="o", - markersize=3, - label=f"|Δ| > {thr:g}", - ) - ax_share.set_xlim(years_arr.min(), years_arr.max()) - ax_share.set_ylim(0, max(panel_max * 1.1, 0.01)) - ax_share.legend(loc="upper left", fontsize=8, framealpha=0.4) - ax_share.set_xlabel("year") - ax_share.set_ylabel("share of cells") - ax_share.set_title("Share with |USEEIO − CEDA-US| above threshold", fontsize=10) - ax_share.grid(True, alpha=0.3) - - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - logger.info("Building A_cells_long.parquet from %s", RESULTS_DIR) - long = build_a_cells_long() - long.to_parquet(A_CELLS_LONG_PATH) - logger.info("Wrote %s (rows=%d)", A_CELLS_LONG_PATH, len(long)) - - div_useeio_df = compute_divergence_quantiles(long, "useeio") - div_ceda_df = compute_divergence_quantiles(long, "ceda") - div_useeio_df.to_csv(RESULTS_DIR / "divergence_vs_useeio.csv", index=False) - div_ceda_df.to_csv(RESULTS_DIR / "divergence_vs_ceda.csv", index=False) - - for kind in ("dom", "imp"): - plot_scatter_vs_baselines( - long, kind, PLOTS_DIR / f"scatter_vs_baselines_{kind}.png" - ) - plot_divergence_share(long, kind, PLOTS_DIR / f"divergence_share_{kind}.png") - plot_baseline_reference( - long, kind, PLOTS_DIR / f"baseline_reference_{kind}.png" - ) - - publish_tabs( - { - "divergence_vs_useeio": div_useeio_df, - "divergence_vs_ceda": div_ceda_df, - } - ) - logger.info("Step 2 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py b/bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py deleted file mode 100644 index 8b328877..00000000 --- a/bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py +++ /dev/null @@ -1,617 +0,0 @@ -"""Step 2.5 of epic #337: cell-level set stability and persistence diagnostics. - -Reads ``A_cells_long.parquet`` from Step 2 and unpacks what the share-of-cells -line chart in Step 2 aggregates away. Two questions: - -- **Set stability**: are the cells above a divergence threshold the *same* - cells year-over-year (structural offset), or is membership rotating - (transient drift averaging to a constant share)? -- **Persistence**: of the cells that *ever* disagree, how is the disagreement - distributed across "always" vs "occasional" — i.e. what fraction of the - ever-above-threshold population is consistently above every year? - -Outputs: - -- ``set_stability_jaccard_thr{thr}_{kind}.png`` — 3×2 grid of year×year - Jaccard heatmaps (approach × baseline). Upper-triangle only with - ``vmin=0.5`` so the meaningful range fills the colorbar; each panel - carries an off-diagonal-mean tag. High off-diagonal mean ⇒ structural - offset; low ⇒ rotating membership. -- ``persistence_by_threshold_{kind}.png`` — multi-threshold conditional - persistence composite. One panel per threshold in ``PLOT_THRESHOLDS`` - (strictest → loosest) plus a shared legend. Bars are baseline-grouped - (left 3 = vs USEEIO, gap, right 3 = vs CEDA-US) and stacked by - years-above-threshold buckets. Denominator is **cells ever above - threshold** so the always-above fraction is directly legible. - Structural offsets keep their always-share roughly constant as the - threshold tightens; rotating membership doesn't. -- ``set_stability_jaccard.csv``, ``persistence_categories.csv`` — long-format - CSVs covering ``ALL_THRESHOLDS``. ``persistence_categories.csv`` carries - both the absolute ``share`` (denominator = all cells) and - ``conditional_share`` (denominator = ever-above-threshold cells). -- ``set_stability_jaccard``, ``persistence_categories`` tabs appended to - the run-report Sheet (sheet ID from ``last_run_sheet_id.txt``). - -Usage: - python -m bedrock.analysis.a_matrix_time_series.derive_A_cells_stability -""" - -from __future__ import annotations - -import logging -from pathlib import Path -from typing import Any - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from matplotlib.axes import Axes -from matplotlib.ticker import PercentFormatter - -from bedrock.analysis.a_matrix_time_series._run_report import publish_tabs -from bedrock.analysis.a_matrix_time_series.constants import ( - FOCUS_APPROACHES, - PLOTS_DIR, - RESULTS_DIR, -) - -logger = logging.getLogger(__name__) - -A_CELLS_LONG_PATH = RESULTS_DIR / "A_cells_long.parquet" - -# Approaches we plot (rows). Reversed from FOCUS_APPROACHES so rows are -# stacked with `useeio_nowcast` at top (external reference reads first), -# then `commodity_price_index`, then `summary_tables` at bottom. -APPROACHES_FOR_STABILITY: tuple[str, ...] = tuple(reversed(FOCUS_APPROACHES)) -# (approach_name_in_long, display_label, delta_column_in_long). Carries the -# delta-column suffix so this file is the only consumer; the shared -# `BASELINES` 2-tuple in `constants.py` covers the simpler (name, label) form. -BASELINES_WITH_DELTA_COL: tuple[tuple[str, str, str], ...] = ( - ("useeio", "USEEIO", "delta_vs_useeio"), - ("ceda_default", "CEDA-US", "delta_vs_ceda"), -) - -# CSV/Sheet thresholds — superset of Step 2 (extended up to 1e-1 so the -# economically-meaningful upper range is available downstream). -ALL_THRESHOLDS: tuple[float, ...] = (1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1) -# Subset rendered as plots — heatmaps (one PNG per threshold) and the -# persistence-by-threshold composite (one panel per threshold). Strictest -# first; tighter thresholds (1e-4 and below) are dominated by rounding-noise -# cells and obscure the signal. -PLOT_THRESHOLDS: tuple[float, ...] = (1e-1, 1e-2, 1e-3) - -# Color stops in the heatmap — meaningful range is ~0.5..1.0; clamp lower so -# the gradient resolves the high end (where most values live) instead of being -# wasted on the empty 0..0.5 region. -HEATMAP_VMIN: float = 0.5 -HEATMAP_VMAX: float = 1.0 - - -# --------------------------------------------------------------------------- -# Core computations -# --------------------------------------------------------------------------- - - -def _wide_above_threshold( - long: pd.DataFrame, - approach: str, - kind: str, - delta_col: str, - threshold: float, -) -> pd.DataFrame: - """``(cell × year)`` boolean DataFrame: True where ``|delta_col| > threshold``. - - Drops cells with NaN in any year so the per-year sets are over an aligned - cell population — same filter as ``plot_divergence_share`` in Step 2. - - Consumed by: ``compute_set_stability``, ``compute_persistence_categories``. - """ - sub = long[(long["approach"] == approach) & (long["dom_or_imp"] == kind)] - pivot = pd.DataFrame( - sub.pivot_table( - index=["row_sector", "col_sector"], - columns="year", - values=delta_col, - ).abs() - ) - nan_per_row = np.asarray(pivot.isna().any(axis=1)) - return pd.DataFrame(pivot.loc[~nan_per_row] > threshold) - - -def compute_set_stability( - long: pd.DataFrame, kind: str, threshold: float -) -> pd.DataFrame: - """Year×year Jaccard of "above-threshold" cell sets, per (approach, baseline). - - Long-format rows keyed by (approach, baseline, dom_or_imp, threshold, - year_a, year_b) with set sizes and the Jaccard ratio. - """ - rows: list[dict[str, object]] = [] - for approach in APPROACHES_FOR_STABILITY: - for baseline_col, _, delta_col in BASELINES_WITH_DELTA_COL: - wide = _wide_above_threshold(long, approach, kind, delta_col, threshold) - if wide.empty: - continue - years = sorted(wide.columns) - arr = wide[years].to_numpy() # (n_cells, n_years) - for ai, year_a in enumerate(years): - set_a = arr[:, ai] - n_a = int(set_a.sum()) - for bi, year_b in enumerate(years): - set_b = arr[:, bi] - n_b = int(set_b.sum()) - intersection = int((set_a & set_b).sum()) - union = int((set_a | set_b).sum()) - jac = intersection / union if union > 0 else float("nan") - rows.append( - { - "approach": approach, - "baseline": baseline_col, - "dom_or_imp": kind, - "threshold": threshold, - "year_a": int(year_a), - "year_b": int(year_b), - "n_a": n_a, - "n_b": n_b, - "n_intersection": intersection, - "n_union": union, - "jaccard": jac, - } - ) - return pd.DataFrame(rows) - - -def compute_persistence_categories( - long: pd.DataFrame, kind: str, threshold: float -) -> pd.DataFrame: - """Histogram of per-cell "years above threshold" + conditional share. - - Returns long-format rows per (approach, baseline, dom_or_imp, threshold, - n_years_above) with: - - ``n_cells``: number of cells with exactly ``n_years_above`` years above. - - ``share``: ``n_cells / total cells`` (denominator = aligned cell pop). - - ``conditional_share``: ``n_cells / cells_ever_above`` (denom excludes - the ``n_years_above == 0`` bucket). NaN for the ``0`` bucket and when - no cell is ever above threshold. - - ``n_ever_above``: count of cells with ``n_years_above >= 1``. - """ - rows: list[dict[str, object]] = [] - for approach in APPROACHES_FOR_STABILITY: - for baseline_col, _, delta_col in BASELINES_WITH_DELTA_COL: - wide = _wide_above_threshold(long, approach, kind, delta_col, threshold) - if wide.empty: - continue - persistence = wide.sum(axis=1).to_numpy().astype(int) - total = persistence.size - n_years = wide.shape[1] - counts = np.bincount(persistence, minlength=n_years + 1) - n_ever_above = int(total - counts[0]) - for n_years_above, n_cells in enumerate(counts): - if n_years_above == 0: - cond_share = float("nan") - elif n_ever_above > 0: - cond_share = float(n_cells) / n_ever_above - else: - cond_share = float("nan") - rows.append( - { - "approach": approach, - "baseline": baseline_col, - "dom_or_imp": kind, - "threshold": threshold, - "n_years_above": int(n_years_above), - "n_cells": int(n_cells), - "share": ( - float(n_cells) / total if total > 0 else float("nan") - ), - "conditional_share": cond_share, - "n_ever_above": n_ever_above, - } - ) - return pd.DataFrame(rows) - - -# --------------------------------------------------------------------------- -# Panel-level plotters (each draws into a provided Axes) -# --------------------------------------------------------------------------- - - -def _bucket_label(n_years_above: int, max_bucket: int) -> str: - if n_years_above == 0: - return "0 (never)" - if n_years_above == max_bucket: - return f"{n_years_above} (always)" - if n_years_above == 1: - return "1 year" - return f"{n_years_above} years" - - -def _draw_jaccard_panel( - panel_df: pd.DataFrame, - ax: Axes, - *, - title: str, - show_yticklabels: bool = True, -) -> None: - """Improved Jaccard heatmap panel: upper triangle only, ``vmin=0.5``, - annotate off-diagonal cells only, off-diagonal mean tag below the panel. - - Lower triangle is masked because the matrix is symmetric (J(a,b)=J(b,a)). - Diagonal is suppressed because it is trivially 1 and adds no information. - """ - if len(panel_df) == 0: - ax.text(0.5, 0.5, "no data", transform=ax.transAxes, ha="center", va="center") - ax.set_xticks([]) - ax.set_yticks([]) - ax.set_title(title, fontsize=10) - return - - heat = ( - panel_df.pivot(index="year_a", columns="year_b", values="jaccard") - .sort_index() - .sort_index(axis=1) - ) - years = list(heat.columns) - heat_arr = heat.to_numpy(dtype=float) - n = len(years) - - display = heat_arr.copy() - lower_mask = np.tri(n, k=-1, dtype=bool) - display[lower_mask] = np.nan - - im = ax.imshow( - display, - vmin=HEATMAP_VMIN, - vmax=HEATMAP_VMAX, - cmap="viridis", - origin="lower", - aspect="equal", - ) - ax.set_xticks(range(n)) - ax.set_yticks(range(n)) - ax.set_xticklabels([str(y) for y in years], fontsize=8, rotation=45) - if show_yticklabels: - ax.set_yticklabels([str(y) for y in years], fontsize=8) - else: - ax.set_yticklabels([]) - - for ai in range(n): - for bi in range(ai + 1, n): - val = float(heat_arr[ai, bi]) - if np.isnan(val): - continue - color = "white" if val < (HEATMAP_VMIN + HEATMAP_VMAX) / 2 else "black" - ax.text( - bi, - ai, - f"{val:.2f}", - ha="center", - va="center", - fontsize=7, - color=color, - ) - - off_diag_mask = ~np.eye(n, dtype=bool) - off_vals = heat_arr[off_diag_mask] - off_vals = off_vals[~np.isnan(off_vals)] - mean = float(off_vals.mean()) if len(off_vals) > 0 else float("nan") - ax.set_title(title, fontsize=10) - ax.set_xlabel(f"off-diag mean = {mean:.2f}", fontsize=9) - - plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04) - - -def _draw_baseline_grouped_persistence( - panel_sub: pd.DataFrame, - ax: Axes, - *, - threshold: float, - show_legend: bool = True, - show_title: bool = True, -) -> tuple[list[Any], list[Any]]: - """Stacked persistence bars grouped by baseline. - - Bar order is ``[USEEIO group, spacer, CEDA-US group]``. Within each - group the three approaches appear in ``APPROACHES_FOR_STABILITY`` order. - A faint dashed vertical separator marks the spacer. Bars stack - years-above-threshold buckets ``1..N_years``; legend is reordered so - the "always" bucket sits at the top, matching the visual stack. - - Returns ``(handles, labels)`` for the legend so callers (e.g. the - dashboard) can build a shared figure-level legend if ``show_legend=False``. - """ - if len(panel_sub) == 0: - ax.text(0.5, 0.5, "no data", transform=ax.transAxes, ha="center", va="center") - if show_title: - ax.set_title(f"|Δ| > {threshold:g}", fontsize=10) - return [], [] - - # Bar order: group by baseline (outer), approach (inner). - bars: list[tuple[str, str, str]] = [] - for baseline_col, baseline_label, _ in BASELINES_WITH_DELTA_COL: - for approach in APPROACHES_FOR_STABILITY: - bars.append((approach, baseline_col, baseline_label)) - bar_labels = [f"{a}\nvs {bl}" for a, _, bl in bars] - - n_per_group = len(APPROACHES_FOR_STABILITY) - # Position 3 (between groups) is empty — creates the visual spacer. - x_positions = np.concatenate( - [ - np.arange(n_per_group, dtype=float), - np.arange(n_per_group, dtype=float) + n_per_group + 1.0, - ] - ) - - max_bucket = int(panel_sub["n_years_above"].max()) - cmap = plt.get_cmap("viridis") - bucket_colors = [ - cmap((k - 1) / max(max_bucket - 1, 1)) for k in range(1, max_bucket + 1) - ] - - n_ever_per_bar: list[int] = [] - for approach, baseline_col, _ in bars: - row = panel_sub.loc[ - (panel_sub["approach"] == approach) - & (panel_sub["baseline"] == baseline_col) - ] - n_ever_per_bar.append(int(row["n_ever_above"].iloc[0]) if len(row) > 0 else 0) - - # Min segment height (as fraction of full bar) to qualify for an in-bar - # n_cells label. Below this, a label would visually collide with neighbors. - min_label_height = 0.025 - - bottom = np.zeros(len(bars)) - for k_idx, n_years_above in enumerate(range(1, max_bucket + 1)): - heights: list[float] = [] - n_cells_per_bar: list[int] = [] - for approach, baseline_col, _ in bars: - row = panel_sub.loc[ - (panel_sub["approach"] == approach) - & (panel_sub["baseline"] == baseline_col) - & (panel_sub["n_years_above"] == n_years_above) - ] - if len(row) > 0: - cond = float(row["conditional_share"].iloc[0]) - heights.append(0.0 if np.isnan(cond) else cond) - n_cells_per_bar.append(int(row["n_cells"].iloc[0])) - else: - heights.append(0.0) - n_cells_per_bar.append(0) - heights_arr = np.asarray(heights, dtype=float) - ax.bar( - x_positions, - heights_arr, - bottom=bottom, - color=bucket_colors[k_idx], - edgecolor="white", - linewidth=0.4, - label=_bucket_label(n_years_above, max_bucket), - ) - # Annotate each segment with its n_cells. White text on dark - # (low-luminance) segments, black on light. Skip segments too thin to - # host a label. - seg_color = bucket_colors[k_idx] - r, g, b = seg_color[0], seg_color[1], seg_color[2] - luminance = 0.299 * r + 0.587 * g + 0.114 * b - text_color = "white" if luminance < 0.55 else "black" - for ti, (h, n_cells) in enumerate( - zip(heights_arr, n_cells_per_bar, strict=True) - ): - if h < min_label_height or n_cells == 0: - continue - center_y = bottom[ti] + h / 2.0 - ax.text( - x_positions[ti], - center_y, - f"{n_cells}", - ha="center", - va="center", - fontsize=8, - color=text_color, - ) - bottom = bottom + heights_arr - - ax.set_xticks(x_positions) - ax.set_xticklabels(bar_labels, fontsize=8, rotation=15, ha="right") - ax.set_ylim(0, 1) - ax.yaxis.set_major_formatter(PercentFormatter(xmax=1.0, decimals=0)) - ax.set_ylabel("conditional share") - if show_title: - ax.set_title(f"|Δ| > {threshold:g}", fontsize=10) - ax.grid(True, axis="y", alpha=0.3) - - # Dashed separator line between the two baseline groups. - ax.axvline(x=n_per_group, color="lightgray", lw=0.8, linestyle="--", zorder=0) - - # n_ever_above annotation above each bar — disambiguates "0% always - # because few cells disagree at all" vs "many cells, none consistent". - for ti, n_ever in enumerate(n_ever_per_bar): - ax.text( - x_positions[ti], - 1.02, - f"n={n_ever}", - ha="center", - va="bottom", - fontsize=11, - color="dimgray", - transform=ax.get_xaxis_transform(), - ) - - handles, labels = ax.get_legend_handles_labels() - # Reverse so the "always" (yellow) handle sits at the top of the legend, - # matching the top of the stack. - handles = handles[::-1] - labels = labels[::-1] - if show_legend: - ax.legend( - handles, - labels, - title="years above threshold", - loc="center left", - bbox_to_anchor=(1.02, 0.5), - fontsize=8, - framealpha=0.4, - ) - return handles, labels - - -# --------------------------------------------------------------------------- -# Top-level plot functions (write a PNG) -# --------------------------------------------------------------------------- - - -def plot_set_stability_heatmap( - stability_df: pd.DataFrame, kind: str, threshold: float, path: Path -) -> None: - """3×2 grid of improved year×year Jaccard heatmaps (approach × baseline).""" - sub: pd.DataFrame = stability_df.loc[ - (stability_df["dom_or_imp"] == kind) & (stability_df["threshold"] == threshold) - ] - n_rows = len(APPROACHES_FOR_STABILITY) - n_cols = len(BASELINES_WITH_DELTA_COL) - fig, axes = plt.subplots( - n_rows, n_cols, figsize=(5.0 * n_cols, 4.5 * n_rows), squeeze=False - ) - fig.suptitle( - f"Year×year Jaccard of cells with |A_approach − A_baseline| > {threshold:g} — {kind}", - fontsize=12, - ) - for i, approach in enumerate(APPROACHES_FOR_STABILITY): - for j, (baseline_col, baseline_label, _) in enumerate(BASELINES_WITH_DELTA_COL): - panel: pd.DataFrame = sub.loc[ - (sub["approach"] == approach) & (sub["baseline"] == baseline_col) - ] - _draw_jaccard_panel( - panel, - axes[i][j], - title=f"{approach} vs {baseline_label}", - ) - fig.tight_layout() - fig.savefig(path, dpi=150) - plt.close(fig) - - -def plot_persistence_by_threshold( - persistence_df: pd.DataFrame, kind: str, path: Path -) -> None: - """Multi-threshold conditional-persistence composite for one ``kind``. - - One panel per threshold in ``PLOT_THRESHOLDS`` (strictest → loosest) - plus a shared legend in the unused grid slot. Each panel uses the - baseline-grouped bar layout (left 3 = vs USEEIO, gap, right 3 = vs - CEDA-US). Reading across panels shows whether the always-share - survives as the threshold tightens — a structural offset stays high; - rotating membership doesn't. - """ - n_thr = len(PLOT_THRESHOLDS) - n_cols = 2 - n_rows = (n_thr + 1 + n_cols - 1) // n_cols # +1 reserves a slot for the legend - fig, axes = plt.subplots( - n_rows, - n_cols, - figsize=(8.5 * n_cols, 5.5 * n_rows), - squeeze=False, - ) - fig.suptitle( - f"Conditional cell persistence by threshold — {kind}\n" - "(denominator = cells ever above threshold)", - fontsize=13, - ) - - legend_handles: list[Any] = [] - legend_labels: list[Any] = [] - for ti, threshold in enumerate(PLOT_THRESHOLDS): - r, c = divmod(ti, n_cols) - ax = axes[r][c] - panel = persistence_df.loc[ - (persistence_df["dom_or_imp"] == kind) - & (persistence_df["threshold"] == threshold) - & (persistence_df["n_years_above"] > 0) - ] - handles, labels = _draw_baseline_grouped_persistence( - panel, - ax, - threshold=threshold, - show_legend=False, - ) - if handles and not legend_handles: - legend_handles, legend_labels = handles, labels - - # Hide unused panels and host the shared legend in the first unused slot. - legend_placed = False - for ti in range(n_thr, n_rows * n_cols): - r, c = divmod(ti, n_cols) - ax = axes[r][c] - ax.axis("off") - if not legend_placed and legend_handles: - ax.legend( - legend_handles, - legend_labels, - title="years above threshold", - loc="center", - fontsize=11, - framealpha=0.4, - ) - legend_placed = True - - fig.tight_layout(rect=(0, 0, 1, 0.97)) - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -# --------------------------------------------------------------------------- -# Sheet publish + main -# --------------------------------------------------------------------------- - - -def main() -> None: - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - if not A_CELLS_LONG_PATH.exists(): - raise FileNotFoundError( - f"{A_CELLS_LONG_PATH} not found. Run derive_A_cells_long first (Step 2)." - ) - - long = pd.read_parquet(A_CELLS_LONG_PATH) - logger.info("Loaded %s (rows=%d)", A_CELLS_LONG_PATH, len(long)) - - stability_chunks: list[pd.DataFrame] = [] - persistence_chunks: list[pd.DataFrame] = [] - for kind in ("dom", "imp"): - for threshold in ALL_THRESHOLDS: - stab = compute_set_stability(long, kind, threshold) - persist = compute_persistence_categories(long, kind, threshold) - stability_chunks.append(stab) - persistence_chunks.append(persist) - if threshold in PLOT_THRESHOLDS: - plot_set_stability_heatmap( - stab, - kind, - threshold, - PLOTS_DIR / f"set_stability_jaccard_thr{threshold:g}_{kind}.png", - ) - - stability_df = pd.concat(stability_chunks, ignore_index=True) - persistence_df = pd.concat(persistence_chunks, ignore_index=True) - stability_df.to_csv(RESULTS_DIR / "set_stability_jaccard.csv", index=False) - persistence_df.to_csv(RESULTS_DIR / "persistence_categories.csv", index=False) - - for kind in ("dom", "imp"): - plot_persistence_by_threshold( - persistence_df, - kind, - PLOTS_DIR / f"persistence_by_threshold_{kind}.png", - ) - - publish_tabs( - { - "set_stability_jaccard": stability_df, - "persistence_categories": persistence_df, - } - ) - logger.info("Step 2.5 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/derive_A_time_series.py b/bedrock/analysis/a_matrix_time_series/derive_A_time_series.py deleted file mode 100644 index 7e36bb9f..00000000 --- a/bedrock/analysis/a_matrix_time_series/derive_A_time_series.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Derive A matrices for every (approach × year) combination. - -Step 1 of epic #337. Produces parquet caches as the raw substrate for -Steps 2–6, and creates a per-run Google Sheet in the analysis Drive folder -(`1UcPmwLnL6MwTq9pMYJw5d43FJQOFQVO_`) with two summary tabs: - -- ``cache_summary`` — per (approach, year, matrix_kind) shape + integrity stats -- ``sanity_2017_identity_check`` — confirms ``useeio`` and - ``commodity_price_index`` collapse to the BEA-2017 base A at - ``model_base_year = 2017``. ``summary_tables`` reads BEA's *summary* - aggregation rather than the *detail* base, so it isn't expected to be - identity at any year (recorded but not pass-fail tested). ``ceda_default`` - does a non-inverse round trip via the legacy CEDA io_year, same treatment. - -The new sheet ID is logged and persisted to ``last_run_sheet_id.txt`` so the -downstream steps (#341–#344) can append their own summary tabs to the same -report. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.derive_A_time_series -""" - -from __future__ import annotations - -import datetime as dt -import logging -import subprocess -from pathlib import Path - -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import ( - ANALYSIS_DRIVE_FOLDER_ID, - APPROACH_YEAR_COVERAGE, - LAST_RUN_SHEET_ID_PATH, - LATEST_TARGET_YEAR, - ORIGINAL_YEAR, - PLOTS_DIR, - RESULTS_DIR, -) -from bedrock.utils.config.config_controllers import clear_caches, force_set_usa_config -from bedrock.utils.config.usa_config import reset_usa_config -from bedrock.utils.io.gcp import create_spreadsheet_in_folder, update_sheet_tab - -logger = logging.getLogger(__name__) - -APPROACH_YAMLS: dict[str, str] = { - "useeio": "2025_usa_cornerstone_A_useeio.yaml", - "summary_tables": "2025_usa_cornerstone_A_summary_tables.yaml", - "commodity_price_index": "2025_usa_cornerstone_A_commodity_price_index.yaml", - "ceda_default": "2025_usa_cornerstone_taxonomy.yaml", # CEDA baseline with Cornerstone schema - "useeio_nowcast": "2025_usa_cornerstone_A_useeio_nowcast.yaml", # external reference -} -APPROACHES: list[str] = list(APPROACH_YAMLS.keys()) - - -def _years_for(approach: str, all_years: list[int]) -> list[int]: - """Filter ``all_years`` to the set this approach has data for. - - ``useeio_nowcast`` has no 2024 upstream — skip silently rather than - fail or extrapolate. Other approaches default to the full range. - """ - allowed = APPROACH_YEAR_COVERAGE.get(approach) - if allowed is None: - return all_years - return [y for y in all_years if y in allowed] - - -# Includes ORIGINAL_YEAR (the BEA detail base year) for the 2017-identity -# sanity check tab. -TARGET_YEARS: list[int] = list(range(ORIGINAL_YEAR, LATEST_TARGET_YEAR + 1)) - -# Modules whose @functools.cache outputs are config-dependent and must be -# invalidated between (approach, year) iterations. -_CACHE_BEARING_MODULE_PATHS = ( - "bedrock.transform.eeio.derived_cornerstone", - "bedrock.transform.eeio.cornerstone_bea_intermediates", - "bedrock.transform.eeio.derived_useeio_nowcast", - "bedrock.utils.economic.inflation_helpers_cornerstone", -) - - -def _derive_one_pair( - approach: str, year: int -) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series]: - """Run the cornerstone A pipeline for one (approach, year) pair. - - Returns ``(Adom, Aimp, q)``. Caches are cleared and the config is - reinstalled before the call to guarantee a fresh derivation. - """ - reset_usa_config() - clear_caches(*_CACHE_BEARING_MODULE_PATHS) - # model_base_year bypasses Literal validation (runs 2017–2024 but schema - # only allows 2022–2024). update_inflation_factors and apply_inflation_to_V - # are set explicitly because __init__.py flips them on the initial config - # load; replacing _usa_config directly would otherwise drop them to defaults. - force_set_usa_config( - APPROACH_YAMLS[approach], - model_base_year=year, - update_inflation_factors=True, - apply_inflation_to_V=True, - ) - - from bedrock.transform.eeio.derived_cornerstone import ( # noqa: PLC0415 - derive_cornerstone_Aq_scaled, - ) - - aq = derive_cornerstone_Aq_scaled() - return aq.Adom, aq.Aimp, aq.scaled_q - - -def _save_parquets( - approach: str, year: int, adom: pd.DataFrame, aimp: pd.DataFrame, q: pd.Series -) -> tuple[Path, Path]: - """Write the (Adom, Aimp, q) triple as two parquets under ``output/results/``. - - Adom + Aimp are stacked into a single multi-index DataFrame (index level - ``dom_or_imp`` ∈ {dom, imp}); q is written separately because its single - column makes a stacked layout awkward. - """ - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - a_path = RESULTS_DIR / f'A_{approach}_{year}.parquet' - q_path = RESULTS_DIR / f'q_{approach}_{year}.parquet' - - combined = pd.concat({'dom': adom, 'imp': aimp}, axis=0, names=['dom_or_imp']) - combined.to_parquet(a_path) - q.to_frame('q').to_parquet(q_path) - - return a_path, q_path - - -def _matrix_stats( - approach: str, year: int, kind: str, df: pd.DataFrame, file_path: Path -) -> dict[str, object]: - """Per-matrix summary row for the cache_summary tab.""" - arr = df.to_numpy() - return { - 'approach': approach, - 'year': year, - 'matrix_kind': kind, - 'n_rows': int(df.shape[0]), - 'n_cols': int(df.shape[1]), - 'nan_count': int(np.isnan(arr).sum()), - 'neg_count': int((arr < 0).sum()), - 'max_col_sum': float(np.nansum(arr, axis=0).max()) if arr.size else 0.0, - 'file_size_bytes': int(file_path.stat().st_size), - } - - -_EXPECTED_IDENTITY_AT_2017 = { - 'useeio', - 'commodity_price_index', -} -"""Approaches whose 2017 output should equal the BEA-2017 base A within rtol. - -``summary_tables`` reads BEA *summary*-level tables rather than the *detail* -base; the two aggregations differ even at the same year, so it isn't expected -to be identity. ``ceda_default`` does ``scale(2017→io_year)`` then -``inflate(io_year→2017)`` — not an inverse pair, so its 2017 output is also -expected to deviate.""" - - -def _build_sanity_2017( - matrices_2017: dict[str, tuple[pd.DataFrame, pd.DataFrame, pd.Series]], - rtol: float = 1e-10, -) -> pd.DataFrame: - """At ``model_base_year = 2017``, the four alternative approaches must - collapse to the BEA-2017 base A (identity transformations); ``ceda_default`` - does a non-inverse round trip and is recorded but not pass-fail tested. - - Reference is ``useeio`` (returns ``base`` directly — simplest semantics). - """ - if not matrices_2017 or 'useeio' not in matrices_2017: - return pd.DataFrame() - - ref_dom, ref_imp, ref_q = matrices_2017['useeio'] - - def _max_rel_dev(a: pd.DataFrame, b: pd.DataFrame) -> float: - diff = a.to_numpy() - b.to_numpy() - denom = np.where(np.abs(b.to_numpy()) > 0, np.abs(b.to_numpy()), 1.0) - return float(np.abs(diff / denom).max()) - - rows = [] - for name, (dom, imp, q) in sorted(matrices_2017.items()): - max_dom = _max_rel_dev(dom, ref_dom) - max_imp = _max_rel_dev(imp, ref_imp) - max_q = _max_rel_dev(q.to_frame(), ref_q.to_frame()) - expected_identity = name in _EXPECTED_IDENTITY_AT_2017 - max_rel_dev = max(max_dom, max_imp, max_q) - rows.append( - { - 'approach': name, - 'reference': 'useeio', - 'expected_identity': expected_identity, - 'max_rel_dev_Adom': max_dom, - 'max_rel_dev_Aimp': max_imp, - 'max_rel_dev_q': max_q, - 'passes': (bool(max_rel_dev <= rtol) if expected_identity else None), - } - ) - return pd.DataFrame(rows) - - -def _git_sha_short() -> str: - try: - sha = subprocess.check_output( - ['git', 'rev-parse', '--short=7', 'HEAD'], text=True - ).strip() - return sha or 'unknown' - except (subprocess.CalledProcessError, FileNotFoundError): - return 'unknown' - - -def _publish_run_report(summary_df: pd.DataFrame, sanity_df: pd.DataFrame) -> str: - """Create the run-report Sheet and write the two summary tabs. - - Returns the new sheet ID. - """ - title = ( - f'a_matrix_time_series__{_git_sha_short()}__' - f'{dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ")}' - ) - sheet_id = create_spreadsheet_in_folder( - title=title, folder_id=ANALYSIS_DRIVE_FOLDER_ID - ) - update_sheet_tab(sheet_id, 'cache_summary', summary_df) - if not sanity_df.empty: - update_sheet_tab(sheet_id, 'sanity_2017_identity_check', sanity_df) - return sheet_id - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - summary_rows: list[dict[str, object]] = [] - matrices_2017: dict[str, tuple[pd.DataFrame, pd.DataFrame, pd.Series]] = {} - - for approach in APPROACHES: - for year in _years_for(approach, TARGET_YEARS): - logger.info("Deriving A matrix: approach=%s year=%d", approach, year) - try: - adom, aimp, q = _derive_one_pair(approach, year) - except Exception as e: # noqa: BLE001 — record any per-pair failure - logger.warning('Pair (%s, %d) failed: %s', approach, year, e) - summary_rows.append( - { - 'approach': approach, - 'year': year, - 'matrix_kind': 'FAILED', - 'error': f'{type(e).__name__}: {e}', - } - ) - continue - - a_path, q_path = _save_parquets(approach, year, adom, aimp, q) - summary_rows.append(_matrix_stats(approach, year, 'Adom', adom, a_path)) - summary_rows.append(_matrix_stats(approach, year, 'Aimp', aimp, a_path)) - summary_rows.append( - _matrix_stats(approach, year, 'q', q.to_frame('q'), q_path) - ) - - if year == 2017: - matrices_2017[approach] = (adom, aimp, q) - - summary_df = pd.DataFrame(summary_rows) - sanity_df = _build_sanity_2017(matrices_2017) - summary_df.to_csv(RESULTS_DIR / 'cache_summary.csv', index=False) - if not sanity_df.empty: - sanity_df.to_csv(RESULTS_DIR / 'sanity_2017_identity_check.csv', index=False) - - try: - sheet_id = _publish_run_report(summary_df, sanity_df) - except Exception as e: # noqa: BLE001 - logger.warning( - 'Sheet publish skipped (%s: %s). Local parquet caches and CSVs in ' - "%s are complete; Steps 2-5 that read last_run_sheet_id.txt won't " - 'have a target until the run is re-published with valid Drive auth.', - type(e).__name__, - e, - RESULTS_DIR, - ) - return - - LAST_RUN_SHEET_ID_PATH.write_text(sheet_id + '\n') - sheet_url = f'https://docs.google.com/spreadsheets/d/{sheet_id}' - logger.info('Run report: %s', sheet_url) - print(f'Run report Sheet: {sheet_url}') - print(f'Sheet ID written to: {LAST_RUN_SHEET_ID_PATH}') - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/derive_useeio_nowcast_A.py b/bedrock/analysis/a_matrix_time_series/derive_useeio_nowcast_A.py deleted file mode 100644 index 37488022..00000000 --- a/bedrock/analysis/a_matrix_time_series/derive_useeio_nowcast_A.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Cache useeio_nowcast A parquets for the time-series analysis (Step N1). - -Loops over ``USEEIO_NOWCAST_YEARS`` (2017–2023), calls the transform-layer -function for each year, and writes parquet caches in the same layout that -``_loaders.load_a_pair`` expects. - -Loaders live in `bedrock.extract.iot.useeio_nowcast`; derivation lives in -`bedrock.transform.eeio.derived_useeio_nowcast`. This file is the -analysis-side driver only. - -CLI: - python -m bedrock.analysis.a_matrix_time_series.derive_useeio_nowcast_A -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import RESULTS_DIR -from bedrock.extract.iot.useeio_nowcast import USEEIO_NOWCAST_YEARS -from bedrock.transform.eeio.derived_useeio_nowcast import ( - derive_useeio_nowcast_Aq_cornerstone, -) -from bedrock.utils.schemas.single_region_types import SingleRegionAqMatrixSet - -logger = logging.getLogger(__name__) - - -def _write_parquets(year: int, aq: SingleRegionAqMatrixSet) -> tuple[Path, Path]: - """Stack (Adom, Aimp) by ``dom_or_imp`` and write to parquet.""" - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - a_path = RESULTS_DIR / f"A_useeio_nowcast_{year}.parquet" - q_path = RESULTS_DIR / f"q_useeio_nowcast_{year}.parquet" - - combined = pd.concat( - {"dom": pd.DataFrame(aq.Adom), "imp": pd.DataFrame(aq.Aimp)}, - axis=0, - names=["dom_or_imp"], - ) - combined.to_parquet(a_path) - aq.scaled_q.to_frame("q").to_parquet(q_path) - return a_path, q_path - - -def _summary_row( - year: int, kind: str, df: pd.DataFrame, file_path: Path -) -> dict[str, object]: - arr = df.to_numpy() - return { - "approach": "useeio_nowcast", - "year": year, - "matrix_kind": kind, - "n_rows": int(df.shape[0]), - "n_cols": int(df.shape[1]), - "nan_count": int(np.isnan(arr).sum()), - "neg_count": int((arr < 0).sum()), - "max_col_sum": float(np.nansum(arr, axis=0).max()) if arr.size else 0.0, - "file_size_bytes": int(file_path.stat().st_size), - } - - -def main() -> None: - logging.basicConfig(level=logging.INFO, format="%(message)s") - summary_rows: list[dict[str, object]] = [] - for year in USEEIO_NOWCAST_YEARS: - logger.info("Deriving useeio_nowcast A for %d ...", year) - derive_useeio_nowcast_Aq_cornerstone.cache_clear() - aq = derive_useeio_nowcast_Aq_cornerstone(year) - a_path, _ = _write_parquets(year, aq) - summary_rows.append(_summary_row(year, "dom", pd.DataFrame(aq.Adom), a_path)) - summary_rows.append(_summary_row(year, "imp", pd.DataFrame(aq.Aimp), a_path)) - - summary_df = pd.DataFrame(summary_rows) - summary_path = RESULTS_DIR / "cache_summary_useeio_nowcast.csv" - summary_df.to_csv(summary_path, index=False) - logger.info("Wrote %d parquet pairs + %s", len(USEEIO_NOWCAST_YEARS), summary_path) - - -if __name__ == "__main__": - main() diff --git a/bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py b/bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py deleted file mode 100644 index 156e79a4..00000000 --- a/bedrock/analysis/a_matrix_time_series/dispatch_ef_time_series.py +++ /dev/null @@ -1,422 +0,0 @@ -"""Dispatch GitHub Actions diagnostics runs for the Step 7 EF time-series. - -Two scenarios: - -- ``isolate_a_matrix`` — vary only the A-matrix scaling methodology, hold - every other config knob to v0 defaults. Reuses the four Step 6 candidate - YAMLs. -- ``bundle_v0_3`` — full v0.3 release-candidate ensembles that bundle - A-matrix selection with every other v0.3 change. YAMLs TBD; populate - ``BUNDLE_V0_3_YAMLS`` when v0.3 is assembled. - -Per ``(scenario, approach, year)`` cell: - -1. Create a Sheet in the Drive folder ``EF_TIME_SERIES_DRIVE_FOLDER_ID`` with a - deterministic title. -2. Trigger the ``generate_diagnostics`` workflow via ``gh workflow run`` with - ``config_name``, ``model_base_year``, ``sheet_id``, and - ``use_useeio_baseline=false`` (CEDA-baseline only for this starting cut). -3. Append a row to ``output/results/ef_run_index.csv`` so the compile step has - a complete audit trail. - -Shared create-sheet / trigger / throttle helpers live in -``bedrock.utils.validation.dispatch_diagnostics``. - -The script is idempotent — already-recorded ``(scenario, approach, year)`` cells -are skipped, so re-running picks up only the unfilled cells. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.dispatch_ef_time_series \\ - --git-ref main \\ - [--scenarios isolate_a_matrix,bundle_v0_3] \\ - [--years 2019,2020,2021,2022,2023] \\ - [--dry-run] -""" - -from __future__ import annotations - -import argparse -import datetime as dt -import logging - -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import RESULTS_DIR -from bedrock.utils.validation.dispatch_diagnostics import ( - create_sheet, - throttle, - trigger_workflow, - wait_for_capacity, -) - -# Backward-compatible aliases for epic scripts / skills that imported the -# private names from this module. -_create_sheet = create_sheet -_throttle = throttle -_trigger_workflow = trigger_workflow -_wait_for_capacity = wait_for_capacity - -logger = logging.getLogger(__name__) - -# A-matrix time-series / v0.3 release-progression Drive folder. -EF_TIME_SERIES_DRIVE_FOLDER_ID = "1M2-Vopqfrx1vGcwoNi6wq55FmoELNV1s" - -EF_RUN_INDEX_PATH = RESULTS_DIR / "ef_run_index.csv" - -DEFAULT_YEARS: tuple[int, ...] = (2019, 2020, 2021, 2022, 2023) - -# `isolate_a_matrix`: candidate A-matrix methodologies, everything else held -# to v0 defaults. Reuses Step 6's YAMLs. -ISOLATE_A_MATRIX_YAMLS: dict[str, str] = { - "useeio": "2025_usa_cornerstone_A_useeio", - "summary_tables": "2025_usa_cornerstone_A_summary_tables", - "commodity_price_index": "2025_usa_cornerstone_A_commodity_price_index", - "useeio_nowcast": "2025_usa_cornerstone_A_useeio_nowcast", -} - -# `bundle_v0_3`: full v0.3 release-candidate ensembles. Each YAML carries -# the full v0.3 flag stack (cornerstone 2026 schema, cornerstone GHG FBS, -# USEEIO B method, waste disagg) PLUS one A-matrix scaling alternative. -# The `model_base_year` and `usa_ghg_data_year` overrides drive the time -# series; the YAMLs themselves are year-agnostic. -BUNDLE_V0_3_YAMLS: dict[str, str] = { - "useeio": "2025_usa_cornerstone_v0_2_A_useeio", - "summary_tables": "2025_usa_cornerstone_v0_2_A_summary_tables", - "commodity_price_index": "2025_usa_cornerstone_v0_2_A_commodity_price_index", - "useeio_nowcast": "2025_usa_cornerstone_v0_2_A_useeio_nowcast", -} - -SCENARIO_YAMLS: dict[str, dict[str, str]] = { - "isolate_a_matrix": ISOLATE_A_MATRIX_YAMLS, - "bundle_v0_3": BUNDLE_V0_3_YAMLS, -} - -# Human-readable labels for sheet titles. `useeio` is renamed to make -# clear that the approach uses BEA 2017 detail benchmark A directly with -# no temporal adjustment — the same "2017 benchmark A" the USEEIO method -# applies. -APPROACH_LABELS: dict[str, str] = { - "useeio": "A matrix with 2017 benchmark A", - "summary_tables": "A matrix with summary tables", - "commodity_price_index": "A matrix with commodity price index", - "useeio_nowcast": "A matrix from USEEIO nowcast", - "full_model": "full v0.3 model", -} -BASELINE_LABELS: dict[str, str] = { - "ceda": "CEDA based", - "useeio": "USEEIO based", -} - -INDEX_COLUMNS = ( - "scenario", - "approach", - "year", - "baseline", - "config_name", - "sheet_id", - "sheet_title", - "useeio_box_ticked", - "git_ref", - "triggered_at", -) - - -def _load_index() -> pd.DataFrame: - if EF_RUN_INDEX_PATH.exists(): - df = pd.read_csv(EF_RUN_INDEX_PATH) - else: - df = pd.DataFrame(columns=list(INDEX_COLUMNS)) - # Backfill any columns this script needs that the CSV doesn't carry. - # The Step 6 CSV pre-dates `scenario` / `year` / `git_ref` etc.; we - # treat missing values as "not recorded by this dispatch." - for col in INDEX_COLUMNS: - if col not in df.columns: - df[col] = "" - return df - - -def _append_index_row(row: dict[str, object]) -> None: - df = _load_index() - df = pd.concat([df, pd.DataFrame([row])], ignore_index=True) - EF_RUN_INDEX_PATH.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(EF_RUN_INDEX_PATH, index=False) - - -def _already_recorded( - df: pd.DataFrame, scenario: str, approach: str, year: int, baseline: str -) -> bool: - if df.empty: - return False - # `year` may be int or empty-string for legacy rows; coerce non-numeric - # entries to NaN so the equality compare safely returns False for those. - year_col = pd.to_numeric(df["year"], errors="coerce") - matches = df[ - (df["scenario"] == scenario) - & (df["approach"] == approach) - & (year_col == year) - & (df["baseline"] == baseline) - ] - return not matches.empty - - -def dispatch( - *, - git_ref: str, - scenarios: tuple[str, ...] = ("bundle_v0_3",), - years: tuple[int, ...] = DEFAULT_YEARS, - approaches: tuple[str, ...] | None = None, - use_useeio_baseline: bool = False, - dry_run: bool = False, - throttle: str = "poll", -) -> None: - baseline_label = "useeio" if use_useeio_baseline else "ceda" - today = dt.datetime.utcnow().strftime("%Y-%m-%d") - - n_planned = 0 - n_skipped = 0 - n_dispatched = 0 - - for scenario in scenarios: - if scenario not in SCENARIO_YAMLS: - raise ValueError( - f"Unknown scenario {scenario!r}. Valid: " f"{sorted(SCENARIO_YAMLS)}" - ) - yamls = SCENARIO_YAMLS[scenario] - if not yamls: - raise ValueError( - f"Scenario {scenario!r} has no YAMLs configured — populate " - f"the corresponding mapping in this script before dispatching." - ) - for approach, config_name in yamls.items(): - if approaches is not None and approach not in approaches: - continue - for year in years: - n_planned += 1 - index_df = _load_index() - if _already_recorded( - index_df, scenario, approach, year, baseline_label - ): - logger.info( - "Skip already-recorded cell (%s, %s, %d, %s)", - scenario, - approach, - year, - baseline_label, - ) - n_skipped += 1 - continue - - approach_label = APPROACH_LABELS[approach] - baseline_text = BASELINE_LABELS[baseline_label] - title = ( - f"[{today}, {year}, {baseline_text}, " - f"{approach_label}, {scenario}] EFs diagnostics" - ) - - if dry_run: - logger.info("DRY-RUN would create sheet: %s", title) - logger.info( - "DRY-RUN would dispatch: config=%s year=%d use_useeio=%s", - config_name, - year, - use_useeio_baseline, - ) - n_dispatched += 1 - continue - - if n_dispatched > 0: - _throttle(throttle) - - sheet_id = _create_sheet(EF_TIME_SERIES_DRIVE_FOLDER_ID, title) - logger.info( - "Created sheet %s for (%s, %s, %d, %s)", - sheet_id, - scenario, - approach, - year, - baseline_label, - ) - _trigger_workflow( - git_ref=git_ref, - config_name=config_name, - sheet_id=sheet_id, - model_base_year=year, - use_useeio_baseline=use_useeio_baseline, - usa_ghg_data_year=year, - ) - _append_index_row( - { - "scenario": scenario, - "approach": approach, - "year": year, - "baseline": baseline_label, - "config_name": config_name, - "sheet_id": sheet_id, - "sheet_title": title, - "useeio_box_ticked": str(use_useeio_baseline).lower(), - "git_ref": git_ref, - "triggered_at": dt.datetime.utcnow().isoformat() + "Z", - } - ) - n_dispatched += 1 - - logger.info( - "Done. planned=%d skipped=%d dispatched=%d", - n_planned, - n_skipped, - n_dispatched, - ) - - -def re_dispatch_from_csv( - *, - git_ref: str, - scenarios: tuple[str, ...] | None = None, - throttle: str = "poll", - dry_run: bool = False, -) -> None: - """Re-trigger workflow runs for cells already recorded in - ``ef_run_index.csv``. Used to recover from rate-limit batch failures — - re-uses the existing Sheets (no new ones created), so the audit trail - keeps the same `sheet_id`s. - - Successful runs that get re-triggered will overwrite their own data - with deterministic identical content; harmless but wastes a few - minutes of runner time. Filter with ``scenarios`` to scope down. - """ - df = _load_index() - if df.empty: - logger.info("No rows in %s; nothing to re-dispatch", EF_RUN_INDEX_PATH) - return - - # Re-dispatch only Step 7 rows (have non-empty scenario + year). - has_step7 = (df["scenario"].astype(str).str.strip() != "") & ( - df["year"].astype(str).str.strip() != "" - ) - df = df[has_step7].copy() - if scenarios: - df = df[df["scenario"].isin(scenarios)] - if df.empty: - logger.info("No matching rows to re-dispatch") - return - - logger.info("Re-dispatching %d cells from %s", len(df), EF_RUN_INDEX_PATH) - n_dispatched = 0 - for _, row in df.iterrows(): - if dry_run: - logger.info( - "DRY-RUN would re-dispatch sheet=%s config=%s year=%s", - row["sheet_id"], - row["config_name"], - row["year"], - ) - n_dispatched += 1 - continue - - if n_dispatched > 0: - _throttle(throttle) - - _trigger_workflow( - git_ref=git_ref, - config_name=str(row["config_name"]), - sheet_id=str(row["sheet_id"]), - model_base_year=int(float(row["year"])), - use_useeio_baseline=str(row["useeio_box_ticked"]).strip().lower() == "true", - usa_ghg_data_year=int(float(row["year"])), - ) - logger.info( - "Re-dispatched sheet %s (%s × %s)", - row["sheet_id"], - row["approach"], - row["year"], - ) - n_dispatched += 1 - - logger.info("Done. re-dispatched=%d", n_dispatched) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--git-ref", - required=True, - help="Git ref (branch or tag) to run the workflow against.", - ) - parser.add_argument( - "--scenarios", - default="bundle_v0_3", - help=( - "Comma-separated scenarios to dispatch. Valid values: " - "'isolate_a_matrix', 'bundle_v0_3'." - ), - ) - parser.add_argument( - "--years", - default=",".join(str(y) for y in DEFAULT_YEARS), - help=f"Comma-separated years (default: {','.join(str(y) for y in DEFAULT_YEARS)}).", - ) - parser.add_argument( - "--approaches", - default="", - help=( - "Optional comma-separated approach filter (e.g. 'useeio_nowcast'). " - "Default empty = all approaches in the chosen scenarios." - ), - ) - parser.add_argument( - "--use-useeio-baseline", - action="store_true", - help="Tick the USEEIO baseline box. Default is CEDA-only baseline.", - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Print the plan without creating Sheets or triggering workflows.", - ) - parser.add_argument( - "--throttle", - default="poll", - help=( - "How to space successive workflow triggers. 'poll' (default) " - "blocks until prior runs clear; 'sleep:N' sleeps N seconds; " - "'none' fires immediately (only safe with bumped Sheets API quota)." - ), - ) - parser.add_argument( - "--re-dispatch-from-csv", - action="store_true", - help=( - "Re-trigger workflows for cells already in ef_run_index.csv. " - "Used to recover from rate-limit batch failures — re-uses " - "existing Sheets, no new ones created." - ), - ) - args = parser.parse_args() - - scenarios = tuple(s.strip() for s in args.scenarios.split(",") if s.strip()) - years = tuple(int(y) for y in args.years.split(",") if y.strip()) - approaches_arg = tuple(a.strip() for a in args.approaches.split(",") if a.strip()) - approaches = approaches_arg if approaches_arg else None - - logging.basicConfig(level=logging.INFO, format="%(message)s") - if args.re_dispatch_from_csv: - re_dispatch_from_csv( - git_ref=args.git_ref, - scenarios=scenarios, - throttle=args.throttle, - dry_run=args.dry_run, - ) - else: - dispatch( - git_ref=args.git_ref, - scenarios=scenarios, - years=years, - approaches=approaches, - use_useeio_baseline=args.use_useeio_baseline, - dry_run=args.dry_run, - throttle=args.throttle, - ) - - -if __name__ == "__main__": - main() diff --git a/bedrock/analysis/a_matrix_time_series/docs/analysis_plan.md b/bedrock/analysis/a_matrix_time_series/docs/analysis_plan.md deleted file mode 100644 index 7d70f48a..00000000 --- a/bedrock/analysis/a_matrix_time_series/docs/analysis_plan.md +++ /dev/null @@ -1,406 +0,0 @@ -# A Matrix Transformation Analysis — Draft Plan - -## Goal - -Analyze four approaches for deriving the A matrix (in commodity-by-commodity format) across a time-series (2017–2024), and recommend one approach as Cornerstone's method for the 2026 model. - -The four approaches partition into: - -- **Two baselines** (always plotted, never the recommendation candidates): - 1. **USEEIO** — derive A in the 2017 benchmark year and use as-is. - 2. **CEDA** — current production default: scale 2017 → IO year via summary-table ratios, then inflate IO year → model year via price index. -- **Two alternative approaches** (the candidates being evaluated): - 1. **Summary tables** — scale 2017 A directly to 2018–2024 via summary-table ratios (price *and* quantity changes captured together; no separate inflation step). - 2. **Commodity price index (V-norm)** — inflate 2017 A directly to 2018–2024 using a *commodity-specific* price index derived from the industry price index and Vnorm (commodity mix). - -Element-wise comparison of every (approach × year) pair informs the recommendation. - -### Comparison baselines (apply to every comparison in this plan) - -**Every divergence / difference / ranking computation in this analysis must be reported against both baselines, side by side:** - -1. **USEEIO baseline** — the "do nothing" invariant; isolates the *combined* effect of any year-scaling method. -2. **CEDA-US baseline** — the current production default (two-step scale + inflate); isolates how each *alternative* method differs from what the model ships today. Inside the `generate_diagnostics` GitHub Actions pipeline this baseline is named **v0** and is applied automatically (see Step 6 for details). - -Reporting against both is non-negotiable: USEEIO alone tells you "did anything change vs not adjusting", CEDA-US alone tells you "did the production path change", and only the pair tells you whether an alternative method is closer to or further from the production status quo while still capturing real year-over-year structural change. Where a single figure can't fit both baselines, produce two figures with a shared color/legend convention. - ---- - -## How to Use This Doc - -### Definition of Done for the whole project - -The project is complete when **all** of the following are true: - -1. A parquet cache of A matrices exists on disk for every (approach × year) pair in the agreed-upon span. -2. All figures in the Deliverables section exist as PNG files in `bedrock/analysis/a_matrix_time_series/output/figures/`. -3. A README in that directory summarizes findings with a single-sentence recommendation per method and a one-paragraph methodology recommendation. -4. The six "Key Questions" below each have a written answer backed by a figure reference. -5. A code change exists on a branch that contains the new `bedrock/analysis/a_matrix_time_series/` module and the README, and at least one peer has reviewed it. - -### Check-in checkpoints (don't skip) - -Show work to a reviewer before proceeding past these gates: - -- **Checkpoint A (after Step 0)**: the V-norm flag implementation code change is open and smoke-tested. Do not start Step 1 until this is merged. -- **Checkpoint B (after Step 1)**: the parquet cache is populated for all (approach × year) combinations and you have printed summary statistics (shape, non-null count, column-sum max) for each matrix. This catches silent caching bugs. -- **Checkpoint C (after Step 3)**: review the cross-approach divergence plots before investing in Steps 4–6. -- **Checkpoint D (before Step 7)**: only proceed if Step 3/4 surfaced a concrete motivation — otherwise, skip. - -### Out of Scope (do NOT do these) - -- Do not modify `derive_cornerstone_Aq_scaled()` or any production pipeline code, other than adding the V-norm branch in Step 0. -- Do not introduce new config flags beyond the V-norm one. -- Do not build a dashboard, UI, or anything interactive. Static PNGs + markdown README only. -- Do not fix unrelated bugs you find along the way. File them as issues. - -### If you get stuck - -- If a config flag produces NaNs or all-zeros: first check `derive_cornerstone_Aq()` base output is sane, then re-read the flag branch in `derive_cornerstone_Aq_scaled()`. -- If you cannot produce a year (e.g. 2018 summary A tables missing): skip that year, document the gap in the README, and continue. Don't invent interpolation. -- If any figure looks implausible (EF shifts > 10×, negative A cells): stop and surface to a reviewer before building on it. - -## Background - -### Current state -- Config flags are wired in `derive_cornerstone_Aq_scaled()` at [bedrock/transform/eeio/derived_cornerstone.py:498](../bedrock/transform/eeio/derived_cornerstone.py#L498): - - `scale_a_matrix_with_useeio_method` → returns 2017 base A unchanged - - `scale_a_matrix_with_summary_tables` → single-step `scale_cornerstone_A(2017 → model_year)`, skips price inflation - - Default (CEDA): two-step `scale_cornerstone_A(2017 → io_year)` then `inflate_cornerstone_A_matrix(io_year → model_year)` -- YAMLs already exist: `2025_usa_cornerstone_a_{useeio,summary_tables}.yaml` -- **Pending** (Step 0 of this plan): a branch + flag `scale_a_matrix_with_commodity_price_index` for the V-norm-derived commodity price index alternative — inflates the 2017 A using commodity-level prices obtained by V-norm transforming the industry-level price index. - -### The four approaches recap (from [notes](https://docs.google.com/document/d/1RlK2ivSnHrku3Q2k5GxRVI_eO25xZ5VE4iIuAlm8TwU/edit?tab=t.0#heading=h.s7guag7kk8nq)) - -| # | Role | Approach | Mechanism | Implicit assumption | -|---|---|---|---|---| -| 1 | Baseline | **USEEIO (do nothing)** | 2017 A used as-is | 2017 technology mix = target year technology mix | -| 2 | Baseline | **CEDA (scale + inflate)** | Two-step: `scale_cornerstone_A(2017 → io_year)` then `inflate_cornerstone_A_matrix(io_year → model_year)` | Separating quantity changes (summary-table ratios) and price changes (price index) is more accurate than either alone | -| 3 | Alternative | **Summary tables** | Element-wise multiply 2017 A by `A_summary(target) / A_summary(2017)`, with 0.98 column-cap | Summary-level structural change is the best signal for detail-level change; captures price + quantity in one step | -| 4 | Alternative | **Commodity price index (V-norm)** | `diag(p) @ A @ diag(1/p)` with `p = price(target) / price(2017)`, where `p` is V-norm transformed from industry → commodity space | Perfect price inelasticity — physical flows constant, A shifts only from relative price changes; commodity prices are more appropriate than industry prices for A matrix inflation, and co-production mixing is non-negligible | - -**Note on #4**: this is the "commodity-transformed" price variant discussed in the notes under *V-Norm Transformation Discussion*. Requires a new config flag and a new branch in `derive_cornerstone_Aq_scaled()` — see Step 0 below. We will use the static Vnorm (2017 benchmark version, same as that used in the B matrix transformation). In the next phase, we will test adjusting Vnorm to years that are consistent with A matrix years. - -### Known caveats from the notes - -- **Summary table scaling** needs a 0.98 cap on column sums (implemented in `scale_cornerstone_A` at [cornerstone_year_scaling.py:111](../bedrock/transform/eeio/cornerstone_year_scaling.py#L111)) and is highly aggregated — especially in manufacturing/services. -- **USEEIO PR is old** — will need a smoke test and possibly a rebase/fix before it produces comparable outputs. -- **No ground-truth A matrix** exists for the target year at BEA detail resolution, so we cannot judge "better" element-wise. Industry gross output (which we do have annually, after redefinition) is the closest external anchor. - ---- - -## Analysis Steps - -### Step −1 — Capitalize the A-matrix YAML filenames (PREREQUISITE) - -Before the V-norm branch lands, rename the A-matrix method YAML files from lowercase-`a` to uppercase-`A` (the "A matrix" noun is capitalized everywhere else in the codebase and docs). Ship this as its own small PR — mechanical renames only, no logic changes. This keeps Step 0 a pure additive change instead of a mixed rename+add diff. - -**Changes required:** - -1. Rename the A-matrix method YAML files in [bedrock/utils/config/configs/](../bedrock/utils/config/configs/): - - `2025_usa_cornerstone_a_useeio.yaml` → `2025_usa_cornerstone_A_useeio.yaml` - - `2025_usa_cornerstone_a_summary_tables.yaml` → `2025_usa_cornerstone_A_summary_tables.yaml` -2. `grep -rn "2025_usa_cornerstone_a_" bedrock/` and update every hit (tests, CI configs, docs). -3. Smoke test: `pytest bedrock/transform/__tests__/test_usa.py -k cornerstone`. - -**Definition of Done for Step −1:** -- Code change lands with only renames — no logic diffs — reviewed and merged. -- Existing diagnostics CI still passes (the rename does not break downstream consumers). -- No references to the old YAML filenames remain in the codebase. - -### Step 0 — Implement Vnorm-derived price index approach (PREREQUISITE) - -Before any analysis can run, the commodity price index approach (alternative #2) must be wired into the pipeline. - -**Changes required:** - -1. Add flag `scale_a_matrix_with_vnorm_price_index: bool = False # DRI: TBD` in [bedrock/utils/config/usa_config.py](../bedrock/utils/config/usa_config.py) alongside the other three. -2. Add a new helper `get_vnorm_adjusted_commodity_price_ratio(original_year, target_year)` in [bedrock/utils/economic/inflate_cornerstone_to_target_year.py](../bedrock/utils/economic/inflate_cornerstone_to_target_year.py) that: - - Takes the existing industry-level price ratio from `get_cornerstone_price_ratio()` (currently indexed by commodity because of how CEDA v7 is structured — confirm by reading the function). - - Transforms to commodity space via `V @ diag(p_industry) @ V^{-1}` (or the equivalent Vnorm transform — verify the exact mechanics with a reviewer before coding). - - Returns a commodity-indexed price ratio Series. -3. Add `inflate_cornerstone_A_matrix_with_commodity_pi()` that uses the Vnorm ratio with the `diag(p) @ A @ diag(1/p)` formula. -4. Add an `if cfg.scale_a_matrix_with_commodity_price_index:` branch in `derive_cornerstone_Aq_scaled()` that applies this inflation directly to the 2017 A. -5. Add YAML `bedrock/utils/config/configs/2025_usa_cornerstone_a_commodity_price_index.yaml`. -6. Smoke test: `pytest bedrock/transform/__tests__/test_usa.py -k cornerstone`. - -**Definition of Done for Step 0:** -- Code change is open with the above changes, tests pass, and a reviewer has approved. -- Running the pipeline with the new YAML produces A matrices with no NaNs. Check column sums: if any column sum is ≥ 1, **surface for review** — do not silently clip or apply a 0.98-cap fix. Unlike summary-table scaling (which has a well-understood cap rationale), a column-sum violation on the V-norm path likely indicates a math error in the transform and needs human review. -- Print the commodity-price-ratio distribution vs the industry-price-ratio distribution (mean, median, 5th/95th percentile) — if they are nearly identical, surface for review since the analysis may be moot. - -**Out of scope for this step:** do not refactor any of the other branches (USEEIO, CEDA default, summary tables). Do not add unit tests beyond what's needed to satisfy existing type checks — the analysis itself is the acceptance test. - -### Step 1 — Produce A matrices for all (approach × year) combinations - -For each `model_base_year ∈ {2017, 2018, 2019, 2020, 2021, 2022, 2024}` and each approach ∈ `{useeio, ceda_default, summary_tables, commodity_price_index}` (2 baselines + 2 alternatives = 4 total): - -1. **Do NOT create new YAML files per year.** Instead, load one of the four approach YAMLs via `get_usa_config()` and programmatically override `model_base_year` in memory (or use a pytest-style config fixture). Rationale: YAMLs are a user-facing deployment surface — committing 28 of them for a one-off analysis clutters the repo and invites copy-paste drift. The four approach YAMLs from Step −1 / Step 0 are sufficient; year is the only thing that varies per run. -2. Invoke `derive_cornerstone_Aq_scaled()` and cache the resulting `(Adom, Aimp, q)` to disk (parquet, keyed by approach + year). - -**Notes:** -- For `Y = 2017`, all four variants should return the same A (ratios are 1.0, price factors are 1.0) — this is a useful sanity check. -- USEEIO is year-invariant by construction — produce it once and reuse; it serves as one of the two fixed baseline lines on every plot (CEDA default is the other; CEDA *does* change with year). -- Output path: `bedrock/analysis/a_matrix_time_series/output/A_{approach}_{year}.parquet` -- Mirror the structure of `bedrock/analysis/time_series_B_matrix/derive_B_time_series.py` — it already solves the "loop over years and cache parquets" problem. - -**Definition of Done for Step 1:** -- Parquet files exist for all (approach, year) combinations in the agreed span. -- A `cache_summary.csv` exists with columns `(approach, year, n_rows, n_cols, nan_count, neg_count, max_col_sum, file_size_bytes)` — one row per matrix. This is the single artifact reviewed at Checkpoint B. -- The 2017 sanity check passes: all approaches produce identical A for year=2017 within `1e-10` relative tolerance. - -### Step 2 — Cell-by-cell time-series diagnostics - -For each approach, compute per-cell summaries across years. - -**Metrics:** -- **Magnitude distribution**: `|A[i,j]|` histogram per year; look for regime shifts. -- **Year-over-year delta**: `A_Y[i,j] - A_{Y-1}[i,j]`, aggregated as L1/L2 norms per column (industry) and per row (commodity). -- **Trajectories**: for sectors in the key-sector list (see Step 4), plot `A[i,j]` vs year per approach on the same axes. - -**Recommended plot formats** (pick based on which question the plot must answer): - -| Plot type | Best for | Notes | -|---|---|---| -| **Small-multiple line plots** (`sector × approach` grid) | Tracking individual A cells over time across approaches | Canonical Tufte-style faceted grid; keep axes consistent per row | -| **ECDF overlay** of `|A[i,j]|` per approach at a fixed year | Tail behavior across approaches | Better than histograms when distributions are long-tailed | -| **Scatter x-y plot** for a single-year vs. baseline comparison (`A_approach[i,j]` on x, `A_baseline[i,j]` on y) | Spotting *systematic* over/under-shoot of an approach against a baseline; locating outlier cells where the two methods disagree most | Plot the identity `y = x` reference line — cells off the diagonal are the divergence story. Use log–log axes because A cells span ~6 orders of magnitude. Color by BEA summary sector group. With ~160k cells, switch to a hexbin (already listed below) once the markers overplot. Produce one scatter per (approach × baseline) pair to honor the two-baseline convention | - -**Reuse existing plot functions from [bedrock/analysis/time_series_B_matrix/derive_B_time_series.py](../bedrock/analysis/time_series_B_matrix/derive_B_time_series.py) wherever possible — Step 2 is structurally the same "metric over years, faceted by sector" problem the B-matrix analysis already solved.** Map to the existing helpers: - -| Step 2 need | Existing function to copy/adapt | How | -|---|---|---| -| Aggregate trend of `sum(abs(A))` or `col_sum_mean` over years per approach | `plot_aggregate_trends()` (L599) | Feed `(year, metric)` tuples per approach; reuse styling | -| Top-N sectors with largest A-cell drift | `plot_top_sector_time_series()` (L675) | Replace gas-axis with approach-axis; sectors stay on y | -| Per-sector time series in a faceted grid | `plot_sector_change_time_series()` (L704) | Swap "gas" grouping for "approach" grouping | -| Cumulative sector contribution | `plot_stacked_bar()` (L774) | As-is for column-sum-contribution view | -| Spaghetti of all sectors | `plot_all_sectors_line()` (L796) | Direct reuse — indexes every sector to 2017 = 100 | - -If any of these functions would require meaningful surgery to reuse, prefer copying + adapting inside the new `a_matrix_time_series/` module rather than generalizing the B-matrix one — keep the two modules independent. - -**Deliverable artifact:** a tall-format table with columns `(row_sector, col_sector, year, approach, A_value, delta_from_2017, delta_yoy, dom_or_imp)` — written once as parquet, used by all downstream plots. Call it `A_cells_long.parquet`. - -**Definition of Done for Step 2:** -- `A_cells_long.parquet` exists and is documented in the module README (schema + row count). -- PNGs `step2_{heatmap,ridgeline,yoy_norms}_{dom,imp}.png` exist in the figures directory. -- Written answer (3–5 sentences) to: "do any approaches show regime shifts — sudden jumps between adjacent years — and if so, where?" - -### Step 2.5 — Cell-level set stability and persistence - -Step 2's `divergence_share_*.png` aggregates over cell identity: a flat year-over-year share of cells above a threshold could be (a) the same cells differing every year (structural offset), (b) a rotating cast of cells averaging to the same count, or (c) the same cells with magnitudes drifting around the bar. (a) and (b) imply very different stories. Step 2.5 distinguishes them. - -**Two diagnostics, both keyed on the existing `A_cells_long.parquet`:** - -1. **Year×year Jaccard heatmap** of "above-threshold" cell sets, per (approach × baseline). High off-diagonal Jaccard ⇒ structural offset (case a). Decay away from the diagonal ⇒ rotating membership (case b). Read the (approach, baseline) panel and the threshold together — at sufficiently tight thresholds even structural offsets drop out of the set. -2. **Persistence histogram** — for each cell, count years above threshold. Stack the share of cells in each `n_years_above` bucket per (approach × baseline). A large "always-above" segment confirms the structural-offset reading; mass spread across middle buckets indicates rotation. - -**Implementation:** `bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py`. Reads the parquet from Step 2 only — no upstream pipeline changes. Computes both diagnostics over `ALL_THRESHOLDS = (1e-6, 1e-5, 1e-4, 1e-3, 1e-2)` (matching Step 2) and renders plots at `PLOT_THRESHOLDS = (1e-4, 1e-3)` where the cell sets are non-degenerate. - -**Definition of Done for Step 2.5:** -- PNGs `set_stability_jaccard_thr{thr}_{kind}.png` for `kind ∈ {dom, imp}` and `thr ∈ PLOT_THRESHOLDS`, plus the multi-threshold composite `persistence_by_threshold_{kind}.png`. -- CSVs `set_stability_jaccard.csv` and `persistence_categories.csv` cover all of `ALL_THRESHOLDS`. -- Sheet tabs `set_stability_jaccard` and `persistence_categories` appended to the run-report Sheet. -- Written answer (3–5 sentences) to: "is the divergence pattern in Step 2's share-of-cells line a stable cell population (structural) or rotating membership, and how does that finding interact with the fixed `usa_io_data_year=2022` leg of `ceda_default`?" - -### Step 3 — Cross-approach comparison at fixed target year (2024) - -Pick `model_base_year = 2024` and compute pairwise differences between approaches on the **same** A matrix grid: - -- Hexbin scatter: `A_summary_tables[i,j]` vs `A_commodity_price_index[i,j]`, colored by column-sector BEA summary code. -- **ECDF (paired)** of relative divergence per approach, computed twice — once vs USEEIO and once vs CEDA. Both ECDFs go in the same figure (or a 1×2 panel) so the reader can read off each approach's distance from each baseline at the same threshold. - - Panel A: `(A_approach - A_useeio) / A_useeio` per approach - - Panel B: `(A_approach - A_ceda) / A_ceda` per approach -- Column-sum diagnostics: verify all columns ≤ 1 (summary tables relies on the 0.98 cap — count how many columns actually hit the cap). - -**Why ECDF specifically (not histogram):** -- **Answers the actual question directly.** The question is "what fraction of A cells diverge by more than X?" — an ECDF lets you read that off any threshold on the x-axis in one glance. A histogram forces the reader to eyeball-integrate. -- **No binning artifacts.** A-matrix cells span many orders of magnitude (`~10^-6` to `~10^-1`), and divergence ratios have a heavy tail. Histograms require either log-bins or wide bins, both of which obscure the shape — ECDFs sidestep the choice entirely. -- **Overlaying four approaches is cleaner than overlaying four histograms.** 4 lines stack readably; 4 colored histograms fight each other visually. -- **Tail focus.** The interesting divergences live in the tails — e.g. "5% of cells diverge by more than 50%". ECDFs show the tail mass directly as the distance from 1.0 on the y-axis. - -**Expected finding from the notes:** summary-table scaling should show the largest divergence and is most prone to capping in manufacturing/service columns. - -**Definition of Done for Step 3:** -- PNG `step3_pairwise_hexbins.png` (grid of hexbins, one per approach-pair). -- PNG `step3_divergence_ecdf_vs_useeio.png` and `step3_divergence_ecdf_vs_ceda.png` — one ECDF panel per baseline, same approach color convention across both. -- CSV `step3_column_cap_audit.csv` listing every column where summary-tables scaling triggered the 0.98 cap. -- Written answer to: "how much do approaches disagree against each baseline (USEEIO and CEDA)?" - -### Step 4 — Zoom into key sectors - -Pick a curated sector list where priors differ across approaches: - -| Sector | Why it matters | -|---|---| -| Energy inputs (e.g. electricity, natural gas) into manufacturing | Near-fixed stoichiometry → price-index approach expected to be most valid | -| Travel / hospitality / discretionary services | Price elasticity likely high → price-index assumption most likely to break | -| Waste (562111–562xxx disaggregated children) | Intragroup treatment + inherited price ratios; want to confirm nothing exploded | -| High-volatility commodities (petroleum refining, primary metals) | Year-scaling noise most visible | -| Sectors with large disaggregation ratios (1 BEA → many Cornerstone children) | Summary-table scaling uses parent ratio — all children move together, may be unrealistic | - -For each: small-multiple line plot per column-industry showing `A[commodity, industry]` across years for each approach. - -**Definition of Done for Step 4:** -- PNG `step4_keysector_{energy,travel,waste,volatility,disagg}.png` — one figure per category above. -- CSV `step4_sector_shortlist.csv` listing the actual BEA codes chosen with one-line justification per row. The shortlist is reviewed before the figures are finalized. -- Written answer to: "where are the largest cross-approach disagreements, and do they line up with the theoretical priors from the notes?" - -### Step 5 — Industry output as ground-truth anchor - -Notes say detail-level industry output time-series is "relatively stable reference/ground truth". This means: we already have annual BEA after-redefinition gross output — see `derive_gross_output_after_redefinition()`. Compare: - -- **Model-derived industry output** from each scaled A matrix (via full-model Leontief computation, Step 6). -- **Observed BEA after-redefinition gross output** for the same year. - -For each approach, per year, compute industry-level relative error vs observed. Aggregate as a single scalar (weighted RMSE over industries) to rank approaches. - -**Caveat to name in the write-up:** industry output is a *column-sum-adjacent* diagnostic; it can agree well even when individual A cells are wrong. It gives directional/magnitude signal, not element-wise truth. - -**Definition of Done for Step 5:** -- CSV `step5_industry_output_errors.csv` with columns `(approach, year, industry_rmse_vs_bea, top_5_worst_industries)`. -- PNG `step5_output_rmse_ranking.png` — bar chart of RMSE per approach per year (4 approaches × 7 years = 28 bars, grouped). -- Written answer to: "does industry output error meaningfully discriminate between approaches, or are they all within noise?" - -### Step 6 — Full-model diagnostics (EF / EI impacts) - -For each approach × 2024, run the full model via the existing `generate_diagnostics` GitHub Actions workflow (same diagnostics Google Sheet format the B-matrix work uses; backed by `calculate_ef_diagnostics.py`). - -#### How the diagnostics pipeline supplies the two baselines - -The two-baseline convention is enforced by the `generate_diagnostics.yml` workflow itself — the engineer does not implement comparison logic, just configures the workflow correctly: - -| Baseline | How it enters the diagnostics output | Engineer action required | -|---|---|---| -| **CEDA-US** | Automatically used as **v0** inside the diagnostics pipeline. Every diagnostics run produces a comparison against CEDA-US whether you ask for it or not. | None — comes for free with every workflow run. | -| **USEEIO** | Opt-in. Activated by ticking the **"Benchmark to USEEIO GCS Excel baseline (URI, SHA, label from `useeio_baseline_pin.json`)"** checkbox in the `Run workflow` dialog. | **Always tick this box** when triggering Step 6 runs, so each approach gets compared against both baselines in a single pass. | - -If the USEEIO checkbox is forgotten on any run, that run only produces the CEDA-US comparison and must be re-triggered to recover the USEEIO comparison — re-running is cheap, but track it. - -#### Triggering the runs - -For each of the four approaches: -1. Open `generate_diagnostics` → `Run workflow`. -2. Set **USA config name** to the approach's YAML stem (e.g. `2025_usa_cornerstone_A_summary_tables`). -3. Set **Google Sheets ID** to the destination sheet for that approach. -4. **Tick** the "Benchmark to USEEIO GCS Excel baseline" checkbox. -5. (Optional) attach the PR URL for that approach. -6. Run. - -The CEDA-default approach is already on `main` and already exercised by the production diagnostics — its sheet may already exist; reuse it rather than re-running unless config has drifted. - -#### Output - -- Emission factors per sector for each approach. -- Percent difference in final-demand-driven emissions and sector-level gross output, against **both** baselines (USEEIO via the checkbox, CEDA-US via v0). Both come from the same diagnostics sheet — no extra computation needed on the engineer's side. - -**Action**: compile the resulting diagnostics sheets (one per approach) into one comparison workbook covering all four approaches rather than re-running from scratch. - -**Definition of Done for Step 6:** -- All four approaches have a completed diagnostics run with the USEEIO checkbox ticked. Save the run URLs in `step6_run_index.csv` (columns: `approach`, `run_id`, `sheet_id`, `useeio_box_ticked`, `triggered_at`) so the workbook compilation step can locate every input. -- One Excel workbook `step6_ef_comparison.xlsx` in the output directory with one tab per approach plus a `summary_vs_useeio` tab and a `summary_vs_ceda` tab. -- PNGs `step6_ef_divergence_scatter_vs_useeio.png` and `step6_ef_divergence_scatter_vs_ceda.png` — EF values per approach against each baseline, shared color convention. -- Written answer to: "which approach's EFs look most plausible against the USEEIO published EFs and the CEDA-US production default?" - ---- - -## Deliverables (concrete artifact list) - -This is the exhaustive, named list of files the engineer must produce. If a file below does not exist by project end, the project is not done. - -### Code -- [ ] `bedrock/utils/config/usa_config.py` — new flag `scale_a_matrix_with_vnorm_price_index` (Step 0) -- [ ] `bedrock/utils/economic/inflate_cornerstone_to_target_year.py` — new `get_vnorm_commodity_price_ratio()` + `inflate_cornerstone_A_matrix_vnorm()` (Step 0) -- [ ] `bedrock/transform/eeio/derived_cornerstone.py` — new branch in `derive_cornerstone_Aq_scaled()` (Step 0) -- [ ] `bedrock/utils/config/configs/2025_usa_cornerstone_a_vnorm_price_index.yaml` (Step 0) -- [ ] `bedrock/analysis/a_matrix_time_series/__init__.py` -- [ ] `bedrock/analysis/a_matrix_time_series/derive_A_time_series.py` — parquet caching driver (Step 1); mirror `derive_B_time_series.py` -- [ ] `bedrock/analysis/a_matrix_time_series/derive_A_cells_stability.py` — Step 2.5 set-stability + persistence diagnostics -- [ ] `bedrock/analysis/a_matrix_time_series/compare_approaches.py` — generates Step 2–4 plots -- [ ] `bedrock/analysis/a_matrix_time_series/compare_industry_output.py` — Step 5 -- [ ] `bedrock/analysis/a_matrix_time_series/compile_ef_diagnostics.py` — Step 6 - -### Data artifacts (`bedrock/analysis/a_matrix_time_series/output/`) -- [ ] `A_{approach}_{year}.parquet` for each (approach, year) — Step 1 -- [ ] `cache_summary.csv` — Step 1 Checkpoint B artifact -- [ ] `A_cells_long.parquet` — Step 2 tall-format table -- [ ] `set_stability_jaccard.csv`, `persistence_categories.csv` — Step 2.5 -- [ ] `step3_column_cap_audit.csv` — Step 3 -- [ ] `step4_sector_shortlist.csv` — Step 4 -- [ ] `step5_industry_output_errors.csv` — Step 5 -- [ ] `step6_run_index.csv` — Step 6 (workflow-run audit trail; one row per approach) -- [ ] `step6_ef_comparison.xlsx` — Step 6 - -### Figures (`bedrock/analysis/a_matrix_time_series/output/figures/`) -- [ ] `step2_heatmap_{dom,imp}.png`, `step2_ridgeline_{dom,imp}.png`, `step2_yoy_norms_{dom,imp}.png` -- [ ] `set_stability_jaccard_thr{1e-1,1e-2,1e-3}_{dom,imp}.png`, `persistence_by_threshold_{dom,imp}.png` — Step 2.5 -- [ ] `step3_pairwise_hexbins.png`, `step3_divergence_ecdf_vs_useeio.png`, `step3_divergence_ecdf_vs_ceda.png` -- [ ] `step4_keysector_{energy,travel,waste,volatility,disagg}.png` -- [ ] `step5_output_rmse_ranking.png` -- [ ] `step6_ef_divergence_scatter_vs_useeio.png`, `step6_ef_divergence_scatter_vs_ceda.png` -- [ ] (optional) `step7_coproduction_sectors.png` - -### Write-up -- [ ] `bedrock/analysis/a_matrix_time_series/README.md` — ≤2 pages. Must contain: - - Method-by-method pros/cons table (4 rows: 2 baselines + 2 alternatives) - - Answers to all six Key Questions (one paragraph each, figure reference inline) - - **Explicit recommendation of one approach** as Cornerstone's method, with reasoning trace tying back to the figures - - Explicit list of limitations + years/methods skipped for data-availability reasons - ---- - -## Key Questions to Answer (from the notes) - -- [ ] How much do the two alternative approaches disagree at the cell level, against each baseline (USEEIO and CEDA)? (Step 3) -- [ ] Where are the biggest disagreements — manufacturing, services, energy, or waste? (Steps 3–4) -- [ ] Does industry-output error meaningfully discriminate between approaches? (Step 5) -- [ ] Which approach's EF results land closest to each baseline (USEEIO published EFs and CEDA-US production default)? (Step 6) -- [ ] Is the price-index assumption defensible for discretionary-spending sectors, or does it introduce noise? (Step 4) -- [ ] Is the summary-table 0.98 cap firing often enough to be a concern? (Step 3) - ---- - -## Open Decisions Before Starting - -1. **Time-series span**: brief calls for 2018–2024 plus the 2017 benchmark = 7 years; the current Step 1 list `{2017, 2018, 2019, 2020, 2021, 2022, 2024}` skips 2023. Confirm — do we have summary A tables and price indices for all 7 listed years? `USA_SUMMARY_MUT_YEARS` in [bedrock/utils/taxonomy/bea/matrix_mappings.py] caps `model_base_year` at `{2022, 2023, 2024}` — so the scale-method approaches likely cannot produce 2018–2021 directly without summary-table availability gaps. **Action: read `derive_summary_Adom_usa()` and enumerate the supported years before committing to the final span; document any year that has to be skipped per approach.** -2. ~~**Reference baseline for "divergence from"**~~ — **resolved**: use both USEEIO and CEDA-US as paired baselines on every comparison (see "Comparison baselines" section at the top). In Step 6, both baselines are produced by the diagnostics workflow itself — CEDA-US automatically (as v0), USEEIO via the workflow's checkbox. -3. **Output destination**: add as a new `bedrock/analysis/a_matrix_time_series/` module vs extend `time_series_B_matrix`? New module keeps scope clean. -4. **Significance threshold for outliers**: the `diagnostics_visual_plan.md` uses `|perc_diff| > 1.0` (100%). Reuse that unless evidence says otherwise. - ---- - -## Dependencies / Risks - -- **USEEIO PR freshness**: notes say "may need minor code updates since PR is from a while ago". Rebase + smoke-test the USEEIO YAML end-to-end before investing in comparisons. (USEEIO is a baseline, so the analysis is dead in the water until this is reliable.) -- **Memory / runtime**: caching × 4 approaches × 7 years = 28 A matrices (USEEIO contributes 1 since it's year-invariant, so practically ~22). Leverage `@functools.cache` + parquet caching; only do full-model propagation (Step 6) for the single target year, not all 7. -- **No element-wise ground truth**: be explicit about this limitation in the write-up. The recommendation must be grounded in a combination of (industry output error + EF plausibility + methodological transparency), not in a single "best" metric. -- **V-norm math must be verified**: Step 0's transform logic must be reviewed before merge. Getting this wrong silently produces a plausible-but-meaningless 5th approach. - ---- - -## Suggested Milestones (rough sizing) - -These are sizing estimates, not commitments. Adjust after Checkpoint A. - -| Milestone | Steps | Rough effort | Gate | -|---|---|---|---| -| M1 — V-norm branch merged | Step 0 | 2–3 days | Checkpoint A | -| M2 — Cache populated | Step 1 | 1 day | Checkpoint B | -| M3 — Cross-approach plots | Steps 2–3 | 2–3 days | Checkpoint C | -| M4 — Sector deep-dives + output ground truth | Steps 4–5 | 2 days | — | -| M5 — EF compile + write-up | Step 6 + README | 1–2 days | Final review | -| M6 (optional) — V-norm deep dive | Step 7 | 1 day | Checkpoint D | - -Total: roughly 1.5–2 working weeks for the core path (M1–M5). - ---- - -## Reference Files - -| File | Role | -|---|---| -| [bedrock/transform/eeio/derived_cornerstone.py](../bedrock/transform/eeio/derived_cornerstone.py) | `derive_cornerstone_Aq_scaled()` — the gated entry point (L498) | -| [bedrock/transform/eeio/cornerstone_year_scaling.py](../bedrock/transform/eeio/cornerstone_year_scaling.py) | `scale_cornerstone_A` (0.98 cap logic) | -| [bedrock/utils/economic/inflate_cornerstone_to_target_year.py](../bedrock/utils/economic/inflate_cornerstone_to_target_year.py) | `inflate_cornerstone_A_matrix` (diagonalization) | -| [bedrock/transform/iot/derived_gross_industry_output.py](../bedrock/transform/iot/derived_gross_industry_output.py) | `derive_gross_output_after_redefinition()` — ground-truth anchor | -| [bedrock/analysis/time_series_B_matrix/derive_B_time_series.py](../bedrock/analysis/time_series_B_matrix/derive_B_time_series.py) | Template for time-series caching + plotting module layout | -| [bedrock/utils/validation/calculate_ef_diagnostics.py](../bedrock/utils/validation/calculate_ef_diagnostics.py) | Existing EF diagnostics pipeline (Step 6) | diff --git a/bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md b/bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md deleted file mode 100644 index f7c40f1a..00000000 --- a/bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md +++ /dev/null @@ -1,245 +0,0 @@ -# USEEIO Nowcast Integration — Plan - -## Goal - -Integrate the upstream USEEIO team's **nowcasted detailed Supply-Use Tables** (2018–2023) into [bedrock/analysis/a_matrix_time_series/](.) as an additional A-matrix approach and as an independent external anchor for Step 5 of [analysis_plan.md](analysis_plan.md). End state: every figure and CSV in the existing analysis carries one extra series (`useeio_nowcast`) so reviewers can read off "how do bedrock's five approaches line up against an externally-balanced 2023 detail SUT?" - -Background on what the upstream pipeline does and what it captures: [/USEEIO_nowcasting.md](../../../USEEIO_nowcasting.md). Source artifacts live at `~/Desktop/nowcasting/final nowcasted tables/` (V_out, U_out, U_imports_out CSVs for 2017–2023). - -## TL;DR - -1. Stage the USEEIO nowcasted CSVs to GCS (read-only, versioned by upload-date). -2. Add a 6th approach `useeio_nowcast` whose A-matrix is derived from the upstream V/U/U_imports tables instead of from the bedrock 2017 base + a scaling/inflation rule. The YAML is configurable but the derivation runs *outside* `derive_cornerstone_Aq_scaled()` — this is exogenous data, not a method. -3. Build a one-shot loader that maps BEA Detail Code_Loc → Cornerstone schema, derives `Adom`/`Aimp` per year, and writes them into the same parquet layout as the existing approaches: `output/results/A_useeio_nowcast_{year}.parquet`. -4. Re-run Steps 1–5 of the existing plan with the new approach added to `APPROACH_ORDER`. Skip Step 0 (no new flag). Skip Step 6 unless the nowcast is selected as a candidate. -5. Step 5 (industry-output anchor) gets a parallel **A-matrix anchor**: compare each of the five internal approaches' `A_2023` against `A_useeio_nowcast_2023` cell-by-cell. This is the closest thing to external ground truth that exists at detail resolution. - ---- - -## Why this is worth doing - -The existing analysis lacks an external A-matrix reference. Step 5 uses observed BEA annual gross output as anchor — but as the plan itself notes, gross output is "column-sum-adjacent" and can agree even when individual A cells are wrong. The USEEIO nowcasted SUTs give us, for the first time, a detail-resolution A matrix for 2018–2023 that was balanced against BEA's annual summary SUTs by an independent EPA-funded team using GRAS (see [USEEIO_nowcasting.md](../../../USEEIO_nowcasting.md)). It's not perfect (it freezes 2017 within-summary technology, the same fundamental limitation as bedrock's `summary_tables` method — documented in that doc's "what is NOT captured" section), but it's the only externally-balanced detail-level series available, and the methodological overlap with `summary_tables` is exactly what makes it informative: it isolates where the bedrock alternatives disagree from a published implementation of the same conceptual approach. - ---- - -## Inputs - -| Input | Source | Format | Status | -|---|---|---|---| -| `V_out_{yr}.csv` | `gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/` | commodity × industry, BEA Detail bare codes (no `/US`), mUSD | ✅ staged | -| `U_out_{yr}.csv` | same | commodity × (industry + FD + VA), BEA Detail bare codes, mUSD | ✅ staged | -| `U_imports_out_{yr}.csv` | same | commodity × (industry + FD), mUSD | ✅ staged | -| BEA Detail → Cornerstone schema map | bedrock's existing `cs_commodity_to_bea_map()` + `expand_square_matrix(zero_intragroup_cross_terms=True)` | reused | ✅ exists | -| 2017 Vnorm (scrap-corrected) | bedrock's existing `bea_Vnorm_scrap_corrected()` | reused | ✅ exists | -| Unorm / Vnorm helpers | bedrock's existing `compute_Unorm_matrix`, `compute_Vnorm_matrix` | reused | ✅ exists | - -Years available: **2017, 2018, 2019, 2020, 2021, 2022, 2023** (7 files per table × 3 tables = 21 CSVs). - -Gap vs. existing plan: **no 2024** (USEEIO pipeline hasn't been run for 2024 upstream). Strategy: drop 2024 from cross-approach plots that include `useeio_nowcast`, OR carry-forward 2023 with an explicit "extrapolated" label. **Recommend drop.** - -Format note: USEEIO column/row headers are **bare BEA Detail codes** (e.g. `1111A0`, `211000`) — no `/US` suffix. Bedrock's internal Cornerstone codes carry no suffix either, so loaders can pass codes through unchanged, but `bea_v2017_to_ceda_v7_helpers.py` and adjacent code that expects `code/US` will need stripping at the boundary. - ---- - -## Pre-requisites - -### P0 — Stage source data to GCS — ✅ DONE - -21 CSVs at `gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/` (verified via `gsutil ls`). Upstream branch state: `cornerstone-data/USEEIO@nowcasting` HEAD `2025-09-30` (per [USEEIO_nowcasting.md](../../../USEEIO_nowcasting.md)). Authoritative method writeup: `USEEIO_nowcasting_2025_10_15_rev1.docx` (Wood/Vendries/Young). - -**Follow-up nice-to-have (non-blocking):** add a `MANIFEST.txt` next to the CSVs recording the upstream branch SHA and rev1 doc version for reproducibility. Can land alongside Step N1. - -### P1 — BEA Detail → Cornerstone schema map — ✅ ALREADY EXISTS - -Bedrock's `derive_cornerstone_Aq()` ([derived_cornerstone.py:448](../../../bedrock/transform/eeio/derived_cornerstone.py#L448)) already does BEA Detail (~400) → Cornerstone (~405) expansion via: - -- `cs_commodity_to_bea_map()` — the mapping itself -- `expand_square_matrix(matrix, CS_COMMODITY_LIST, com_map, zero_intragroup_cross_terms=True)` — applies it with intragroup-zeroing to prevent Leontief-inverse inflation -- `expand_vector(vec, CS_COMMODITY_LIST, com_map)` — vector variant - -These are the same helpers used when waste disaggregation is **off**. `useeio_nowcast` will use them as-is — no new crosswalk needed. - -**Caveat:** when production YAML has waste disagg **on** (`get_waste_disagg_weights()` returns non-None), `derive_cornerstone_Aq()` takes a different path with disaggregated V/U in Cornerstone space. USEEIO upstream does **not** disaggregate waste — its V/U are at BEA Detail. So `useeio_nowcast` always goes through the `expand_square_matrix` path regardless of comparison-baseline YAML settings. For an apples-to-apples comparison, **run all 6 approaches with waste disagg off** in this analysis, OR explicitly call out that waste rows are not comparable. - ---- - -## Steps - -### Step N1 — Load USEEIO nowcasted tables and derive A matrices - -**Pre-flight smoke-test findings (2017 + 2023, see [`investigation/useeio_nowcast_smoke_2023.py`](investigation/useeio_nowcast_smoke_2023.py)):** - -- Industry sets match exactly (bedrock and USEEIO both use bare BEA Detail codes, no `/US` suffix). -- USEEIO `U_out` includes 3 VA rows (`V00100, V00200, V00300`) appended below 402 commodity rows — harmless, dropped by reindex to bedrock's commodity space. -- Bedrock's `bea_Vnorm_scrap_corrected` has an index-alignment quirk that pads to 406 commodity columns (`331314, S00101, S00201, S00202`) — these are non-real commodities and get dropped by `expand_square_matrix`. **Pre-existing bedrock behavior; not a USEEIO issue.** -- 2023 Cornerstone-space output is clean: `(405, 405)`, no NaN, no negatives, all column sums ≤ `COLUMN_CAP` after cap is applied. -- Decisions 8/9/10 above were settled by the smoke test outputs. - -**Code lives in three layers, matching bedrock's `extract/` → `transform/` → `analysis/` convention:** - -| Layer | File | Role | -|---|---|---| -| Extract | [`bedrock/extract/iot/useeio_nowcast.py`](../../extract/iot/useeio_nowcast.py) | GCS loaders for `V_out`, `U_out`, `U_imports_out` (transposes V to industry × commodity, slices intermediate-Use block, `@functools.cache` per year). Owns `USEEIO_NOWCAST_YEARS` and `USEEIO_NOWCAST_INDUSTRY_COUNT`. | -| Transform | [`bedrock/transform/eeio/derived_useeio_nowcast.py`](../../transform/eeio/derived_useeio_nowcast.py) | A-matrix derivation: `compute_Unorm_matrix` → Vnorm (inline scrap-corrected) → `compute_A_matrix` → `expand_square_matrix`. Applies negative-clip + 0.98 cap. Exposes `derive_useeio_nowcast_Aq_cornerstone(year)` returning `SingleRegionAqMatrixSet`. Called from `derive_cornerstone_Aq_scaled()`'s new branch. | -| Analysis | [`derive_useeio_nowcast_A.py`](derive_useeio_nowcast_A.py) | Thin driver: loops `USEEIO_NOWCAST_YEARS`, calls the transform function, writes parquet caches in the same layout `_loaders.load_a_pair` expects. | - -**Per year `yr ∈ {2018..2023}` (and 2017 as identity sanity check):** - -1. Read the three CSVs from `gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/` into pandas DataFrames. Row/column headers are bare BEA Detail codes. -2. Slice the intermediate Use blocks: `U_out_{yr}` columns `[0:numIndustries]` → intermediate use (commodity × industry); drop FD and VA columns. Same slicing for `U_imports_out_{yr}` to get the imports intermediate block (commodity × industry). Confirm column counts match `len(detail_industry_codes)` from `derive_cornerstone_Aq()`. -3. Subtract: `U_dom_intermediate = U_intermediate − U_imports_intermediate`. -4. Compute industry gross output: `x = V_out_{yr}.sum(axis=0)`. Sanity-check for `yr=2017` that this matches bedrock's `Detail_GrossOutput_IO` to numerical tolerance. -5. Apply bedrock's existing helpers (no new math) to produce BEA-space A matrices: - ```python - Unorm_dom = compute_Unorm_matrix(U=U_dom_intermediate, x=x) - Unorm_imp = compute_Unorm_matrix(U=U_imports_intermediate, x=x) - Vnorm = bea_Vnorm_scrap_corrected() # bedrock's 2017 V-norm — see decision #2 below - Adom_bea = Unorm_dom @ Vnorm - Aimp_bea = Unorm_imp @ Vnorm - ``` -6. Expand to Cornerstone schema using bedrock's existing path: - ```python - com_map = cs_commodity_to_bea_map() - Adom = expand_square_matrix(Adom_bea, CS_COMMODITY_LIST, com_map, zero_intragroup_cross_terms=True) - Aimp = expand_square_matrix(Aimp_bea, CS_COMMODITY_LIST, com_map, zero_intragroup_cross_terms=True) - ``` -7. **Clip negative U cells to 0** with magnitude logging (decision #9 above). Worst negatives in 2023 are ~$2k mUSD on intra-electronics intermediate blocks — reconciliation noise, not real demand. -8. **Apply the 0.98 column cap** to Adom and Aimp separately (decision #10 above), mirroring `scale_cornerstone_A`'s post-processing. Only `S00102` (noncomparable imports) is observed to need it in 2023. -9. Sanity asserts (mirror `derive_cornerstone_Aq()`): `(Adom >= 0).all()`, `(Aimp >= 0).all()`, no NaNs, all column sums ≤ 1. -10. Write to `output/results/A_useeio_nowcast_{yr}.parquet` in the same long-form (kind={dom,imp}) layout that `_loaders.py::load_a_pair` expects. - -**Decision recap embedded in step 5:** -- **Vnorm year**: use bedrock's 2017 `bea_Vnorm_scrap_corrected()` rather than computing a year-specific V-norm from `V_out_{yr}`. Rationale: keeps the variable-of-interest (input shares from `U_out_{yr}`) clean — only one channel of structural change. Mirrors the `commodity_price_index` approach's convention. -- **Imports split**: USEEIO already published `U_imports_out_{yr}` as a separate matrix balanced to summary imports. Use it directly — do not re-derive imports shares from a 2017 import share assumption (this is one of the things the nowcast actually updates). - -**Module entrypoint:** -```python -python -m bedrock.analysis.a_matrix_time_series.derive_useeio_nowcast_A -``` -which loops over years 2017–2023. - -**Definition of Done for Step N1:** -- 7 parquet files written. -- `load_a_pair("useeio_nowcast", 2023)` returns `dom` and `imp` DataFrames with the same shape as `load_a_pair("summary_tables", 2023)`. -- Cache-summary row appended for each year/kind. -- Column sums of `Adom` (and `Aimp`) are ≤ `COLUMN_CAP` for every column after the cap is applied. Log which columns hit the cap. -- ~~2017 identity check vs bedrock `bea_Aq()`~~ — **NOT required.** The smoke test confirms USEEIO 2017 differs from bedrock 2017 by ~9% L1 (Adom) / 22% L1 (Aimp) due to different BEA file vintages (see decision #8). Record the gap in the cache-summary as informational; do not block on it. - -### Step N2 — Wire `useeio_nowcast` into shared constants - -Edit `constants.py`: - -1. Append `"useeio_nowcast"` to `APPROACH_ORDER` after `commodity_price_index`. **Do not** add it to `ALTERNATIVE_APPROACHES` — it is not a candidate for the production method; it is an external reference. -2. Add a color (`#9467bd` — distinct from existing 5). -3. Define a new tuple `EXTERNAL_REFERENCES: tuple[str, ...] = ("useeio_nowcast",)` to formalize the "reference, not candidate" role. Update plot helpers (Step N3) to honor it. - -`APPROACH_YAMLS` in `derive_A_time_series.py` does **not** get a new entry — the nowcast approach has no YAML because it isn't derived through `derive_cornerstone_Aq_scaled()`. Step N1's standalone module produces its parquet files directly. - -### Step N3 — Re-run Steps 1–5 with the new approach - -For each downstream step, the diff is bounded: - -| Existing step | Change | -|---|---| -| **Step 1** (`derive_A_time_series.py`) | No change. `useeio_nowcast` parquets come from Step N1's separate module. Cache-summary tab gets 7 extra rows from Step N1. | -| **Step 2** (cell-level long-form table `A_cells_long.parquet`) | Loader iterates `APPROACH_ORDER` so it picks up `useeio_nowcast` automatically. Verify with row count = old × 6/5. | -| **Step 2.5** (Jaccard + persistence) | Auto-included via `APPROACH_ORDER`. | -| **Step 3** (`compare_approaches.py`) | Pairwise hexbins: add `(useeio_nowcast, summary_tables)`, `(useeio_nowcast, industry_price_index)`, `(useeio_nowcast, commodity_price_index)` to `PAIRS`. ECDF panels: add `useeio_nowcast` as a third reference series ("alternatives vs USEEIO-do-nothing", "vs CEDA-US", "vs USEEIO-nowcast"). Column-cap audit unchanged. | -| **Step 4** (key sectors) | Auto-included; small-multiple lines pick up the 6th series. | -| **Step 5** (industry output anchor) | Add a parallel **A-matrix anchor** sub-step: per approach and year, compute `||A_approach - A_useeio_nowcast||` (L1 per column, RMSE overall) for 2018–2023. Report as `step5_a_matrix_rmse_vs_useeio_nowcast.csv`. The headline message becomes "which approach tracks the externally-balanced nowcast most closely on both industry output and A-cell values?" | - -**Step 6 is deferred.** Running EFs through the nowcast is expensive (full diagnostics workflow per approach × year) and only worth it if Step N3 surfaces the nowcast as a contender for the production recommendation. Decision gate: if `useeio_nowcast` lines up systematically with one of the three alternative methods in Step N3, add it as a 6th tile in Step 6 outputs. Otherwise skip. - -### Step N4 — Update the write-up - -Edit [analysis_plan.md](analysis_plan.md) (or write a sibling `README.md` for this sub-plan): - -1. Add a row to the "five approaches" table for `useeio_nowcast` — role = "external reference", mechanism = "upstream EPA team's GRAS-balanced detail SUT, 2018–2023". -2. Add to **Key Questions**: "do any of the three alternative bedrock methods track the externally-balanced nowcast more closely than the others, and at which sectors?" -3. In the final recommendation section, treat `useeio_nowcast` alignment as one of three pieces of evidence (alongside industry-output error and EF plausibility). Caveat: the nowcast freezes 2017 within-summary technology — the same fundamental limitation as `summary_tables` — so close agreement with `summary_tables` is partly a methodological tautology, not validation. Call this out. - ---- - -## Deliverables - -### Code -- [x] `bedrock/extract/iot/useeio_nowcast.py` — GCS loaders (extract layer) -- [x] `bedrock/transform/eeio/derived_useeio_nowcast.py` — A-matrix derivation (transform layer) -- [x] `bedrock/analysis/a_matrix_time_series/derive_useeio_nowcast_A.py` — Step N1 driver (analysis layer) -- [x] `bedrock/utils/config/usa_config.py` — `load_useeio_nowcast_A_matrix` flag -- [x] `bedrock/transform/eeio/derived_cornerstone.py` — new branch in `derive_cornerstone_Aq_scaled()` -- [x] `bedrock/analysis/a_matrix_time_series/constants.py` — Step N2 updates (`APPROACH_ORDER`, color, `EXTERNAL_REFERENCES`, `APPROACH_YEAR_COVERAGE`) -- [x] `bedrock/analysis/a_matrix_time_series/derive_A_time_series.py` — `APPROACH_YAMLS` + `_years_for()` filter + cache-clearing of the new module -- [x] `bedrock/analysis/a_matrix_time_series/compare_approaches.py` — `USEEIO_NOWCAST_PAIRS` + plot block -- [x] `bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml` — minimal config for A-matrix-only analysis (Steps N1–N3) -- [x] `bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml` — full v0.3 model for EF diagnostics (Step N4 — runs `generate_diagnostics` workflow end-to-end) - -### Data artifacts (under `bedrock/analysis/a_matrix_time_series/output/results/`) -- [ ] `A_useeio_nowcast_{yr}.parquet` for `yr ∈ 2017..2023` -- [ ] Updated `cache_summary.csv` (7 extra rows × `dom`/`imp` = 14) -- [ ] Updated `A_cells_long.parquet` (+ ~1M rows) -- [ ] `step5_a_matrix_rmse_vs_useeio_nowcast.csv` — new ground-truth-anchor diagnostic -- [ ] `useeio_nowcast_MANIFEST.txt` — provenance: upstream branch SHA, report version, GCS upload date - -### Figures (under `output/plots/`) -- [ ] All existing Step 2–5 figures rebuilt with the 6th series visible (`useeio_nowcast` color = `#9467bd`) -- [ ] New `step3_divergence_ecdf_vs_useeio_nowcast.png` — alternatives' divergence against the nowcast (parallel to the existing two ECDF panels against USEEIO-do-nothing and CEDA-US) -- [ ] New `step5_a_matrix_rmse_ranking.png` — bar chart of L1/RMSE per approach × year vs. `useeio_nowcast` (parallel to the existing industry-output RMSE bar chart) - -### Write-up -- [ ] `bedrock/analysis/a_matrix_time_series/README.md` — append a "USEEIO nowcast integration" section summarizing what changed and how to interpret the new series. -- [ ] One-paragraph addition to `analysis_plan.md` listing `useeio_nowcast` in the approach table with its role. - ---- - -## Open decisions before starting - -1. **Year coverage**: drop 2024 from the `useeio_nowcast` line, OR carry-forward 2023 with an "extrapolated" label. **Recommend drop**; the gap is informative. -2. ~~**V-norm year**: 2017 vs year-specific~~ — **resolved in Step N1: use bedrock's 2017 `bea_Vnorm_scrap_corrected()`** for cleanest comparison and to keep one channel of structural change at a time. -3. ~~**Imports treatment**~~ — **resolved: use USEEIO's `U_imports_out_{yr}` directly** rather than re-deriving from a 2017 imports-share assumption. The imports update is one of the things the nowcast meaningfully adds. -4. ~~**GCS bucket**~~ — **resolved: `gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/` (already uploaded).** -5. ~~**Crosswalk for 1→N**~~ — **resolved: bedrock's `expand_square_matrix(zero_intragroup_cross_terms=True)` + `cs_commodity_to_bea_map()` already handle this.** Same as `derive_cornerstone_Aq()`'s non-disagg branch. -6. **Should we run Step 6 (EF diagnostics) with the nowcast at all?** Default: no, unless Step N3 surfaces it as a contender. Re-evaluate after Step N3. -7. **Waste disaggregation flag during the comparison**: turn the comparison runs' `get_waste_disagg_weights()` **off** (so all 6 approaches go through the BEA-detail → Cornerstone expand path) — OR keep it on for the other 5 and mark waste rows non-comparable for `useeio_nowcast`. **Recommend off for the comparison runs**, since the diagnostic value is comparing input-share methodology, not waste disaggregation. -8. ~~**2017 identity tolerance**~~ — **resolved: PARKED.** The 2017 USEEIO_nowcast A does **not** equal bedrock's 2017 BEA A. Empirically (from the smoke test): Adom relative L1 ≈ 9.1%, Aimp relative L1 ≈ 22.1%, with diagonal manufacturing the hot spot (`336500/336500`, `336991/336991`, `333993`, `333994`, `334220/334210`). Probable root cause: USEEIO's R-data `Detail_Use_2017_PRO_AfterRedef` files differ from the BEA XLSX bedrock loads via `load_2017_Utot_usa` (different vintages of the same nominal benchmark). Definitive diagnosis is **parked** — we proceed treating `useeio_nowcast` as an independent reference, not as a strict identity-at-2017 baseline. Document the gap in the README; do not gate Step N1 on it. -9. ~~**Negative cells in intermediate U from GRAS reconciliation**~~ — **resolved: clip to 0 and log magnitude.** ~1854 cells in U_dom for 2023 with summed magnitude ≈ 0.27% of \|U_dom\|; worst offenders are intra-electronics blocks (`334220→336411`, `334220→518200`, `333318→48A000`). These are reconciliation noise, not real negative input demand. The clip + log is implemented in the smoke test; carry it into Step N1 unchanged. -10. ~~**0.98 column-sum cap**~~ — **resolved: apply per matrix** (Adom and Aimp separately) inside Step N1, mirroring `scale_cornerstone_A`'s post-processing. Without the cap, USEEIO's `S00102` column (noncomparable imports) reaches 1.55 — a known edge case the cap already handles for the other approaches. Applying the cap keeps the apples-to-apples comparison across all 6 approaches clean. - ---- - -## Risks / caveats - -- **Methodological overlap with `summary_tables`.** USEEIO's GRAS pipeline conceptually overlaps with bedrock's `summary_tables` method — both use BEA summary SUTs as the constraint and freeze 2017 within-summary technology. Don't claim "USEEIO nowcast validates `summary_tables`" if they align — that's circular. The diagnostic value is the *opposite*: where bedrock's `summary_tables` *disagrees* with `useeio_nowcast`, the difference comes from bedrock-specific schema (Cornerstone disaggregation), the 0.98 column cap, or different aggregation timing — all worth investigating. -- **2017 identity check is approximate, not exact.** USEEIO's 2017 GRAS output ≠ the BEA 2017 benchmark exactly; the rev1 report documents persistent ~$0.1–0.3M deviations at 5412OP, GSLG, 81, 722, 23. Set the tolerance accordingly (1% L1 norm rather than `1e-10` like the other approaches' identity check). -- **Disaggregation may dominate the signal.** If a single BEA Detail code maps to N Cornerstone children, the nowcast carries no information about within-Cornerstone-child structure — all children inherit the parent's coefficients. For sectors with heavy disaggregation (waste, software), the nowcast tells us nothing the existing approaches don't. -- **Crosswalk fragility.** If the existing bedrock BEA-to-Cornerstone mapping isn't a strict refinement (i.e. some Cornerstone codes draw from multiple BEA Detail rows), the row-derivation is more complex than the simple distribute-by-share recipe. Verify direction of mapping early in P1. -- **Sign conventions.** USEEIO tables may include negative cells (inventory changes in FD, taxes/subsidies in VA). The intermediate-Use block is non-negative by construction, so deriving Adom/Aimp from intermediate Use only should produce non-negative A — but double-check on first 2023 run. -- **Imports row-sum identity.** USEEIO enforces `U_imports.sum(axis=1) ≈ 0` (US-tables identity); bedrock's Adom + Aimp split may use a different convention. Reconcile before claiming any disagreement is "real". - ---- - -## Reference files - -| File | Role | -|---|---| -| [/USEEIO_nowcasting.md](../../../USEEIO_nowcasting.md) | Full description of the upstream nowcasting pipeline, what it captures, and limitations | -| [analysis_plan.md](analysis_plan.md) | Parent plan; this plan extends Steps 1–5 | -| [_loaders.py](_loaders.py) | Parquet loader convention to mirror | -| [derive_A_time_series.py](derive_A_time_series.py) | Existing approach driver — useeio_nowcast bypasses this | -| [compare_approaches.py](compare_approaches.py) | Step 3 cross-approach hexbins — gets new PAIRS entries | -| [constants.py](constants.py) | `APPROACH_ORDER`, `APPROACH_COLORS` — extension point | -| `~/Desktop/nowcasting/final nowcasted tables/` | Source CSVs (21 files, 7 years × 3 tables) | -| `~/Desktop/USEEIO_nowcasting_2025_10_15_rev1.docx` | Authoritative method write-up by Wood/Vendries/Young | - ---- - -## Suggested sizing - -| Phase | Effort | Gate | -|---|---|---| -| P0 + P1 (staging + crosswalk) | 0.5–1 day | crosswalk reviewed | -| Step N1 (derive A from nowcast) | 1–2 days | 2017 identity check passes (1% L1 tol) | -| Step N2 + N3 (rewire + re-run downstream) | 1 day | Updated figures reviewed | -| Step N4 (write-up) | 0.5 day | README amended | - -**Total: ~3–4 working days** for the core path, assuming the BEA-Detail→Cornerstone crosswalk already exists in usable form. If P1 needs new disaggregation weights, add 1–2 days. diff --git a/bedrock/analysis/a_matrix_time_series/plot_ef_diagnostics.py b/bedrock/analysis/a_matrix_time_series/plot_ef_diagnostics.py deleted file mode 100644 index 6b9f0c95..00000000 --- a/bedrock/analysis/a_matrix_time_series/plot_ef_diagnostics.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Phase 3 of Step 6: scatter + histogram plots comparing approach EFs vs each baseline. - -Reads ``ef_scatter_coords.parquet`` (Phase 2 output) and produces, for -each ``(scenario, baseline, ef_kind)`` triple, two figures: - -- ``ef_scatter_{scenario}_{baseline}_{ef_kind}.png`` — 2×2 grid of - approach panels. Per panel: linear-scale scatter of ``(x_baseline, - y_approach)`` per Cornerstone sector, ``y=x`` reference dashed in - black. Top-left annotation reports ``R²`` of ``y ~ x`` against the - ``y=x`` line, ``p95`` of ``|y - x| / |x|``, and the count of sectors - with that quotient exceeding ``SIGNIFICANT_PCT_THRESHOLD``. -- ``ef_pct_hist_{scenario}_{baseline}_{ef_kind}.png`` — 2×2 grid of - approach panels. Per panel: per-sector - ``% diff = (y - x) / x × 100`` clipped to ±100%, 60-bin histogram. - Vertical zero line. Mirrors the ``baseline_snapshot_comparison`` - ``compare_B_Adom`` convention. - -The ``scenario`` axis separates runs that **isolate the A-matrix -derivation** (``isolate_a_matrix``: only one A-matrix flag flipped -versus the Cornerstone 2026 schema) from runs that **bundle the -A-matrix change with the full bedrock v0.3 stack** -(``bundle_v0_3``). Pooling them would mix two different counterfactuals -against the same CEDA-US (v0) baseline. - -vs CEDA (v0) panels: 3 candidates (``useeio``, ``summary_tables``, -``commodity_price_index``). -vs USEEIO panels: 2 candidates (``useeio`` is the comparator and is -hidden). - -Usage: - python -m bedrock.analysis.a_matrix_time_series.plot_ef_diagnostics -""" - -from __future__ import annotations - -import logging -import typing as ta - -import matplotlib.pyplot as plt -import pandas as pd -from matplotlib.axes import Axes -from matplotlib.figure import Figure - -from bedrock.analysis.a_matrix_time_series.compile_ef_diagnostics import ( - EF_SCATTER_COORDS_PATH, - SIGNIFICANT_PCT_THRESHOLD, -) -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_COLORS, - PLOTS_DIR, -) -from bedrock.utils.validation.analysis.ef_hist_panels import ( - EF_KIND_LABEL, - HIST_FONT_SCALE, - draw_per_sector_pct_hist_panel, -) - -logger = logging.getLogger(__name__) - -# Last positional arg is a font-scale multiplier so the same panel -# functions can render at default size (scatter) and enlarged size -# (histograms). -PanelFn = ta.Callable[[Axes, pd.DataFrame, str, float], None] - -# EF panels show only the v0.3 focus approaches — summary_tables and -# commodity_price_index are the two top internal candidates, useeio_nowcast -# is the external reference. industry_price_index is dropped (superseded by -# commodity_price_index) and useeio (BEA-2017 do-nothing) is omitted because -# it doesn't vary across years and confuses the time-series story. -# ceda_default is the baseline on the x-axis, not a panel. -APPROACH_ORDER: tuple[str, ...] = ( - "summary_tables", - "commodity_price_index", - "useeio_nowcast", -) -BASELINE_LABEL: dict[str, str] = {"ceda": "CEDA-US (v0)", "useeio": "USEEIO"} -SCENARIO_LABEL: dict[str, str] = { - "isolate_a_matrix": "isolate A-matrix method", - "bundle_v0_3": "A-matrix method bundled with bedrock v0.3", -} - -# Base font sizes (multiplied by ``font_scale`` per panel). -TITLE_FONTSIZE = 20 -AXIS_LABEL_FONTSIZE = 11 -SUPTITLE_FONTSIZE = 14 -STATS_FONTSIZE = 10 -LEGEND_FONTSIZE = 9 -TICK_LABEL_FONTSIZE = 10 - - -def _panel_stats(panel_df: pd.DataFrame) -> dict[str, float]: - """``R²`` against ``y=x``, ``p95`` of ``|y-x|/|x|``, and ``n_significant``. - - ``R²`` is the coefficient of determination of ``y`` against the ``y=x`` - reference (not an OLS fit), so values can go negative when ``y=x`` is - worse than predicting the mean of ``y``. - """ - df = panel_df[panel_df["x_baseline"].abs() > 0] - if len(df) < 2: - return {"r2": float("nan"), "p95": float("nan"), "n_sig": 0.0} - x = df["x_baseline"].to_numpy(dtype=float) - y = df["y_approach"].to_numpy(dtype=float) - ss_tot = float(((y - y.mean()) ** 2).sum()) - ss_res = float(((y - x) ** 2).sum()) - r2 = 1 - (ss_res / ss_tot) if ss_tot > 0 else float("nan") - perc = (df["y_approach"] - df["x_baseline"]).abs() / df["x_baseline"].abs() - return { - "r2": float(r2), - "p95": float(perc.quantile(0.95)), - "n_sig": float((perc > SIGNIFICANT_PCT_THRESHOLD).sum()), - } - - -def _scatter_panel( - ax: Axes, panel_df: pd.DataFrame, approach: str, font_scale: float -) -> None: - color = APPROACH_COLORS.get(approach, "#000000") - sub = panel_df.dropna(subset=["x_baseline", "y_approach"]) - ax.scatter( - sub["x_baseline"], - sub["y_approach"], - marker="o", - color=color, - s=18, - alpha=0.55, - linewidths=0, - label=f"n={len(sub)}", - ) - if not sub.empty: - lo = float(min(sub["x_baseline"].min(), sub["y_approach"].min())) - hi = float(max(sub["x_baseline"].max(), sub["y_approach"].max())) - ax.plot( - [lo, hi], [lo, hi], color="black", linestyle="--", linewidth=0.7, alpha=0.5 - ) - - ax.grid(True, which="both", alpha=0.2) - - stats = _panel_stats(panel_df) - ax.text( - 0.04, - 0.96, - f"R²={stats['r2']:.3f}\np95={stats['p95']:.3f}\nn_sig={int(stats['n_sig'])}", - transform=ax.transAxes, - fontsize=STATS_FONTSIZE * font_scale, - va="top", - ha="left", - bbox=dict( - boxstyle="round,pad=0.3", facecolor="white", alpha=0.4, edgecolor="0.7" - ), - ) - ax.set_title(approach, fontsize=TITLE_FONTSIZE * font_scale, color="black") - ax.legend(loc="lower right", fontsize=LEGEND_FONTSIZE * font_scale, framealpha=0.4) - - -def _hist_panel( - ax: Axes, panel_df: pd.DataFrame, approach: str, font_scale: float -) -> None: - color = APPROACH_COLORS.get(approach, "tab:blue") - df = panel_df[panel_df["x_baseline"].abs() > 0].dropna( - subset=["x_baseline", "y_approach"] - ) - if df.empty: - ax.text(0.5, 0.5, "no data", transform=ax.transAxes, ha="center", va="center") - ax.set_title(approach, fontsize=TITLE_FONTSIZE * font_scale, color="black") - return - pct_fraction = ( - (df["y_approach"] - df["x_baseline"]) / df["x_baseline"].abs() - ).to_numpy(dtype=float) - draw_per_sector_pct_hist_panel( - ax, - pct_fraction, - title=approach, - color=color, - font_scale=font_scale, - ylabel="", - ) - - -def _latest_year_in(sub: pd.DataFrame) -> str: - """Return the latest year string in ``sub`` or '' if none populated. - - Year stored as the original ``year`` string from the run index (e.g. - ``"2023.0"`` for time-series cells, ``""`` for legacy single-year - cells). When a scenario has multiple years pooling them is rarely - what we want — picking the latest gives a single-year snapshot per - panel, matching the convention the histograms were designed for. - """ - years = [y for y in sub["year"].dropna().astype(str).unique() if y] - if not years: - return "" - return max(years, key=lambda y: float(y)) - - -def _grid_2x2( - coords: pd.DataFrame, - scenario: str, - baseline: str, - ef_kind: str, - panel_fn: PanelFn, - xlabel: str, - ylabel: str, - suptitle: str, - font_scale: float = 1.0, -) -> Figure | None: - sub = coords[ - (coords["scenario"] == scenario) - & (coords["baseline"] == baseline) - & (coords["ef_kind"] == ef_kind) - ] - approaches = [a for a in APPROACH_ORDER if a in sub["approach"].unique()] - if not approaches: - logger.warning( - "No approaches for scenario=%s baseline=%s ef_kind=%s", - scenario, - baseline, - ef_kind, - ) - return None - # When a scenario carries multiple years, pool would mix dollar-years - # and inflate `n` per panel — restrict to the latest year so each - # panel is a single-year snapshot. Legacy single-year runs predate the - # `year` column and store it as empty string; treat those as - # participating in the latest year (otherwise approaches without a - # tagged year would render "no data" while sibling approaches' year- - # tagged rows take over the panel). - latest_year = _latest_year_in(sub) - if latest_year: - year_str = sub["year"].fillna("").astype(str) - sub = sub[(year_str == latest_year) | (year_str == "")] - suptitle = f"{suptitle} — year {int(float(latest_year))}" - - # Layout: 1×N for N ≤ 3 (typical focus comparison), else 2×ceil(N/2) - # so the canvas stays roughly square as more approaches are added. - n = len(approaches) - if n <= 3: - nrows, ncols = 1, n - base_w, base_h = 5.5 * ncols, 6.0 - else: - nrows = 2 - ncols = (n + 1) // 2 - base_w, base_h = 5.5 * ncols, 5.5 * nrows - - fig, axes_grid = plt.subplots( - nrows, ncols, figsize=(base_w * font_scale, base_h * font_scale), squeeze=False - ) - flat_axes = list(axes_grid.flat) - for ax, approach in zip(flat_axes, approaches): - panel_fn(ax, sub[sub["approach"] == approach], approach, font_scale) - ax.tick_params(axis="both", labelsize=TICK_LABEL_FONTSIZE * font_scale) - for ax in flat_axes[len(approaches) :]: - ax.axis("off") - - for ax in axes_grid[:, 0]: - ax.set_ylabel(ylabel, fontsize=AXIS_LABEL_FONTSIZE * font_scale) - for ax in axes_grid[-1, :]: - ax.set_xlabel(xlabel, fontsize=AXIS_LABEL_FONTSIZE * font_scale) - - fig.suptitle(suptitle, fontsize=SUPTITLE_FONTSIZE * font_scale, y=1.0) - fig.tight_layout() - return fig - - -def main() -> None: - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - if not EF_SCATTER_COORDS_PATH.exists(): - raise FileNotFoundError( - f"{EF_SCATTER_COORDS_PATH} not found — run " - "`compile_ef_diagnostics` first." - ) - coords = pd.read_parquet(EF_SCATTER_COORDS_PATH) - if "scenario" not in coords.columns: - raise ValueError( - f"{EF_SCATTER_COORDS_PATH} is missing the `scenario` column — " - "rebuild with the current `compile_ef_diagnostics`." - ) - - scenarios = [s for s in coords["scenario"].dropna().unique() if s] - for scenario in scenarios: - scen_label = SCENARIO_LABEL.get(scenario, scenario) - for baseline in ("ceda", "useeio"): - for ef_kind in ("N", "D"): - kind_label = EF_KIND_LABEL[ef_kind] - base_label = BASELINE_LABEL[baseline] - - scatter_fig = _grid_2x2( - coords, - scenario, - baseline, - ef_kind, - _scatter_panel, - xlabel=f"{base_label} {kind_label} (inflation-adjusted)", - ylabel=f"approach {kind_label}", - suptitle=( - f"{kind_label} per sector — approach vs {base_label} " - f"[{scen_label}]" - ), - ) - if scatter_fig is not None: - out = PLOTS_DIR / f"ef_scatter_{scenario}_{baseline}_{ef_kind}.png" - scatter_fig.savefig(out, dpi=150, bbox_inches="tight") - plt.close(scatter_fig) - logger.info("Wrote %s", out) - - hist_fig = _grid_2x2( - coords, - scenario, - baseline, - ef_kind, - _hist_panel, - xlabel="Percentage Diff (%)", - ylabel="sector count", - suptitle=( - f"{kind_label} per-sector % diff distribution — " - f"vs {base_label} [{scen_label}]" - ), - font_scale=HIST_FONT_SCALE, - ) - if hist_fig is not None: - out = PLOTS_DIR / f"ef_pct_hist_{scenario}_{baseline}_{ef_kind}.png" - hist_fig.savefig(out, dpi=150, bbox_inches="tight") - plt.close(hist_fig) - logger.info("Wrote %s", out) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format="%(message)s") - main() diff --git a/bedrock/analysis/a_matrix_time_series/plot_v0_3_n_pct_hist.py b/bedrock/analysis/a_matrix_time_series/plot_v0_3_n_pct_hist.py deleted file mode 100644 index d1d17ce3..00000000 --- a/bedrock/analysis/a_matrix_time_series/plot_v0_3_n_pct_hist.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Render per-sector N + D % diff histograms from a single diagnostics Sheet. - -The reference 2×2 plot from ``plot_ef_diagnostics`` (e.g. -``ef_pct_hist_bundle_v0_3_ceda_N.png``) panels four A-matrix methods side -by side. Each panel is one run's ``N_and_diffs`` / ``D_and_diffs`` tab vs -the CEDA-US (v0) baseline. This script renders the same panel style for a -*single* diagnostics Sheet — useful when the run you want to inspect lives -outside the dispatched ``ef_run_index.csv`` set (e.g. a one-off -investigation sheet, an ad-hoc config variation). - -The default Sheet ID points at the v0.3 "all changes" diagnostics run; pass -a different ID on the command line to render any other run. - -Source schema: each tab must contain ``{kind}_perc_diff`` (the script also -accepts ``{kind}_new`` / ``{kind}_old_inflated`` as a fallback), where -``kind`` is ``N`` (total EF) or ``D`` (direct EF). Percent-formatted cells -(``"14.2%"``) are coerced automatically. - -The approach label (used in the title and panel color) is derived from the -``config_summary`` tab — whichever of the three A-matrix scaling flags is -``TRUE`` wins; if all three are ``FALSE`` the run is labeled -``cornerstone_default`` (price-index-free default Cornerstone path). - -Outputs: -- ``output/plots/ef_pct_hist_v0_3_sheet_ceda_N.png`` -- ``output/plots/ef_pct_hist_v0_3_sheet_ceda_D.png`` - -Usage: - python -m bedrock.analysis.a_matrix_time_series.plot_v0_3_n_pct_hist - python -m bedrock.analysis.a_matrix_time_series.plot_v0_3_n_pct_hist -""" - -from __future__ import annotations - -import logging -import sys - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.analysis.a_matrix_time_series.constants import ( - APPROACH_COLORS, - PLOTS_DIR, -) -from bedrock.utils.io.gcp import read_sheet_tab -from bedrock.utils.validation.analysis.ef_hist_panels import ( - EF_KIND_LABEL, - HIST_FONT_SCALE, - PANEL_SUPTITLE_FONTSIZE, - draw_per_sector_pct_hist_panel, - pct_values, -) - -logger = logging.getLogger(__name__) - -DEFAULT_SHEET_ID = "1pCSgLD14lmrQg3OtfHvnK4lQrFtqiavrjCt-C3R_bSw" -CONFIG_TAB = "config_summary" - -_KIND_TAB = {"N": "N_and_diffs", "D": "D_and_diffs"} - -# Map config_summary flag → display approach name. The flag set in the -# v0.3 config_summary mirrors the YAML names referenced in epic #337. -_FLAG_TO_APPROACH: tuple[tuple[str, str], ...] = ( - ("scale_a_matrix_with_useeio_method", "useeio"), - ("scale_a_matrix_with_summary_tables", "summary_tables"), - ("scale_a_matrix_with_price_index", "price_index"), -) -_DEFAULT_APPROACH_LABEL = "cornerstone_default" - - -def _output_path(ef_kind: str) -> object: - return PLOTS_DIR / f"ef_pct_hist_v0_3_sheet_ceda_{ef_kind}.png" - - -def _approach_from_config(config_df: pd.DataFrame) -> str: - """Pick a label for this run from the ``config_summary`` flags. - - Returns the first flag that is ``TRUE``; falls back to - ``cornerstone_default`` when no scaling flag is enabled (the - price-index-free Cornerstone default path). - """ - if config_df.empty: - return _DEFAULT_APPROACH_LABEL - flags = ( - config_df.set_index("config_field")["value"].astype(str).str.strip().str.upper() - ) - for flag, label in _FLAG_TO_APPROACH: - if flags.get(flag) == "TRUE": - return label - return _DEFAULT_APPROACH_LABEL - - -def _render( - pct: np.ndarray, - approach: str, - sheet_id: str, - ef_kind: str, - out_path: str, -) -> None: - """Render a single-panel histogram matching the per-panel style of - ``ef_pct_hist_*`` figures. - - Font scale, bins, clipping, stats-box layout, and percent x-axis match - ``ef_hist_panels.draw_per_sector_pct_hist_panel`` so this output reads as - one of the four panels in the reference bundle plot. - """ - # Fall back to grey (the useeio palette entry) for unknown approaches like - # ``cornerstone_default`` — visually signals "no scaling flag set". - color = APPROACH_COLORS.get(approach, "#7f7f7f") - kind_label = EF_KIND_LABEL[ef_kind] - font_scale = HIST_FONT_SCALE - - fig, ax = plt.subplots(figsize=(11.0 * font_scale * 0.6, 10.5 * font_scale * 0.6)) - draw_per_sector_pct_hist_panel( - ax, - pct, - title=approach, - color=color, - font_scale=font_scale, - ) - - fig.suptitle( - f"{kind_label} per-sector % diff distribution — vs CEDA-US (v0) " - f"[A-matrix method bundled with bedrock v0.3] — sheet {sheet_id[:10]}…", - fontsize=PANEL_SUPTITLE_FONTSIZE * font_scale, - y=1.0, - ) - fig.tight_layout() - fig.savefig(out_path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def _read_config(sheet_id: str) -> pd.DataFrame: - try: - return read_sheet_tab(sheet_id, CONFIG_TAB) - except Exception as e: - logger.warning( - "Could not read %r from sheet %s (%s); using default approach label.", - CONFIG_TAB, - sheet_id, - e, - ) - return pd.DataFrame({"config_field": [], "value": []}) - - -def main(sheet_id: str = DEFAULT_SHEET_ID) -> None: - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - approach = _approach_from_config(_read_config(sheet_id)) - logger.info("Resolved approach=%r for sheet=%s", approach, sheet_id) - - for ef_kind in ("N", "D"): - tab = _KIND_TAB[ef_kind] - df = read_sheet_tab(sheet_id, tab) - if df.empty: - logger.warning("Tab %r in sheet %s is empty; skipping.", tab, sheet_id) - continue - pct = pct_values(df, ef_kind) - if pct.size == 0: - logger.warning( - "No finite pct values in %r of sheet %s; skipping.", tab, sheet_id - ) - continue - out_path = _output_path(ef_kind) - _render( - pct, - approach=approach, - sheet_id=sheet_id, - ef_kind=ef_kind, - out_path=str(out_path), - ) - logger.info("Wrote %s", out_path) - print(f"Wrote: {out_path}") - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format="%(message)s") - sheet_id = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SHEET_ID - main(sheet_id) diff --git a/bedrock/analysis/a_matrix_time_series/recover_ef_run_index.py b/bedrock/analysis/a_matrix_time_series/recover_ef_run_index.py deleted file mode 100644 index 9377d1c6..00000000 --- a/bedrock/analysis/a_matrix_time_series/recover_ef_run_index.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Reconstruct ``ef_run_index.csv`` from existing Sheets in a Drive folder. - -For users who triggered EF diagnostics manually via the GH Actions workflow -(rather than via ``dispatch_ef_time_series.py``), this script lists every -Google Sheet in the diagnostics Drive folder, parses each title to recover -the run dimensions, and writes ``ef_run_index.csv`` with one row per Sheet. - -Sheet title formats supported: -- Manual single-cell: - ``[YYYY-MM-DD, BASELINE based, A matrix with APPROACH] EFs diagnostics`` -- Time-series dispatch (``dispatch_ef_time_series.py``): - ``[YYYY-MM-DD, YYYY, BASELINE based, A matrix with APPROACH, SCENARIO] EFs diagnostics`` - -The approach phrase is "A matrix with APPROACH" for the in-repo derivations and -"A matrix from USEEIO nowcast" for the external nowcast reference, so the parser -accepts both "with" and "from" (see ``APPROACH_LABELS`` in -``dispatch_ef_time_series.py``). Titles that don't match this shape (e.g. ad-hoc -``[..., mecs year updates] EFs diagnostics`` runs) are intentionally skipped. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.recover_ef_run_index \\ - --folder-id -""" - -from __future__ import annotations - -import argparse -import logging -import re -import typing as ta - -import google.auth -import pandas as pd -from googleapiclient.discovery import build - -from bedrock.analysis.a_matrix_time_series.constants import RESULTS_DIR - -logger = logging.getLogger(__name__) - -EF_RUN_INDEX_PATH = RESULTS_DIR / "ef_run_index.csv" - -# Output columns match what compile_ef_diagnostics.py + dispatch script -# already use; required = approach, baseline, sheet_id. -INDEX_COLUMNS: tuple[str, ...] = ( - "approach", - "baseline", - "sheet_id", - "sheet_title", - "created_at", - "scenario", - "year", -) - -# Title regex: -# prefix '[' then 1 or 2 leading date-like tokens (date + optional year) -# then 'BASELINE based, A matrix with/from APPROACH' and optional trailing -# ', SCENARIO' then '] EFs diagnostics'. "with" covers the in-repo -# derivations; "from" covers the external "A matrix from USEEIO nowcast". -TITLE_RE = re.compile( - r""" - ^\[ - (?P\d{4}-\d{2}-\d{2}) # YYYY-MM-DD - (?:,\s*(?P\d{4}))? # optional ', YYYY' (time-series cell) - ,\s*(?P[A-Za-z]+)\sbased - ,\s*A\smatrix\s(?:with|from)\s(?P[^,\]]+) - (?:,\s*(?P[^\]]+))? # optional ', SCENARIO' - \]\sEFs\sdiagnostics$ - """, - re.VERBOSE, -) - -# Maps the human-readable approach phrase to the canonical config key used -# in ef_run_index.csv. -APPROACH_BY_TEXT: dict[str, str] = { - "USEEIO method": "useeio", - "2017 benchmark A": "useeio", - "summary tables": "summary_tables", - "commodity price index": "commodity_price_index", - "USEEIO nowcast": "useeio_nowcast", -} - -BASELINE_BY_TEXT: dict[str, str] = { - "CEDA": "ceda", - "USEEIO": "useeio", -} - - -def _drive_client() -> ta.Any: - """Build a read-only Drive API client.""" - credentials, _ = google.auth.default( - scopes=["https://www.googleapis.com/auth/drive.readonly"] - ) - return build("drive", "v3", credentials=credentials) - - -def _list_sheets_in_folder(folder_id: str) -> list[dict[str, str]]: - """List Google Sheets in the given Drive folder (non-trashed only). - - Returns one dict per Sheet with keys ``id``, ``name``, ``createdTime``. - """ - if not folder_id: - raise ValueError("folder_id is required") - client = _drive_client() - query = ( - f"'{folder_id}' in parents and trashed=false " - "and mimeType='application/vnd.google-apps.spreadsheet'" - ) - files: list[dict[str, str]] = [] - page_token: str | None = None - while True: - resp = ( - client.files() - .list( - q=query, - fields="nextPageToken, files(id, name, createdTime)", - pageSize=1000, - pageToken=page_token, - supportsAllDrives=True, - includeItemsFromAllDrives=True, - ) - .execute() - ) - files.extend(resp.get("files", [])) - page_token = resp.get("nextPageToken") - if not page_token: - break - return files - - -def _parse_title(title: str) -> dict[str, str] | None: - """Parse a Sheet title into run-index columns. Returns None if unparseable.""" - m = TITLE_RE.match(title) - if m is None: - return None - approach_text = m.group("approach_text").strip() - approach = APPROACH_BY_TEXT.get(approach_text) - if approach is None: - logger.warning("Unrecognized approach %r in title %r", approach_text, title) - return None - baseline_text = m.group("baseline").upper() - baseline = BASELINE_BY_TEXT.get(baseline_text) - if baseline is None: - logger.warning("Unrecognized baseline %r in title %r", baseline_text, title) - return None - return { - "approach": approach, - "baseline": baseline, - "sheet_title": title, - "scenario": (m.group("scenario") or "").strip(), - "year": m.group("year") or "", - } - - -def rebuild_index(folder_id: str, output_path: str | None = None) -> pd.DataFrame: - """Build ``ef_run_index.csv`` from Sheet titles in ``folder_id``. - - Returns the resulting DataFrame and writes it to ``output_path`` - (default: ``output/results/ef_run_index.csv``). - """ - out = output_path or str(EF_RUN_INDEX_PATH) - files = _list_sheets_in_folder(folder_id) - logger.info("Found %d Sheet(s) in folder %s", len(files), folder_id) - - rows: list[dict[str, str]] = [] - skipped: list[str] = [] - for f in files: - parsed = _parse_title(f["name"]) - if parsed is None: - skipped.append(f["name"]) - continue - rows.append( - { - "approach": parsed["approach"], - "baseline": parsed["baseline"], - "sheet_id": f["id"], - "sheet_title": parsed["sheet_title"], - "created_at": f.get("createdTime", ""), - "scenario": parsed["scenario"], - "year": parsed["year"], - } - ) - - df = pd.DataFrame(rows, columns=pd.Index(INDEX_COLUMNS)) - df = df.sort_values(["scenario", "approach", "year", "baseline"]).reset_index( - drop=True - ) - EF_RUN_INDEX_PATH.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(out, index=False) - logger.info("Wrote %d row(s) to %s", len(df), out) - if skipped: - logger.warning( - "Skipped %d Sheet(s) with unparseable titles: %s", - len(skipped), - ", ".join(repr(t) for t in skipped[:5]) - + (" ..." if len(skipped) > 5 else ""), - ) - return df - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--folder-id", - required=True, - help=( - "Drive folder ID containing the diagnostics Sheets. Find it in " - "the Drive folder URL: drive.google.com/drive/folders/." - ), - ) - parser.add_argument( - "--output", - default=str(EF_RUN_INDEX_PATH), - help=f"Output CSV path (default: {EF_RUN_INDEX_PATH}).", - ) - args = parser.parse_args() - logging.basicConfig(level=logging.INFO, format="%(message)s") - rebuild_index(args.folder_id, args.output) - - -if __name__ == "__main__": - main() diff --git a/bedrock/analysis/a_matrix_time_series/summary_a_and_q_errors.py b/bedrock/analysis/a_matrix_time_series/summary_a_and_q_errors.py deleted file mode 100644 index 413d2c2d..00000000 --- a/bedrock/analysis/a_matrix_time_series/summary_a_and_q_errors.py +++ /dev/null @@ -1,498 +0,0 @@ -"""Step 5 of epic #337 (Option 2 refined): cell-level A comparison at BEA -summary aggregation, ranked by weighted RMSE per (approach, year). - -The original Step 5 spec — comparing the model's saved ``q`` vector to BEA -gross output — turned out not to test A: ``q`` is produced by -``scale_cornerstone_q`` + ``inflate_cornerstone_q_or_y``, helpers that run -in parallel to A and never depend on it. - -The first reframing (Option 2) tried to compare ``col_sum(A)`` to BEA -observed II/GO share per industry. That ran into a data wall (annual II -isn't in the codebase at detail level) AND was column-sum-only — a cell -with the wrong row distribution but right column total would pass. - -This module ships **Option 2 refined**: it uses BEA's *summary-level* A -matrix, available annually via ``derive_summary_Adom_usa(year)`` + -``derive_summary_Aimp_usa(year)``, as ground truth. For each -(approach, year) it aggregates the Cornerstone detail A to summary level -via observed dollar-flow weighting (``Z = A · diag(q)`` then groupby -summary parent, then divide by aggregated q) and compares to the BEA -observed summary A cell by cell. Tests A's row distribution AND column -structure at summary granularity — column-sum caveat resolved. - -See [issue #344 reframing comment](https://github.com/cornerstone-data/bedrock/issues/344#issuecomment-4357342817). - -Reads the ``A_{approach}_{year}.parquet`` and ``q_{approach}_{year}.parquet`` -caches from Step 1 and the BEA summary A from -``bedrock.transform.eeio.derived_2017``. Produces: - -- ``summary_a_errors.csv`` — one row per (approach, year, dom_or_imp) with - ``rmse_vs_bea_summary_a`` (Z-magnitude weighted), ``mean_abs_diff``, - ``top_5_worst_cells`` (semicolon-joined ``ROW->COL:diff`` pairs). - -- ``q_deviation_summary_tables_vs_cpi.csv`` — one row per year with three - deviation statistics of ``q_summary_tables`` relative to - ``q_commodity_price_index`` across all Cornerstone commodities: - ``q_weighted_rmse`` (q_cpi-magnitude-weighted), ``mean_abs_pct_err``, and - ``median_abs_pct_err`` (percentage units). - -- ``summary_a_rmse_ranking.png`` — grouped bar chart, x = year, five bars - per group (one per approach), y = weighted RMSE for the combined - (dom + imp) A matrix. - -- ``q_commodity_pi_vs_summary_tables.png`` — 8-panel log-log scatter (one - subplot per year) comparing ``q_summary_tables`` (y) against - ``q_commodity_price_index`` (x) for all Cornerstone commodities. The - dashed identity line marks perfect agreement; deviation reveals where - the two q-scaling approaches diverge at the commodity level. - -- Sheet tab ``summary_a_errors`` appended to the run-report Sheet. - -**Caveat:** weighting and aggregation use each approach's own ``q`` -vector (from the parquet cache). Approaches with poorer q-scaling get a -slightly biased weight. A future refinement would use BEA observed -detail GO as the common weight — pending a Cornerstone-detail-to-BEA- -detail GO split for sectors where Cornerstone disaggregates. - -Usage: - python -m bedrock.analysis.a_matrix_time_series.summary_a_errors -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd - -from bedrock.transform.eeio.derived_2017 import ( - derive_summary_Adom_usa, - derive_summary_Aimp_usa, -) -from bedrock.utils.io.gcp import update_sheet_tab -from bedrock.utils.taxonomy.bea_v2017_to_ceda_v7_helpers import ( - load_bea_v2017_summary_to_cornerstone, -) -from bedrock.utils.taxonomy.cornerstone.commodities import COMMODITIES - -CORNERSTONE_COMMODITIES: list[str] = list(COMMODITIES) - -logger = logging.getLogger(__name__) - -OUTPUT_DIR = Path(__file__).parent / "output" -RESULTS_DIR = OUTPUT_DIR / "results" -PLOTS_DIR = OUTPUT_DIR / "plots" -LAST_RUN_SHEET_ID_PATH = RESULTS_DIR / "last_run_sheet_id.txt" - -YEARS: tuple[int, ...] = (2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024) -APPROACH_ORDER: tuple[str, ...] = ( - "useeio", - "ceda_default", - "summary_tables", - "industry_price_index", - "commodity_price_index", -) -APPROACH_COLORS: dict[str, str] = { - "useeio": "#7f7f7f", - "ceda_default": "#bcbd22", - "summary_tables": "#1f77b4", - "industry_price_index": "#ff7f0e", - "commodity_price_index": "#2ca02c", -} - -TOP_K_WORST = 5 - - -def _cornerstone_to_summary() -> dict[str, str]: - """Invert ``load_bea_v2017_summary_to_cornerstone`` into a - cornerstone_code → bea_summary_code lookup.""" - summary_to_cornerstone = load_bea_v2017_summary_to_cornerstone() - return { - code: str(summary) - for summary, codes in summary_to_cornerstone.items() - for code in codes - } - - -def _load_a_total(approach: str, year: int) -> pd.DataFrame: - """A_total = A_dom + A_imp, str-typed indices.""" - combined = pd.read_parquet(RESULTS_DIR / f"A_{approach}_{year}.parquet") - adom = pd.DataFrame(combined.loc["dom"]) - aimp = pd.DataFrame(combined.loc["imp"]) - a = adom.add(aimp, fill_value=0.0) - a.index = a.index.astype(str) - a.columns = a.columns.astype(str) - return a - - -def _load_q_detail(approach: str, year: int) -> pd.Series: - df = pd.read_parquet(RESULTS_DIR / f"q_{approach}_{year}.parquet") - series = pd.Series(df["q"].astype(float)) - series.index = series.index.astype(str) - return series - - -def aggregate_detail_a_to_summary( - a_detail: pd.DataFrame, q_detail: pd.Series, cs_to_summary: dict[str, str] -) -> pd.DataFrame: - """Aggregate detail-level A to summary level via dollar-flow weighting. - - Method: ``Z_detail[i,j] = A_detail[i,j] · q_detail[j]`` (commodity i - used to make commodity j, in $). Group rows and columns by their BEA - summary parent, sum, and divide by aggregated ``q`` to get summary A - coefficients. This is the same construction BEA uses to derive - summary A from a detail Use table. - """ - common_idx = a_detail.index.intersection(q_detail.index) - a_aligned = a_detail.loc[common_idx, common_idx] - q_aligned = q_detail.reindex(common_idx).fillna(0.0) - - z = a_aligned.multiply(q_aligned.to_numpy(), axis=1) - cs_to_sum = {k: v for k, v in cs_to_summary.items() if k in common_idx} - detail_to_summary_index = pd.Index( - [cs_to_sum.get(c, "UNMAPPED") for c in common_idx] - ) - - z.index = detail_to_summary_index - z.columns = detail_to_summary_index - z_summary = z.groupby(level=0).sum().T.groupby(level=0).sum().T - - q_summary = pd.Series(q_aligned.to_numpy(), index=detail_to_summary_index) - q_summary = q_summary.groupby(level=0).sum() - - safe_q = q_summary.replace(0, np.nan) - a_summary = z_summary.divide(safe_q, axis=1) - a_summary = a_summary.fillna(0.0) - a_summary = a_summary.drop(index="UNMAPPED", errors="ignore").drop( - columns="UNMAPPED", errors="ignore" - ) - return a_summary - - -def _bea_observed_summary_a(year: int) -> pd.DataFrame: - """BEA observed summary A = ``A_dom_summary + A_imp_summary`` at year.""" - return derive_summary_Adom_usa(year).add( - derive_summary_Aimp_usa(year), fill_value=0.0 - ) - - -def _cell_errors_one_pair( - a_pred: pd.DataFrame, a_obs: pd.DataFrame -) -> tuple[float, float, list[tuple[str, str, float]]]: - """Per (approach, year) cell-level summary stats. - - Both inputs are summary-level A. Returns: - - Z-magnitude-weighted RMSE (cells with bigger flows dominate) - - mean absolute diff (uniform-weighted) - - top-K worst cells by signed difference - """ - common_rows = a_pred.index.intersection(a_obs.index) - common_cols = a_pred.columns.intersection(a_obs.columns) - pred = a_pred.loc[common_rows, common_cols].to_numpy() - obs = a_obs.loc[common_rows, common_cols].to_numpy() - diff = pred - obs - - abs_obs = np.abs(obs) - weight_sum = abs_obs.sum() - weights = ( - abs_obs / weight_sum if weight_sum > 0 else np.ones_like(abs_obs) / abs_obs.size - ) - rmse = float(np.sqrt(np.sum(weights * diff**2))) - mean_abs = float(np.abs(diff).mean()) - - flat = diff.flatten() - flat_abs = np.abs(flat) - worst = np.argsort(flat_abs)[::-1][:TOP_K_WORST] - rows_arr = np.asarray(common_rows) - cols_arr = np.asarray(common_cols) - n_cols = len(common_cols) - worst_cells = [ - (str(rows_arr[i // n_cols]), str(cols_arr[i % n_cols]), float(flat[i])) - for i in worst - ] - return rmse, mean_abs, worst_cells - - -def compute_errors_table() -> pd.DataFrame: - """Per (approach, year) cell-level errors at summary aggregation.""" - cs_to_summary = _cornerstone_to_summary() - rows: list[dict[str, object]] = [] - for year in YEARS: - try: - a_summary_obs = _bea_observed_summary_a(year) - except Exception as e: # noqa: BLE001 - logger.warning("BEA summary A unavailable for year=%d (%s)", year, e) - continue - for approach in APPROACH_ORDER: - a_path = RESULTS_DIR / f"A_{approach}_{year}.parquet" - q_path = RESULTS_DIR / f"q_{approach}_{year}.parquet" - if not (a_path.exists() and q_path.exists()): - logger.warning("Missing %s or %s — skipping", a_path, q_path) - continue - a_detail = _load_a_total(approach, year) - q_detail = _load_q_detail(approach, year) - a_summary_pred = aggregate_detail_a_to_summary( - a_detail, q_detail, cs_to_summary - ) - rmse, mean_abs, worst_cells = _cell_errors_one_pair( - a_summary_pred, a_summary_obs - ) - rows.append( - { - "approach": approach, - "year": year, - "rmse_vs_bea_summary_a": rmse, - "mean_abs_diff": mean_abs, - "n_summary_cells": int( - a_summary_pred.shape[0] * a_summary_pred.shape[1] - ), - "top_5_worst_cells": "; ".join( - f"{r}->{c}:{d:+.3f}" for r, c, d in worst_cells - ), - } - ) - return pd.DataFrame(rows) - - -def compute_q_deviation_table() -> pd.DataFrame: - """Per-year deviation of q_summary_tables from q_commodity_price_index. - - Three statistics per year, all over the full set of CORNERSTONE_COMMODITIES - present in both parquets: - - - ``q_weighted_rmse`` — q_cpi-magnitude-weighted RMSE, analogous to the - Z-weighted RMSE used for A-matrix errors. Large-q commodities dominate. - - ``mean_abs_pct_err`` — mean |q_st − q_cpi| / q_cpi × 100 (uniform weight, - percentage units). - - ``median_abs_pct_err`` — median of the same per-commodity ratio × 100. - More robust to a handful of extreme deviants. - - ``n_commodities`` — number of commodities included (positive in both series). - """ - rows: list[dict[str, object]] = [] - for year in YEARS: - cpi_path = RESULTS_DIR / f"q_commodity_price_index_{year}.parquet" - st_path = RESULTS_DIR / f"q_summary_tables_{year}.parquet" - if not (cpi_path.exists() and st_path.exists()): - logger.warning("Missing q parquet for year=%d — skipping q deviation", year) - continue - - q_cpi = _load_q_detail("commodity_price_index", year) - q_st = _load_q_detail("summary_tables", year) - - common = ( - pd.Index(CORNERSTONE_COMMODITIES) - .intersection(q_cpi.index) - .intersection(q_st.index) - ) - cpi_vals = q_cpi.reindex(common).to_numpy(dtype=float) - st_vals = q_st.reindex(common).to_numpy(dtype=float) - - mask = (cpi_vals > 0) & (st_vals > 0) - cpi_vals = cpi_vals[mask] - st_vals = st_vals[mask] - - diff = st_vals - cpi_vals - weights = cpi_vals / cpi_vals.sum() - q_weighted_rmse = float(np.sqrt(np.sum(weights * diff**2))) - abs_pct = np.abs(diff / cpi_vals) * 100.0 - rows.append( - { - "year": year, - "q_weighted_rmse": q_weighted_rmse, - "mean_abs_pct_err": float(abs_pct.mean()), - "median_abs_pct_err": float(np.median(abs_pct)), - "n_commodities": int(mask.sum()), - } - ) - return pd.DataFrame(rows) - - -def plot_rmse_ranking(errors_df: pd.DataFrame, path: Path) -> None: - """Grouped bar chart: x = year, 5 bars per group (one per approach). - - y = Z-magnitude-weighted cell-level RMSE between predicted summary A - (Cornerstone aggregated) and observed summary A (BEA). Tests A's full - cell shape at summary aggregation, not just column sums. - """ - pivot = errors_df.pivot_table( - index="year", columns="approach", values="rmse_vs_bea_summary_a" - ).reindex(columns=APPROACH_ORDER) - - n_years = len(pivot.index) - n_approaches = len(APPROACH_ORDER) - bar_w = 0.8 / n_approaches - x = np.arange(n_years) - - fig, ax = plt.subplots(figsize=(1.4 * n_years + 2, 5)) - fig.suptitle( - "Cell-level RMSE of Cornerstone A vs BEA observed summary A — by approach", - fontsize=11, - ) - - for i, approach in enumerate(APPROACH_ORDER): - if approach not in pivot.columns: - continue - offset = (i - (n_approaches - 1) / 2) * bar_w - vals = pivot[approach].to_numpy() - ax.bar( - x + offset, - vals, - width=bar_w, - color=APPROACH_COLORS[approach], - label=approach, - edgecolor="white", - linewidth=0.4, - ) - - ax.set_xticks(x) - ax.set_xticklabels([str(y) for y in pivot.index]) - ax.set_xlabel("year") - ax.set_ylabel("Z-weighted RMSE") - ax.grid(True, axis="y", alpha=0.3) - ax.legend(loc="upper left", fontsize=9, frameon=False) - - fig.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def _publish_summary_a_errors_tab(errors_df: pd.DataFrame) -> None: - if not LAST_RUN_SHEET_ID_PATH.exists(): - logger.warning("No %s found — skipping Sheet publish.", LAST_RUN_SHEET_ID_PATH) - return - sheet_id = LAST_RUN_SHEET_ID_PATH.read_text().strip() - try: - update_sheet_tab(sheet_id, "summary_a_errors", errors_df) - except Exception as e: # noqa: BLE001 - logger.warning( - "Sheet publish skipped (%s: %s). Local artifacts still complete.", - type(e).__name__, - e, - ) - return - logger.info("Updated summary_a_errors tab on sheet %s", sheet_id) - - -def _publish_q_deviation_tab(q_dev_df: pd.DataFrame) -> None: - if not LAST_RUN_SHEET_ID_PATH.exists(): - return - sheet_id = LAST_RUN_SHEET_ID_PATH.read_text().strip() - try: - update_sheet_tab(sheet_id, "q_deviation_st_vs_cpi", q_dev_df) - except Exception as e: # noqa: BLE001 - logger.warning( - "Sheet publish skipped (%s: %s). Local CSV still complete.", - type(e).__name__, - e, - ) - return - logger.info("Updated q_deviation_st_vs_cpi tab on sheet %s", sheet_id) - - -def plot_q_commodity_pi_vs_summary_tables(years: tuple[int, ...], path: Path) -> None: - """Log-log scatter of q_summary_tables vs q_commodity_price_index per year. - - One subplot per year (2 × 4 grid). Each point is one Cornerstone commodity. - The identity line (y = x) marks where both approaches agree. Points above - the line have a higher q in the summary_tables approach; points below are - lower. Missing parquet files for a given year produce an empty panel. - """ - ncols = 4 - nrows = (len(years) + ncols - 1) // ncols - fig, axes = plt.subplots(nrows, ncols, figsize=(ncols * 4, nrows * 4)) - axes_flat = np.asarray(axes).flatten() - - fig.suptitle( - "q: summary_tables vs commodity_price_index (all Cornerstone commodities)", - fontsize=11, - ) - - for idx, year in enumerate(years): - ax = axes_flat[idx] - cpi_path = RESULTS_DIR / f"q_commodity_price_index_{year}.parquet" - st_path = RESULTS_DIR / f"q_summary_tables_{year}.parquet" - - if not (cpi_path.exists() and st_path.exists()): - ax.text( - 0.5, - 0.5, - f"{year}\n(missing parquet)", - transform=ax.transAxes, - ha="center", - va="center", - fontsize=9, - ) - ax.set_title(str(year), fontsize=9) - continue - - q_cpi = _load_q_detail("commodity_price_index", year) - q_st = _load_q_detail("summary_tables", year) - - common = ( - pd.Index(CORNERSTONE_COMMODITIES) - .intersection(q_cpi.index) - .intersection(q_st.index) - ) - x = q_cpi.reindex(common).to_numpy(dtype=float) - y = q_st.reindex(common).to_numpy(dtype=float) - - # Mask non-positive values so log scale works cleanly - mask = (x > 0) & (y > 0) - ax.scatter( - x[mask], - y[mask], - s=6, - alpha=0.6, - color=APPROACH_COLORS["commodity_price_index"], - zorder=2, - ) - - # Identity line - lo = min(x[mask].min(), y[mask].min()) * 0.9 - hi = max(x[mask].max(), y[mask].max()) * 1.1 - ax.plot( - [lo, hi], [lo, hi], color="black", linewidth=0.8, linestyle="--", zorder=3 - ) - - ax.set_xscale("log") - ax.set_yscale("log") - ax.set_title(str(year), fontsize=9) - ax.set_xlabel("q commodity_price_index ($)", fontsize=7) - ax.set_ylabel("q summary_tables ($)", fontsize=7) - ax.grid(True, alpha=0.25, which="both") - - for i in range(len(years), len(axes_flat)): - axes_flat[i].set_visible(False) - - fig.tight_layout() - fig.savefig(path, dpi=150, bbox_inches="tight") - plt.close(fig) - - -def main() -> None: - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - PLOTS_DIR.mkdir(parents=True, exist_ok=True) - - logger.info("Computing summary-A cell-level errors across approaches × years") - errors_df = compute_errors_table() - errors_df.to_csv(RESULTS_DIR / "summary_a_errors.csv", index=False) - - logger.info("Computing q deviation: summary_tables vs commodity_price_index") - q_dev_df = compute_q_deviation_table() - q_dev_df.to_csv(RESULTS_DIR / "q_deviation_summary_tables_vs_cpi.csv", index=False) - - plot_rmse_ranking(errors_df, PLOTS_DIR / "summary_a_rmse_ranking.png") - plot_q_commodity_pi_vs_summary_tables( - YEARS, PLOTS_DIR / "q_commodity_pi_vs_summary_tables.png" - ) - - _publish_summary_a_errors_tab(errors_df) - _publish_q_deviation_tab(q_dev_df) - logger.info("Step 5 outputs written to %s and %s", RESULTS_DIR, PLOTS_DIR) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() diff --git a/bedrock/analysis/a_matrix_time_series/useeio_nowcast_ef_runbook.md b/bedrock/analysis/a_matrix_time_series/useeio_nowcast_ef_runbook.md deleted file mode 100644 index bdd75801..00000000 --- a/bedrock/analysis/a_matrix_time_series/useeio_nowcast_ef_runbook.md +++ /dev/null @@ -1,98 +0,0 @@ -# USEEIO Nowcast — EF Diagnostics Runbook - -Step N4 of [`docs/implement_useeio_nowcast_plan.md`](docs/implement_useeio_nowcast_plan.md). Steps N1–N3 are code changes that landed in this branch; this step is operator-triggered. - -For `USAConfig` flag and diagnostics playbooks (any methodology flag, not this -approach), see -[`../../utils/config/feature_flag.md`](../../utils/config/feature_flag.md) and -[`../../utils/validation/evaluate_feature_impact.md`](../../utils/validation/evaluate_feature_impact.md). - -## Pre-flight checklist - -- [ ] All 7 useeio_nowcast parquets exist locally (sanity for the analyst path; the workflow re-derives via the new branch in CI): - ``` - ls bedrock/analysis/a_matrix_time_series/output/results/A_useeio_nowcast_*.parquet - ``` - Expected: one per year 2017–2023. - -- [ ] Both YAMLs exist: - - `bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml` — minimal config for the A-matrix-only analysis (Steps N1–N3). - - `bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml` — **full v0.3 model** (`new_ghg_method`, `use_E_data_year_for_x_in_B`, `implement_waste_disaggregation`, `load_useeio_nowcast_A_matrix`) — required for EF diagnostics so apples-to-apples vs the other 4 `full_model_A_*` YAMLs. - -- [ ] Flag wired per [`feature_flag.md`](../../utils/config/feature_flag.md): `load_useeio_nowcast_A_matrix` on `USAConfig` and gated in `derive_cornerstone_Aq_scaled()`. - -- [ ] PR is open and CI is green (especially `pytest bedrock/transform/__tests__/test_usa.py -k cornerstone`). - -## Trigger the workflow - -The `generate_diagnostics` workflow consumes the **full-model** YAML (NOT the minimal one — the minimal one is for A-matrix-only analysis) and produces EFs in a Google Sheet, automatically benchmarked against CEDA-US (v0) and (with one checkbox) the USEEIO Excel baseline. - -For each `model_base_year ∈ {2018, 2019, 2020, 2021, 2022, 2023}`: - -1. Open GitHub Actions → `generate_diagnostics` → **Run workflow**. -2. Set parameters: - | Parameter | Value | - |---|---| - | **Branch** | the branch with this change | - | **USA config name** | `2025_usa_cornerstone_v0_2_A_useeio_nowcast` | - | **Model base year override** | `2023` (or whichever target year) | - | **Google Sheets ID** | a fresh sheet ID for this approach × year | - | **Benchmark to USEEIO GCS Excel baseline** | ☑ **tick this** (gives the USEEIO-do-nothing comparison automatically) | -3. Click **Run workflow**. -4. The workflow: - - Loads `2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml` → sets `load_useeio_nowcast_A_matrix=True` (plus all v0.3 full-model flags). - - Calls `derive_cornerstone_Aq_scaled()` → short-circuits to our new branch → calls `derive_useeio_nowcast_Aq_cornerstone(year=2023)`. - - That function loads V/U/U_imports from `gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/`, derives Cornerstone A, applies 0.98 cap, returns `SingleRegionAqMatrixSet`. - - Workflow runs full model end-to-end and writes EFs + sector outputs to the Sheet. - -If the workflow runs but the USEEIO checkbox was forgotten, re-trigger — it's cheap. Track run URLs in `step6_run_index.csv` (matches the parent plan's audit convention). - -### Waste disaggregation (now applied — apples-to-apples across approaches) - -All 5 full-model A YAMLs set `implement_waste_disaggregation: True`, including `useeio_nowcast`. `derive_useeio_nowcast_Aq_cornerstone()` mirrors the other approaches' Cornerstone disagg path: - -1. Map USEEIO's year-specific BEA-detail V/U to Cornerstone schema via correspondence multiplication (`industry_corresp() @ V @ commodity_corresp().T`, etc.). -2. Apply `apply_waste_disagg_to_V` and `apply_waste_disagg_to_U` from `bedrock.transform.eeio.waste_disaggregation` — the **same** helpers and **same** 2017 benchmark weights (`WasteDisaggregationDetail2017`) used by `_derive_cornerstone_Aq_from_disaggregated`. - -Net: the BEA `562000` row/col is split into 7 Cornerstone children (`562111, 562HAZ, 562212, 562213, 562910, 562920, 562OTH`) using 2017 make/use weights identically to the other approaches. **Waste-sector EFs are comparable across all 6 approaches** with no asterisk. - -The only methodological asymmetry that remains: the *2017 weights are applied to year-specific V/U*, which assumes the waste-make/use mix is stable across years (the same stability assumption the other approaches make implicitly). - -## Compile cross-approach EF comparison - -After all 6 approaches have completed runs (5 existing + `useeio_nowcast`): - -1. Open each sheet, copy the "summary" tab. -2. Compile into `bedrock/analysis/a_matrix_time_series/output/results/step6_ef_comparison.xlsx`: - - One tab per approach. - - `summary_vs_useeio` tab — diff of every approach against USEEIO baseline EFs. - - `summary_vs_ceda` tab — same vs CEDA-US. -3. Produce diagnostic figures: - - `output/plots/step6_ef_divergence_scatter_vs_useeio.png` — EF values per approach against USEEIO, log-log scatter, color = approach. - - `output/plots/step6_ef_divergence_scatter_vs_ceda.png` — same vs CEDA-US. - - `output/plots/step6_ef_useeio_nowcast_vs_alternatives.png` — focused 1×3 hexbin: useeio_nowcast EFs vs (summary_tables, industry_PI, commodity_PI). This is the figure that answers "does any internal alternative match the externally-balanced nowcast more closely than the others?" - -## Year-coverage caveat for plots - -`useeio_nowcast` has no 2024 upstream data. Anywhere the plot iterates years and may try to load `A_useeio_nowcast_2024.parquet`: -- `derive_A_time_series.py` already filters via `_years_for(approach, TARGET_YEARS)` → only 2017–2023 for `useeio_nowcast`. -- `compare_approaches.py` `USEEIO_NOWCAST_PAIRS` is evaluated at `USEEIO_NOWCAST_TARGET_YEAR = 2023`. -- Step 6 EF workflow: only trigger for model_base_year ∈ {2018..2023}. Running 2024 will fail with `ValueError: USEEIO nowcast not available for 2024.` — that's intended. - -## What to write up after the workflow runs - -Answer in 1 paragraph each: -1. **Does any internal alternative track useeio_nowcast more closely than the others?** Compare `summary_tables`, `industry_price_index`, `commodity_price_index` against `useeio_nowcast` cell-by-cell (Step 3 `pairwise_hexbins_useeio_nowcast_*.png`) AND in EF space (Step 6 scatter). -2. **Are the persistent USEEIO reconciliation outliers (`5412OP`, `GSLG`, `81`, `722`, `23`) visible in the EF differences?** If so, flag in the README so downstream users know these sectors carry larger uncertainty in the nowcast. -3. **Methodological circularity caveat**: USEEIO's GRAS and bedrock's `summary_tables` both reconcile 2017 detail to BEA summary aggregates. Close agreement is partly a methodological tautology, not validation. Highlight this in the recommendation section. - -## Caveats baked into the upstream nowcast (carry these forward to the EF write-up) - -From [`USEEIO_nowcasting.md`](../../../USEEIO_nowcasting.md): -- Within-summary detail-level technology is frozen at 2017 (the dominant limitation). -- Real input substitution is not captured (constant-volume recipe assumption). -- Margin rates are held fixed at 2017 PCE Bridge values. -- Most FD columns are 2017-shape + RAS rebalance — only 12 curated columns have fresh NIPA-driven estimates. -- VA structural change uses a single commodity-weighted average Rho across all VA rows. - -These limits mean `useeio_nowcast` and `summary_tables` give similar A-matrix updates by design — both freeze the same things. The interesting divergences are between `useeio_nowcast` and the price-index methods. diff --git a/bedrock/analysis/a_matrix_time_series/view_vnorm_and_price_ratios.py b/bedrock/analysis/a_matrix_time_series/view_vnorm_and_price_ratios.py deleted file mode 100644 index a048befc..00000000 --- a/bedrock/analysis/a_matrix_time_series/view_vnorm_and_price_ratios.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Print side-by-side industry vs V-norm-derived commodity price ratios. - -Diagnostic helper for the ``commodity_price_index`` approach: derives Vnorm and -the two price-ratio series, checks index alignment between ``V`` and the -upstream PI, and prints distributional summaries. Run when the V-norm code path -changes or to sanity-check a new ``model_base_year``. -""" - -from __future__ import annotations - -from bedrock.analysis.a_matrix_time_series.constants import ( - LATEST_TARGET_YEAR, - ORIGINAL_YEAR, - RESULTS_DIR, -) -from bedrock.transform.eeio.derived_cornerstone import ( - derive_cornerstone_V, - derive_cornerstone_Vnorm_scrap_corrected, -) -from bedrock.utils.config.usa_config import get_usa_config -from bedrock.utils.economic.inflation_helpers_cornerstone import ( - get_cornerstone_industry_price_ratio, - get_vnorm_adjusted_commodity_price_ratio, -) - -original_year = ORIGINAL_YEAR -target_year = LATEST_TARGET_YEAR - - -def main() -> None: - Vnorm = derive_cornerstone_Vnorm_scrap_corrected( - apply_inflation=get_usa_config().apply_inflation_to_V, - target_year=target_year, - ) - industry = get_cornerstone_industry_price_ratio(original_year, target_year).rename( - "industry_ratio" - ) - - V = derive_cornerstone_V() - inddiff = set(industry.index) - set(V.index) - print(f"Industries in V but not in price index: {inddiff}") - - # Check if index order is different - v_industries = V.index.tolist() - industry_indices = industry.index.tolist() - order_diff = v_industries != industry_indices - print(f"Index order is different: {order_diff}") - if order_diff: - print(f"First 5 V indices: {v_industries[:5]}") - print(f"First 5 industry indices: {industry_indices[:5]}") - - commodity = get_vnorm_adjusted_commodity_price_ratio( - original_year, target_year - ).rename("commodity_ratio") - - vnorm_path = RESULTS_DIR / f"Vnorm{target_year}.csv" - industry_path = RESULTS_DIR / f"industry_price_ratio_{target_year}.csv" - commodity_path = RESULTS_DIR / f"commodity_price_ratio_{target_year}.csv" - Vnorm.to_csv(vnorm_path) - industry.to_csv(industry_path) - commodity.to_csv(commodity_path) - print( - f"Vnorm written to {vnorm_path.name}. " - f"Industry ratios written to {industry_path.name}. " - f"Commodity ratios written to {commodity_path.name}." - ) - return None - - -if __name__ == "__main__": - main() diff --git a/bedrock/analysis/electricity/d_85/scenario_ef_pipeline.py b/bedrock/analysis/electricity/d_85/scenario_ef_pipeline.py index a3d9a31c..19974d2d 100644 --- a/bedrock/analysis/electricity/d_85/scenario_ef_pipeline.py +++ b/bedrock/analysis/electricity/d_85/scenario_ef_pipeline.py @@ -85,9 +85,7 @@ def scenario_vnorm( def derive_Aq_from_scenario(scenario: DisaggScenarioResult) -> SingleRegionAqMatrixSet: """Unscaled 2017 A/q from scenario IO.""" - Vnorm = scenario_vnorm( - scenario, apply_inflation=get_usa_config().apply_inflation_to_V - ) + Vnorm = scenario_vnorm(scenario, apply_inflation=False) x = scenario.x Adom = compute_Unorm_matrix(U=scenario.Udom, x=x) @ Vnorm Aimp = compute_Unorm_matrix(U=scenario.Uimp, x=x) @ Vnorm @@ -197,9 +195,7 @@ def derive_B_from_scenario(scenario: DisaggScenarioResult) -> pd.DataFrame: E = derive_E_usa() x = scenario.x.reindex(E.columns, fill_value=np.nan) x = x.fillna(1.0) - Vnorm = scenario_vnorm( - scenario, apply_inflation=get_usa_config().apply_inflation_to_V - ) + Vnorm = scenario_vnorm(scenario, apply_inflation=False) return (E.div(x, axis=1)) @ Vnorm diff --git a/bedrock/analysis/electricity_disagg_diagnostics/year_alignment/year_alignment_bly_e.py b/bedrock/analysis/electricity_disagg_diagnostics/year_alignment/year_alignment_bly_e.py index d85e6e76..768b91cc 100644 --- a/bedrock/analysis/electricity_disagg_diagnostics/year_alignment/year_alignment_bly_e.py +++ b/bedrock/analysis/electricity_disagg_diagnostics/year_alignment/year_alignment_bly_e.py @@ -656,9 +656,7 @@ def main(argv: list[str] | None = None) -> None: "usa_io_data_year": 2017, "usa_ghg_data_year": 2017, "scale_a_matrix_with_useeio_method": True, - "scale_a_matrix_with_summary_tables": False, "scale_a_matrix_with_ceda_method_as_fallback": False, - "scale_a_matrix_with_commodity_price_index": False, "adjust_summary_A_and_q_dollar_year": False, }, use_2017_e_patch=True, diff --git a/bedrock/extract/iot/useeio_nowcast.py b/bedrock/extract/iot/useeio_nowcast.py deleted file mode 100644 index f2b8d321..00000000 --- a/bedrock/extract/iot/useeio_nowcast.py +++ /dev/null @@ -1,101 +0,0 @@ -"""USEEIO nowcasted detail MUTs (V, U total, U imports) for 2017–2023. - -Files were produced by the upstream USEEIO/useeior nowcasting team -(``cornerstone-data/USEEIO`` @ ``nowcasting`` branch, commit 2025-09-30) and -staged to GCS by the bedrock team. See: - -- [USEEIO_nowcasting.md](../../../../USEEIO_nowcasting.md) — methodology, what the nowcast does and doesn't capture -- [bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md](../../analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md) — integration plan - -Source CSVs at ``gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/``: - -- ``V_out_{yr}.csv`` — Make table, **commodity × industry** (BEA 2017 Detail - schema, bare codes; mUSD) -- ``U_out_{yr}.csv`` — full Use table, **commodity+VA × {industry + FD + VA}** - (last 3 rows are VA: V00100/V00200/V00300; FD columns vary by year) -- ``U_imports_out_{yr}.csv`` — imports portion of Use, **commodity × {industry + FD}** - -All values in millions of USD. Codes are bare (no ``/US`` suffix). -""" - -from __future__ import annotations - -import functools - -import pandas as pd - -from bedrock.utils.io.gcp import load_from_gcs -from bedrock.utils.io.local_extract_input_data import local_dir_for_gcs_sub_bucket - -GCS_USEEIO_NOWCAST_DIR = "extract/input-data/USEEIO_nowcasted_MUTs" -LOCAL_USEEIO_NOWCAST_DIR = local_dir_for_gcs_sub_bucket(GCS_USEEIO_NOWCAST_DIR) - -# Years for which upstream USEEIO has produced nowcasted tables. 2024 is NOT -# in this set — upstream has not run the pipeline for that year yet. -USEEIO_NOWCAST_YEARS: tuple[int, ...] = (2017, 2018, 2019, 2020, 2021, 2022, 2023) - -# Cols 0..401 of U_out and U_imports_out are industries; cols 402+ are -# Final-Demand columns (F010, F02N, …) and VA columns (V00100, V00200, -# V00300). See upstream R script ``load_suts_from_r.py`` for the slicing. -USEEIO_NOWCAST_INDUSTRY_COUNT = 402 - - -def _validate_year(year: int) -> None: - if year not in USEEIO_NOWCAST_YEARS: - raise ValueError( - f"USEEIO nowcast not available for {year}. " - f"Available years: {USEEIO_NOWCAST_YEARS}" - ) - - -def _load_csv(name: str) -> pd.DataFrame: - return load_from_gcs( - name=name, - sub_bucket=GCS_USEEIO_NOWCAST_DIR, - local_dir=str(LOCAL_USEEIO_NOWCAST_DIR), - loader=lambda pth: pd.read_csv(pth, index_col=0), - ) - - -@functools.cache -def load_useeio_nowcast_V_usa(year: int) -> pd.DataFrame: - """Make table in bedrock's ``industry × commodity`` convention. - - Upstream files are ``commodity × industry`` (the transposed Make); - this loader transposes back so the return matches ``load_2017_V_usa``. - """ - _validate_year(year) - raw = _load_csv(f"V_out_{year}.csv") - V = raw.T - V.index = V.index.astype(str) - V.columns = V.columns.astype(str) - return V - - -@functools.cache -def load_useeio_nowcast_Utot_intermediate_usa(year: int) -> pd.DataFrame: - """Total intermediate Use, commodity × industry (FD and VA columns dropped). - - USEEIO ``U_out`` is the full Use matrix (commodity + 3 VA rows × {industry + - FD + VA cols}). For A-matrix derivation we want only the - ``commodity × industry`` intermediate-Use block — first ``402`` columns, - full set of rows (VA rows are filtered out at the reindex-to-Vnorm step - in the transform layer since they don't appear in Vnorm's commodity axis). - """ - _validate_year(year) - raw = _load_csv(f"U_out_{year}.csv") - df = raw.iloc[:, :USEEIO_NOWCAST_INDUSTRY_COUNT].copy() - df.index = df.index.astype(str) - df.columns = df.columns.astype(str) - return df - - -@functools.cache -def load_useeio_nowcast_Uimp_intermediate_usa(year: int) -> pd.DataFrame: - """Imports portion of intermediate Use, commodity × industry.""" - _validate_year(year) - raw = _load_csv(f"U_imports_out_{year}.csv") - df = raw.iloc[:, :USEEIO_NOWCAST_INDUSTRY_COUNT].copy() - df.index = df.index.astype(str) - df.columns = df.columns.astype(str) - return df diff --git a/bedrock/transform/eeio/__tests__/test_summary_a_realigned.py b/bedrock/transform/eeio/__tests__/test_summary_a_realigned.py deleted file mode 100644 index f2ffee06..00000000 --- a/bedrock/transform/eeio/__tests__/test_summary_a_realigned.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Tests for the summary-tables A branch with dollar-year-aligned scaling. - -``scale_a_matrix_with_summary_tables=True`` now always: - 1. deflates the target-year summary A to 2017 USD before forming the ratio - against the 2017 summary A (so the ratio is in matched dollar years), - 2. applies the structural ratio to the 2017 detail A, - 3. inflates the result 2017 → model_year. -""" - -from __future__ import annotations - -import pytest - -from bedrock.transform.eeio.derived_cornerstone import ( - derive_cornerstone_Aq, - derive_cornerstone_Aq_scaled, -) -from bedrock.utils.config.usa_config import get_usa_config - - -def _set_summary_tables(monkeypatch: pytest.MonkeyPatch) -> None: - cfg = get_usa_config() - monkeypatch.setattr(cfg, 'use_cornerstone_2026_model_schema', True) - monkeypatch.setattr(cfg, 'scale_a_matrix_with_summary_tables', True) - monkeypatch.setattr(cfg, 'adjust_summary_A_and_q_dollar_year', True) - monkeypatch.setattr(cfg, 'scale_a_matrix_with_useeio_method', False) - monkeypatch.setattr(cfg, 'scale_a_matrix_with_commodity_price_index', False) - derive_cornerstone_Aq_scaled.cache_clear() - - -def test_summary_tables_branch_is_noop_at_model_year_equals_detail_year( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """When ``model_year == detail_year == 2017``, the cross-year ratio is - identically 1, the summary-A deflation is a no-op, and the final inflate - is a no-op — so the summary-tables branch must return the base 2017 A - unchanged. - """ - cfg = get_usa_config() - _set_summary_tables(monkeypatch) - monkeypatch.setattr(cfg, 'model_base_year', 2017) - - base = derive_cornerstone_Aq() - result = derive_cornerstone_Aq_scaled() - - max_dev_dom = (result.Adom.to_numpy() - base.Adom.to_numpy()).max() - max_dev_imp = (result.Aimp.to_numpy() - base.Aimp.to_numpy()).max() - assert ( - abs(max_dev_dom) < 1e-9 - ), f"Adom drifted at model_year=2017 (max |Δ| = {max_dev_dom:.2e})" - assert ( - abs(max_dev_imp) < 1e-9 - ), f"Aimp drifted at model_year=2017 (max |Δ| = {max_dev_imp:.2e})" - - -def test_summary_tables_branch_no_nan_no_negatives( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Summary-tables A and q should have no NaN and no negative entries — - the deflate step preserves sign, ``scale_cornerstone_A``'s downstream - cap enforces non-negativity on the per-matrix output, and the final - inflate is sign-preserving. - - Column-sum ≤ 1 is not asserted: the final price-index inflation - (``diag(p) @ A @ diag(1/p)``) can push some columns above 1, matching - the default CEDA branch's behavior — the codebase does not re-cap after - inflation. - """ - cfg = get_usa_config() - _set_summary_tables(monkeypatch) - monkeypatch.setattr(cfg, 'model_base_year', 2023) - - result = derive_cornerstone_Aq_scaled() - assert not result.Adom.isna().to_numpy().any(), 'Adom has NaN' - assert not result.Aimp.isna().to_numpy().any(), 'Aimp has NaN' - assert (result.Adom.to_numpy() >= 0).all(), 'Adom has negatives' - assert (result.Aimp.to_numpy() >= 0).all(), 'Aimp has negatives' - assert not result.scaled_q.isna().to_numpy().any(), 'scaled_q has NaN' diff --git a/bedrock/transform/eeio/derived_cornerstone.py b/bedrock/transform/eeio/derived_cornerstone.py index 8f063d20..bcff5bbb 100644 --- a/bedrock/transform/eeio/derived_cornerstone.py +++ b/bedrock/transform/eeio/derived_cornerstone.py @@ -278,12 +278,7 @@ def derive_cornerstone_x_after_redefinition(year: int = 0) -> pd.Series[float]: @functools.cache def derive_cornerstone_q() -> pd.Series[float]: - cfg = get_usa_config() - q = compute_q( - V=derive_cornerstone_V( - apply_inflation=cfg.apply_inflation_to_V, target_year=cfg.model_base_year - ) - ) + q = compute_q(V=derive_cornerstone_V()) validate_cornerstone(q, 'Q') return q @@ -295,18 +290,14 @@ def derive_cornerstone_Vnorm_scrap_corrected( ) -> pd.DataFrame: """Scrap-corrected V norm. Inflation is applied via ``derive_cornerstone_V``. - When ``apply_inflation`` is omitted, uses ``USAConfig.apply_inflation_to_V``. When ``target_year`` is not positive and inflation is on, uses ``USAConfig.model_base_year``. """ - cfg = get_usa_config() - use_inflation = ( - cfg.apply_inflation_to_V if apply_inflation is None else apply_inflation - ) + use_inflation = bool(apply_inflation) effective_year = ( target_year if target_year > 0 - else (cfg.model_base_year if use_inflation else 0) + else (get_usa_config().model_base_year if use_inflation else 0) ) V = derive_cornerstone_V(use_inflation, effective_year) @@ -317,7 +308,7 @@ def derive_cornerstone_Vnorm_scrap_corrected( scrap_2017 = load_2017_V_usa().loc[:, 'S00401'] scrap_fraction = industry_corresp() @ scrap_2017 - if cfg.implement_electricity_disaggregation: + if get_usa_config().implement_electricity_disaggregation: parent_scrap = float(scrap_fraction.get(ELECTRICITY_AGGREGATE_SECTOR, 0.0)) scrap_fraction = scrap_fraction.drop( ELECTRICITY_AGGREGATE_SECTOR, errors='ignore' @@ -473,10 +464,6 @@ def derive_cornerstone_Aq() -> SingleRegionAqMatrixSet: def _derive_cornerstone_Aq_from_disaggregated() -> SingleRegionAqMatrixSet: """A and q from disaggregated Cornerstone V and U (no intragroup treatment).""" - # When apply_inflation_to_V is True: q and x use uninflated derive_cornerstone_V() - # (2017 $), while Vnorm uses derive_cornerstone_Vnorm_scrap_corrected() (model-year $). - # derive_cornerstone_q() applies the flag but is not used here. Intentional for now; - # see inflation/A dollar-year design notes. V = derive_cornerstone_V() uset = derive_cornerstone_U_set() Udom: pd.DataFrame = uset.Udom @@ -515,80 +502,12 @@ def derive_cornerstone_Aq_scaled() -> SingleRegionAqMatrixSet: if cfg.scale_a_matrix_with_useeio_method: return base - # USEEIO nowcast: load externally-balanced detail SUTs from GCS and - # derive Cornerstone A directly. Bypasses all internal scaling/inflation; - # treats the upstream USEEIO team's GRAS-balanced 2018–2023 SUTs as the - # source of structural change. Loaders: bedrock.extract.iot.useeio_nowcast; - # derivation: bedrock.transform.eeio.derived_useeio_nowcast. - if cfg.load_useeio_nowcast_A_matrix: - from bedrock.transform.eeio.derived_useeio_nowcast import ( # noqa: PLC0415 - derive_useeio_nowcast_Aq_cornerstone, - ) - - return derive_useeio_nowcast_Aq_cornerstone(year=model_year) - - # Summary tables: scale 2017 → model_year using summary A ratios. - # - # When `cfg.adjust_summary_A_and_q_dollar_year` is set, `scale_cornerstone_A` - # rebases the target-year summary A into 2017 USD before thebase.scaled_q, ratio is taken, - # so the structural cross-year ratio is formed entirely in 2017 USD; the - # scaled detail A is then inflated 2017 → model_year. When the flag is off, - # the ratio carries the raw target-year-vs-2017 price drift and no final - # inflation is applied (pre-realignment behavior). - if cfg.scale_a_matrix_with_summary_tables: - Adom = scale_cornerstone_A( - base.Adom, - target_year=model_year, - original_year=detail_year, - dom_or_imp_or_total='dom', - ) - Aimp = scale_cornerstone_A( - base.Aimp, - target_year=model_year, - original_year=detail_year, - dom_or_imp_or_total='imp', - ) - q = scale_cornerstone_q( - base.scaled_q, - target_year=model_year, - original_year=detail_year, - ) - if cfg.adjust_summary_A_and_q_dollar_year: - Adom = inflate_cornerstone_A_matrix_with_commodity_pi( - Adom, original_year=detail_year, target_year=model_year - ) - Aimp = inflate_cornerstone_A_matrix_with_commodity_pi( - Aimp, original_year=detail_year, target_year=model_year - ) - q = inflate_cornerstone_q_or_y_with_commodity_pi( - q, original_year=detail_year, target_year=model_year - ) - return _cornerstone_aq_matrix_set(Adom=Adom, Aimp=Aimp, scaled_q=q) - - # Commodity price index (V-norm-derived): like the industry-price branch, - # but uses V_norm to weight industry price ratios into commodity space - # before applying diag(p) @ A @ diag(1/p). - if cfg.scale_a_matrix_with_commodity_price_index: - Adom = inflate_cornerstone_A_matrix_with_commodity_pi( - base.Adom, original_year=detail_year, target_year=model_year - ) - Aimp = inflate_cornerstone_A_matrix_with_commodity_pi( - base.Aimp, original_year=detail_year, target_year=model_year - ) - q = inflate_cornerstone_q_or_y_with_commodity_pi( - base.scaled_q, original_year=detail_year, target_year=model_year - ) - return _cornerstone_aq_matrix_set(Adom=Adom, Aimp=Aimp, scaled_q=q) - # CEDA method: our fallback option as of CY26Q2. # Scale to 2022 (io_year), then inflate to model_base_year. # However, we are applying some subtle changes to this method: # 1. scale detail A and q with dollar year adjusted summary numbers # 2. inflate with commodity pi instead of industry pi # - # Codepath of this approach is very similar to the scale_a_matrix_with_summary_tables approach, - # the only difference is which year to scale to. - # # When `cfg.adjust_summary_A_and_q_dollar_year` is set, `scale_cornerstone_A` # rebases the target-year summary A into 2017 USD before the ratio is taken, # so the structural cross-year ratio is formed entirely in 2017 USD; the diff --git a/bedrock/transform/eeio/derived_useeio_nowcast.py b/bedrock/transform/eeio/derived_useeio_nowcast.py deleted file mode 100644 index 7de018fb..00000000 --- a/bedrock/transform/eeio/derived_useeio_nowcast.py +++ /dev/null @@ -1,212 +0,0 @@ -"""Cornerstone A matrices derived from the upstream USEEIO nowcast. - -Loaders live in [bedrock.extract.iot.useeio_nowcast](../../extract/iot/useeio_nowcast.py). -This module mirrors ``derive_cornerstone_Aq()``'s waste-disagg-aware path -applied to USEEIO's year-specific V/U: - -1. Map BEA-detail V/U/Uimp to Cornerstone schema via - ``industry_corresp() @ V @ commodity_corresp().T`` (and the U analog). -2. If ``implement_waste_disaggregation`` is enabled, apply - ``apply_waste_disagg_to_V`` and ``apply_waste_disagg_to_U`` — both use - the 2017 benchmark weights (``WasteDisaggregationDetail2017``) to split - the BEA ``562000`` row/col into 5 Cornerstone child codes. -3. Clip GRAS reconciliation negatives in U to 0 (~0.27% of ``|U_dom|``; - magnitude logged). -4. Build Vnorm with year-specific scrap correction (``scrap / x`` from the - year's V, mapped to Cornerstone industries). -5. Compute ``A = Unorm @ Vnorm`` per matrix; apply the 0.98 column cap. -6. Return year-specific q derived directly from the year's V (Cornerstone - space). - -Settled policies (see [implement_useeio_nowcast_plan.md](../../analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md)): - -- **2017 identity vs bedrock's BEA Aq is NOT enforced.** USEEIO and bedrock load - different BEA 2017 vintages; expect ~9-22% L1 deviation. -- **Negatives in intermediate U are clipped to 0** (reconciliation noise). -- **0.98 column cap is applied per matrix** (Adom, Aimp separately). -- **Year-specific Vnorm + q** — V from each year drives both, so input - shares AND commodity output reflect the year's nowcasted Make. -- **Waste disagg uses 2017 weights** when ``implement_waste_disaggregation`` - is True — splits ``562000`` into 5 Cornerstone children with the same - weights the other approaches use. -""" - -from __future__ import annotations - -import functools -import logging - -import pandas as pd -import pandera.typing as pt - -from bedrock.extract.iot.useeio_nowcast import ( - USEEIO_NOWCAST_YEARS, - load_useeio_nowcast_Uimp_intermediate_usa, - load_useeio_nowcast_Utot_intermediate_usa, - load_useeio_nowcast_V_usa, -) -from bedrock.transform.eeio.cornerstone_expansion import ( - commodity_corresp, - industry_corresp, -) -from bedrock.transform.eeio.waste_disaggregation import ( - apply_waste_disagg_to_U, - apply_waste_disagg_to_V, -) -from bedrock.utils.math.formulas import ( - compute_A_matrix, - compute_q, - compute_Unorm_matrix, - compute_Vnorm_matrix, - compute_x, -) -from bedrock.utils.schemas.cornerstone_schemas import CornerstoneAMatrix -from bedrock.utils.schemas.single_region_types import SingleRegionAqMatrixSet - -logger = logging.getLogger(__name__) - -COLUMN_CAP = 0.98 # matches bedrock.transform.eeio.cornerstone_year_scaling - - -# ========================================================================= -# Public API -# ========================================================================= - - -@functools.cache -def derive_useeio_nowcast_Aq_cornerstone(year: int) -> SingleRegionAqMatrixSet: - """Cornerstone-space (Adom, Aimp, q) for the given nowcast year. - - Available years: ``USEEIO_NOWCAST_YEARS`` (2017–2023). 2024 raises. - - Cached because ``derive_cornerstone_Aq_scaled()`` calls this from a - ``@functools.cache``-decorated function; the analysis driver clears - the cache between (approach, year) iterations. - """ - if year not in USEEIO_NOWCAST_YEARS: - raise ValueError( - f"USEEIO nowcast not available for {year}. " - f"Available years: {USEEIO_NOWCAST_YEARS}" - ) - - # Lazy import — ``get_waste_disagg_weights`` lives in cornerstone_disagg_pipeline, - # which transitively pulls flowsa via allocation.derived at module import. - # Keeping this lazy lets the analysis driver populate parquets without - # initializing the flowsa log handler at module-import time. - from bedrock.transform.eeio.cornerstone_disagg_pipeline import ( # noqa: PLC0415 - get_waste_disagg_weights, - ) - - # 1. Load USEEIO inputs (BEA detail). U_out includes 3 VA rows - # (V00100/V00200/V00300) that U_imports_out doesn't — align both to the - # commodity axis of V (402 BEA detail commodities) before subtracting. - V_bea = load_useeio_nowcast_V_usa(year) - U_int_bea = load_useeio_nowcast_Utot_intermediate_usa(year) - Uimp_int_bea = load_useeio_nowcast_Uimp_intermediate_usa(year) - commodity_idx = V_bea.columns - industry_idx = V_bea.index - U_int_bea = U_int_bea.reindex( - index=commodity_idx, columns=industry_idx, fill_value=0.0 - ) - Uimp_int_bea = Uimp_int_bea.reindex( - index=commodity_idx, columns=industry_idx, fill_value=0.0 - ) - - # 2. Map BEA detail → Cornerstone schema via correspondence multiplication - # (mirrors derive_cornerstone_V / derive_cornerstone_U_with_negatives). - ind_corresp = industry_corresp() - com_corresp = commodity_corresp() - V_cs = ind_corresp @ V_bea @ com_corresp.T - Udom_bea = U_int_bea - Uimp_int_bea - Udom_cs = com_corresp @ Udom_bea @ ind_corresp.T - Uimp_cs = com_corresp @ Uimp_int_bea @ ind_corresp.T - for df in (V_cs, Udom_cs, Uimp_cs): - df.index.name = "sector" - df.columns.name = "sector" - - # 3. Apply waste disagg if enabled (mirrors derive_cornerstone_V + - # derive_cornerstone_U_with_negatives). Uses 2017 benchmark weights. - weights = get_waste_disagg_weights() - if weights is not None: - V_cs = apply_waste_disagg_to_V(V_cs, weights) - Udom_cs, Uimp_cs = apply_waste_disagg_to_U(Udom_cs, Uimp_cs, weights) - for df in (V_cs, Udom_cs, Uimp_cs): - df.index.name = "sector" - df.columns.name = "sector" - - # 4. Log negatives, clip to 0. - _log_and_clip_negatives(year, Udom_cs, Uimp_cs) - Udom_cs = Udom_cs.clip(lower=0) - Uimp_cs = Uimp_cs.clip(lower=0) - - # 5. Vnorm with year-specific scrap correction (mirrors - # derive_cornerstone_Vnorm_scrap_corrected, but with year-specific V). - x = compute_x(V=V_cs) - q = compute_q(V=V_cs) - Vnorm = compute_Vnorm_matrix(V=V_cs, q=q) - scrap_bea = V_bea.loc[:, "S00401"] - scrap_fraction = ind_corresp @ scrap_bea - Vnorm = Vnorm.divide((1.0 - (scrap_fraction / x).fillna(0.0)), axis=0) - - # 6. A = U_norm @ Vnorm, per matrix. - Adom = compute_A_matrix(U_norm=compute_Unorm_matrix(U=Udom_cs, x=x), V_norm=Vnorm) - Aimp = compute_A_matrix(U_norm=compute_Unorm_matrix(U=Uimp_cs, x=x), V_norm=Vnorm) - - # 7. Column cap. - Adom = _apply_column_cap(Adom, label=f"Adom_{year}") - Aimp = _apply_column_cap(Aimp, label=f"Aimp_{year}") - - assert not Adom.isna().values.any(), f"Adom has NaN at year={year}" - assert not Aimp.isna().values.any(), f"Aimp has NaN at year={year}" - assert (Adom.values >= 0).all(), f"Adom has negatives at year={year}" - assert (Aimp.values >= 0).all(), f"Aimp has negatives at year={year}" - - return SingleRegionAqMatrixSet( - Adom=pt.DataFrame[CornerstoneAMatrix](Adom), # type: ignore[arg-type] - Aimp=pt.DataFrame[CornerstoneAMatrix](Aimp), # type: ignore[arg-type] - scaled_q=q, - ) - - -# ========================================================================= -# Private helpers -# ========================================================================= - - -def _log_and_clip_negatives( - year: int, U_dom: pd.DataFrame, Uimp_int: pd.DataFrame -) -> None: - """Report the magnitude of negative cells before they get clipped to 0.""" - n_neg_dom = int((U_dom < 0).sum().sum()) - n_neg_imp = int((Uimp_int < 0).sum().sum()) - total_abs = float(U_dom.abs().sum().sum()) - neg_abs = float(U_dom.where(U_dom < 0).abs().sum().sum()) - pct = 100.0 * neg_abs / total_abs if total_abs else 0.0 - logger.info( - "[%d] U_dom negatives: %d cells, |sum|=%.3e (%.4f%% of |U_dom|); " - "U_imp negatives: %d cells. Clipping all to 0.", - year, - n_neg_dom, - neg_abs, - pct, - n_neg_imp, - ) - - -def _apply_column_cap(A: pd.DataFrame, label: str) -> pd.DataFrame: - """Cap any column whose sum > 1 down to ``COLUMN_CAP``, mirroring - ``scale_cornerstone_A``'s post-processing.""" - A = A.copy() - col_sums = A.sum(axis=0) - over = col_sums[col_sums > 1.0].sort_values(ascending=False) - if len(over): - logger.info( - "[%s] %d column(s) > 1.0 capped to %.2f: %s", - label, - len(over), - COLUMN_CAP, - over.head(5).to_dict(), - ) - for col, total in over.items(): - A[col] *= COLUMN_CAP / total - return A diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_A_commodity_price_index.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_A_commodity_price_index.yaml deleted file mode 100644 index d4ad2aa0..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_A_commodity_price_index.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Tests V-norm-derived commodity-price-index approach for deriving the A matrix. -# A is inflated 2017 -> target year using a V_norm-weighted commodity price ratio -# (industry ratios aggregated into commodity space). Step 0 of epic #337. -use_cornerstone_2026_model_schema: True -scale_a_matrix_with_commodity_price_index: True -apply_inflation_to_V: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_A_summary_tables.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_A_summary_tables.yaml deleted file mode 100644 index b586bbe7..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_A_summary_tables.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# Tests summary-tables-only approach for deriving the A matrix (Issue #182). -# A is scaled directly 2017 → target year using summary table ratios, no price inflation. -use_cornerstone_2026_model_schema: True -scale_a_matrix_with_summary_tables: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio.yaml deleted file mode 100644 index a37a2eb6..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# Tests USEEIO's approach for deriving the A matrix (Issue #182). -# No scaling or inflation applied — 2017 technology mix assumed representative of target year. -use_cornerstone_2026_model_schema: True -scale_a_matrix_with_useeio_method: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml deleted file mode 100644 index a7fc0e9f..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_A_useeio_nowcast.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# USEEIO nowcast approach: load externally-balanced detail SUTs from GCS -# (gs://cornerstone-default/extract/input-data/USEEIO_nowcasted_MUTs/) and -# derive Cornerstone A directly, bypassing bedrock's scale/inflate path. -# -# Background: USEEIO_nowcasting.md (repo root) and -# bedrock/analysis/a_matrix_time_series/docs/implement_useeio_nowcast_plan.md. -# -# Supported model_base_year: 2018..2023 (USEEIO upstream pipeline has not -# been run for 2024). -use_cornerstone_2026_model_schema: True -load_useeio_nowcast_A_matrix: True -implement_waste_disaggregation: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy.yaml deleted file mode 100644 index bb8ed43c..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_taxonomy.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This config is used to run the Cornerstone 2026 model schema on the 2025 USA data. - -##### -# Model base settings -##### - -##### -# Data selection -##### - -##### -# Methodology selection -##### -use_cornerstone_2026_model_schema: True - - -##### -# Bugfix -##### \ No newline at end of file diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_commodity_price_index.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_commodity_price_index.yaml deleted file mode 100644 index acc1e649..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_commodity_price_index.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# Full v0.2 model bundled with V-norm-derived commodity-price-index -# A-matrix scaling. A is inflated 2017 → target year using a V-norm- -# weighted commodity price ratio; V is itself inflated to model_base_year -# via apply_inflation_to_V. -use_cornerstone_2026_model_schema: True -new_ghg_method: True -use_E_data_year_for_x_in_B: True -implement_waste_disaggregation: True -scale_a_matrix_with_commodity_price_index: True -apply_inflation_to_V: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_summary_tables.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_summary_tables.yaml deleted file mode 100644 index 6a03aad9..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_summary_tables.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Full v0.2 model bundled with summary-tables A-matrix scaling: -# A is scaled 2017 → target year via BEA summary-table ratios. -use_cornerstone_2026_model_schema: True -new_ghg_method: True -use_E_data_year_for_x_in_B: True -implement_waste_disaggregation: True -scale_a_matrix_with_summary_tables: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio.yaml deleted file mode 100644 index 762c3477..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Full v0.2 model bundled with USEEIO A-matrix method: -# A is the BEA 2017 detail benchmark with no temporal scaling. -use_cornerstone_2026_model_schema: True -new_ghg_method: True -use_E_data_year_for_x_in_B: True -implement_waste_disaggregation: True -scale_a_matrix_with_useeio_method: True diff --git a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml b/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml deleted file mode 100644 index 84a96fe4..00000000 --- a/bedrock/utils/config/configs/2025_usa_cornerstone_v0_2_A_useeio_nowcast.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Full v0.2 model bundled with USEEIO-nowcast A-matrix method: -# A is derived directly from the upstream USEEIO team's GRAS-balanced -# detail SUTs for the target year (2018-2023 supported; 2024 raises). -# See bedrock/transform/eeio/derived_useeio_nowcast.py and the in-repo -# summary USEEIO_nowcasting.md for methodology. -# -# Waste disaggregation: ON. derive_useeio_nowcast_Aq_cornerstone() applies -# the 2017 benchmark WasteDisaggregationDetail2017 weights to V and U -# (via apply_waste_disagg_to_V / apply_waste_disagg_to_U) before computing -# A — same helper and same weights the other 4 full-model A approaches -# use, so waste rows are comparable across approaches. -use_cornerstone_2026_model_schema: True -new_ghg_method: True -use_E_data_year_for_x_in_B: True -implement_waste_disaggregation: True -load_useeio_nowcast_A_matrix: True diff --git a/bedrock/utils/config/configs/useeio_phoebe_23.yaml b/bedrock/utils/config/configs/useeio_phoebe_23.yaml index 5a132dfd..54f721a0 100644 --- a/bedrock/utils/config/configs/useeio_phoebe_23.yaml +++ b/bedrock/utils/config/configs/useeio_phoebe_23.yaml @@ -21,7 +21,6 @@ use_E_data_year_for_x_in_B: true implement_waste_disaggregation: true # USEEIO-style A: keep 2017 base A when scaling A,q (PR #229 / Issue #182). -# Mutually exclusive in practice with scale_a_matrix_with_summary_tables / _price_index. scale_a_matrix_with_useeio_method: true use_useeio_schema: true diff --git a/bedrock/utils/config/usa_config.py b/bedrock/utils/config/usa_config.py index c975eb86..35d31062 100644 --- a/bedrock/utils/config/usa_config.py +++ b/bedrock/utils/config/usa_config.py @@ -86,9 +86,6 @@ class USAConfig(BaseModel): implement_electricity_mixed_units: bool = False # DRI: jorge.vendries scale_a_matrix_with_ceda_method_as_fallback: bool = False # DRI: mo.li scale_a_matrix_with_useeio_method: bool = False # DRI: mo.li - scale_a_matrix_with_summary_tables: bool = False # DRI: mo.li - scale_a_matrix_with_commodity_price_index: bool = False # DRI: mo.li - load_useeio_nowcast_A_matrix: bool = False # DRI: mo.li adjust_summary_A_and_q_dollar_year: bool = False # DRI: mo.li ceda_margins: bool = False # DRI: WesIngwersen useeio_margins: bool = False # DRI: WesIngwersen @@ -100,7 +97,6 @@ class USAConfig(BaseModel): v0_3_umd_2024_ghgia: bool = False # DRI: catherine.birney use_ghg_national_2023_m2: bool = False ### Inflation factors - apply_inflation_to_V: bool = False # DRI: WesIngwersen update_inflation_factors: bool = False ##### diff --git a/bedrock/utils/economic/inflation_helpers_cornerstone.py b/bedrock/utils/economic/inflation_helpers_cornerstone.py index 6d4a9509..183460a8 100644 --- a/bedrock/utils/economic/inflation_helpers_cornerstone.py +++ b/bedrock/utils/economic/inflation_helpers_cornerstone.py @@ -267,21 +267,15 @@ def get_vnorm_adjusted_commodity_price_ratio( price levels; the two coincide when industry prices are uniform within a commodity's supplying mix. - V is inflated to ``cfg.model_base_year`` when ``cfg.apply_inflation_to_V`` - is set; the V-norm weights then reflect supplier mix at the model year - rather than at ``cfg.usa_base_io_data_year``. + V-norm weights reflect supplier mix at ``cfg.usa_base_io_data_year``. """ # local import to avoid a circular dependency on transform.eeio from bedrock.transform.eeio.derived_cornerstone import ( # noqa: PLC0415 derive_cornerstone_Vnorm_scrap_corrected, ) - cfg = get_usa_config() industry_ratio = get_cornerstone_industry_price_ratio(original_year, target_year) - Vnorm = derive_cornerstone_Vnorm_scrap_corrected( - apply_inflation=cfg.apply_inflation_to_V, - target_year=cfg.model_base_year, - ) + Vnorm = derive_cornerstone_Vnorm_scrap_corrected() aligned = industry_ratio.reindex(Vnorm.index, fill_value=1.0) # Normalize V_norm columns to sum to 1 so the dot-product is a true weighted