From deb80a708a007e24b914a742789bab058169d0c0 Mon Sep 17 00:00:00 2001 From: Giovanni Cozzolongo <79092266+giovannicozzolongo@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:39:30 +0200 Subject: [PATCH 1/3] Validate sector weather year Signed-off-by: Giovanni Cozzolongo <79092266+giovannicozzolongo@users.noreply.github.com> --- docs/source/config-sectors.md | 3 +++ tests/static/test_config_schema.py | 31 +++++++++++++++++++++++++++++ tests/static/test_dag_dryrun.py | 21 +++++++++++++++++++ workflow/schemas/config.schema.yaml | 16 +++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/docs/source/config-sectors.md b/docs/source/config-sectors.md index cfd32458..9591ebe8 100644 --- a/docs/source/config-sectors.md +++ b/docs/source/config-sectors.md @@ -6,6 +6,9 @@ the following configuration options are exposed to the user. ```{note} Only single-period studies are currently supported when running sector studies. +Set `renewable_weather_years: [2018]` for sector-coupled runs (`G` or `E-G`): +residential and commercial demand profiles use 2018 weather data. +Other weather years are rejected when the workflow loads the configuration. ``` ## Carbon Limits diff --git a/tests/static/test_config_schema.py b/tests/static/test_config_schema.py index d07f5e77..9c8d36f9 100644 --- a/tests/static/test_config_schema.py +++ b/tests/static/test_config_schema.py @@ -129,3 +129,34 @@ def test_godeeep_requires_renewable_land_access_key(schema): cfg.pop("renewable_land_access") with pytest.raises(jsonschema.ValidationError): jsonschema.validate(cfg, schema) + + +@pytest.mark.fast +@pytest.mark.parametrize("sector", ["G", "E-G"]) +@pytest.mark.parametrize("weather_years", [[2019], [2018, 2019], []]) +def test_sector_weather_year_must_be_2018(sector, weather_years, schema): + cfg = _merged("config.default.yaml") + cfg["scenario"]["sector"] = sector + cfg["renewable_weather_years"] = weather_years + with pytest.raises(jsonschema.ValidationError) as exc_info: + jsonschema.validate(cfg, schema) + assert list(exc_info.value.path) == ["renewable_weather_years"] + + +@pytest.mark.fast +@pytest.mark.parametrize("sector", ["G", "E-G"]) +def test_sector_weather_year_2018_is_supported(sector, schema): + cfg = _merged("config.default.yaml") + cfg["scenario"]["sector"] = sector + cfg["renewable_weather_years"] = [2018] + jsonschema.validate(cfg, schema) + + +@pytest.mark.fast +@pytest.mark.parametrize("sector", ["", "E"]) +@pytest.mark.parametrize("weather_years", [[2019], [2018, 2019]]) +def test_electricity_weather_years_remain_supported(sector, weather_years, schema): + cfg = _merged("config.default.yaml") + cfg["scenario"]["sector"] = sector + cfg["renewable_weather_years"] = weather_years + jsonschema.validate(cfg, schema) diff --git a/tests/static/test_dag_dryrun.py b/tests/static/test_dag_dryrun.py index 1666a7cd..62323df3 100644 --- a/tests/static/test_dag_dryrun.py +++ b/tests/static/test_dag_dryrun.py @@ -75,6 +75,27 @@ def test_snakemake_dryrun_resolves(configfile, target, overrides): ) +@pytest.mark.fast +@pytest.mark.parametrize("sector", ["G", "E-G"]) +def test_sector_weather_year_is_checked_at_workflow_start(tmp_path, sector): + configfile = tmp_path / "sector.yaml" + configfile.write_text(f"scenario:\n sector: {sector}\nrenewable_weather_years: [2019]\n") + result = subprocess.run( + ["snakemake", "--list", "--configfile", str(configfile)], + cwd=WORKFLOW_DIR, + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode != 0 + output = result.stdout + result.stderr + assert "Error validating config file" in output + assert "renewable_weather_years" in output + assert "Sector-coupled runs require" in output + assert "[2018]" in output + assert "[2019]" in output + + def _solve_network_inputs(overrides): """Return the ``solve_network`` input paths snakemake resolves in a dry run.""" cmd = [ diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index 746dc40b..555a237f 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -506,6 +506,22 @@ properties: eia: {type: ["string", "null"]} allOf: +# Residential and commercial sector demand profiles use 2018 weather data. +- if: + properties: + scenario: + properties: + sector: {pattern: "(^|-)G(-|$)"} + required: [sector] + required: [scenario] + then: + properties: + renewable_weather_years: + description: >- + Sector-coupled runs require renewable_weather_years: [2018]. + Residential and commercial demand profiles use 2018 weather data. + const: [2018] + # The selected solver's options block must exist in solver_options. - if: properties: From 190ae9fdf82896fd8f407bd577cd980a25fbed22 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 06:55:28 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 1 - tests/equivalence/build.py | 8 +-- tests/equivalence/compare.py | 4 +- tests/equivalence/conftest.py | 5 +- tests/equivalence/context.py | 8 +-- .../diagnostics/p_max_pu_quantiles.py | 63 +++++++++++++------ tests/equivalence/hotfixes.yaml | 28 ++++----- tests/equivalence/metrics.py | 5 +- tests/equivalence/paths.py | 11 +++- tests/equivalence/plots.py | 12 +++- .../test_demand_conservation_diagnostic.py | 23 ++++--- .../test_master_benchmark_branch.py | 7 ++- tests/equivalence/test_nomenclature.py | 12 +--- tests/equivalence/test_submit_benchmark.py | 2 +- tests/static/test_paths.py | 10 +-- workflow/report_benchmarks.py | 2 +- workflow/scripts/_helpers.py | 12 ++-- workflow/scripts/build_demand.py | 3 +- workflow/scripts/build_renewable_profiles.py | 3 +- workflow/scripts/plot_docs_figures.py | 14 +++-- .../scripts/test/test_cluster_wildcard.py | 2 +- workflow/scripts/test/test_policy.py | 1 - 22 files changed, 131 insertions(+), 105 deletions(-) diff --git a/.gitignore b/.gitignore index 2218656c..f7bf973c 100644 --- a/.gitignore +++ b/.gitignore @@ -312,4 +312,3 @@ slurm-*.err !workflow/repo_data/** !workflow/geodata_repo/** !natura.tiff - diff --git a/tests/equivalence/build.py b/tests/equivalence/build.py index 3ea66184..980410ae 100644 --- a/tests/equivalence/build.py +++ b/tests/equivalence/build.py @@ -122,7 +122,7 @@ def resolve_baseline_sha() -> str: "create it first (T1 of memory/plans/harness-master-vs-develop.md — " "`git switch -c master-benchmark master`, then port the documented " "commits), or set EQ_BASELINE_REF to an existing ref.\n" - f"git said: {(cp.stderr or cp.stdout).strip()[-500:]}" + f"git said: {(cp.stderr or cp.stdout).strip()[-500:]}", ) return sha @@ -175,7 +175,7 @@ def provision_baseline_worktree() -> Path: f"it points at {_foreign_gitdir(wt)}. This is a leftover from a " "previous checkout location — running git against it would target " "another repository. Remove or rename it, then re-run; refusing to " - "adopt it." + "adopt it.", ) if key in registered: @@ -292,14 +292,14 @@ def assert_clean_checkout(side: str, root: Path) -> list[str]: if os.environ.get("EQ_ALLOW_DIRTY") == "1": log( f"WARNING: {side} checkout {root} has uncommitted changes to tracked files; " - f"building anyway because EQ_ALLOW_DIRTY=1:\n{listing}{more}" + f"building anyway because EQ_ALLOW_DIRTY=1:\n{listing}{more}", ) return dirt raise RuntimeError( f"{side} checkout {root} has uncommitted changes to tracked files, so the sha " f"its manifest records would not describe the code that ran:\n{listing}{more}\n" "Commit or stash them, or set EQ_ALLOW_DIRTY=1 to build anyway and have the " - "manifest record dirty: true." + "manifest record dirty: true.", ) diff --git a/tests/equivalence/compare.py b/tests/equivalence/compare.py index 74293600..81332b7c 100644 --- a/tests/equivalence/compare.py +++ b/tests/equivalence/compare.py @@ -818,8 +818,8 @@ def compare_profiles( "detail": { "hours_compared": int(n), "hours_mismatched": int(bad.sum()), - "len_develop": int(len(sc)), - "len_master": int(len(sa)), + "len_develop": len(sc), + "len_master": len(sa), # Per-hour relative errors. ``worst_rel_pct`` is # dominated by dawn/dusk hours where master is a # few MW, so a fraction of a MW reads as 100 %; diff --git a/tests/equivalence/conftest.py b/tests/equivalence/conftest.py index 19b4d496..a21b4b32 100644 --- a/tests/equivalence/conftest.py +++ b/tests/equivalence/conftest.py @@ -71,10 +71,7 @@ def make_network( sns = n.snapshots if solved: gen_p = pd.DataFrame( - { - name: np.full(len(sns), 10.0 * (i + 1) * scale) - for i, name in enumerate(n.generators.index) - }, + {name: np.full(len(sns), 10.0 * (i + 1) * scale) for i, name in enumerate(n.generators.index)}, index=sns, ) n.generators_t["p"] = gen_p diff --git a/tests/equivalence/context.py b/tests/equivalence/context.py index 9e438001..618d6c20 100644 --- a/tests/equivalence/context.py +++ b/tests/equivalence/context.py @@ -362,8 +362,7 @@ def merged_config(side: str) -> dict: if not out.exists(): tail = "\n".join((cp.stderr or cp.stdout).splitlines()[-60:]) raise RuntimeError( - f"could not dump the merged config for side {side!r} from {wf} " - f"(snakemake exit {cp.returncode}).\n{tail}", + f"could not dump the merged config for side {side!r} from {wf} (snakemake exit {cp.returncode}).\n{tail}", ) return json.loads(out.read_text()) @@ -491,8 +490,7 @@ def _is_disabled_block(v: object) -> bool: ), ( "plotting", - "master-only plot axis limits and thresholds; no plotting rule is in the benchmark target " - "chain on either side", + "master-only plot axis limits and thresholds; no plotting rule is in the benchmark target chain on either side", ), # --- opt-gated, and this run's opts do not select them ------------------- ( @@ -552,7 +550,7 @@ def _is_disabled_block(v: object) -> bool: ), ( "run.benchmark_cpuc_horizons", - "read only when run.benchmark_cpuc is true, which is itself develop-only and false " "(HF-21)", + "read only when run.benchmark_cpuc is true, which is itself develop-only and false (HF-21)", ), # --- values that match the other side's inline fallback ------------------ ( diff --git a/tests/equivalence/diagnostics/p_max_pu_quantiles.py b/tests/equivalence/diagnostics/p_max_pu_quantiles.py index 4b2cf84c..68fda270 100644 --- a/tests/equivalence/diagnostics/p_max_pu_quantiles.py +++ b/tests/equivalence/diagnostics/p_max_pu_quantiles.py @@ -28,11 +28,11 @@ import matplotlib matplotlib.use("Agg") -import matplotlib.pyplot as plt # noqa: E402 -import numpy as np # noqa: E402 -import pandas as pd # noqa: E402 -import pypsa # noqa: E402 -import xarray as xr # noqa: E402 +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import pypsa +import xarray as xr REPO = Path(__file__).resolve().parents[3] sys.path.insert(0, str(REPO)) @@ -76,7 +76,9 @@ def master_own_s300(tech: str) -> pd.DataFrame: ts = n.generators_t.p_max_pu cols = [g for g in gens.index if g in ts.columns] log.info("master elec_s300 %s: %d generators, %d with a p_max_pu series", tech, len(gens), len(cols)) - da = xr.DataArray(ts[cols].to_numpy().T, dims=("bus", "time"), coords={"bus": gens.loc[cols, "bus"].astype(str).values}) + da = xr.DataArray( + ts[cols].to_numpy().T, dims=("bus", "time"), coords={"bus": gens.loc[cols, "bus"].astype(str).values} + ) return quantiles_over_time(da) @@ -104,7 +106,12 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da dropped = int(ds_rec.attrs.get("eq_dropped_buses", 0)) log.info( "%s: %d common clusters (develop %d, recon %d, master own %d); master nodal buses absent from busmap: %d", - tech, len(paired), len(q_dev), len(q_rec), len(q_own), dropped, + tech, + len(paired), + len(q_dev), + len(q_rec), + len(q_own), + dropped, ) fig, axes = plt.subplots(3, 3, figsize=(15, 11.5), facecolor="#fcfcfb") @@ -112,7 +119,8 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da f"USA s300 — {tech} p_max_pu per resource cluster, master vs develop\n" f"paired view: master's nodal profile rolled onto develop's {len(paired)} clusters " f"(busmap_s300, p_nom_max-weighted = develop's rule, HF-25; HF-24-dropped substations absent)", - fontsize=12, color=INK, + fontsize=12, + color=INK, ) summary_rows = [] for i, q in enumerate(Q): @@ -141,11 +149,16 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da rmse = float(np.sqrt((d**2).mean())) within = int((d.abs() <= 0.005).sum()) ax.text( - 0.03, 0.97, + 0.03, + 0.97, f"n = {len(d)}\nmax |Δ| = {d.abs().max():.4f}\nRMSE = {rmse:.5f}\n|Δ| ≤ 0.005: {within}/{len(d)}", - transform=ax.transAxes, va="top", fontsize=9, color=INK, + transform=ax.transAxes, + va="top", + fontsize=9, + color=INK, ) - ax.set_xlim(lim); ax.set_ylim(lim) + ax.set_xlim(lim) + ax.set_ylim(lim) ax.set_xlabel(f"develop p{q}", color=INK) ax.set_ylabel(f"master, reconstructed p{q}", color=INK) ax.set_title(f"p{q}: paired per cluster after reconstruction", fontsize=10, color=INK) @@ -157,19 +170,33 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da ax.axhspan(-0.005, 0.005, color=GRID, alpha=0.6, zorder=0) worst = ds.abs().sort_values(ascending=False).head(3) ax.text( - 0.03, 0.97, + 0.03, + 0.97, "largest |Δ|: " + ", ".join(f"{c} ({paired.loc[c, 'reeds_zone']}) {ds[c]:+.4f}" for c in worst.index), - transform=ax.transAxes, va="top", fontsize=8, color=INK, wrap=True, + transform=ax.transAxes, + va="top", + fontsize=8, + color=INK, + wrap=True, ) ax.set_xlabel("clusters, ranked by Δ", color=MUTED) ax.set_ylabel(f"Δ p{q} = master recon − develop", color=INK) ax.set_title(f"p{q}: residual per cluster (band = ±0.005)", fontsize=10, color=INK) summary_rows.append( - dict(tech=tech, quantile=f"p{q}", n_clusters=len(d), max_abs_delta=float(d.abs().max()), - rmse=rmse, median_abs_delta=float(d.abs().median()), n_within_0p005=within, - develop_median=float(x.median()), master_recon_median=float(y.median()), - master_own_s300_median=float(q_own[col].median()), master_own_n=len(q_own), - master_nodal_buses_dropped=dropped) + dict( + tech=tech, + quantile=f"p{q}", + n_clusters=len(d), + max_abs_delta=float(d.abs().max()), + rmse=rmse, + median_abs_delta=float(d.abs().median()), + n_within_0p005=within, + develop_median=float(x.median()), + master_recon_median=float(y.median()), + master_own_s300_median=float(q_own[col].median()), + master_own_n=len(q_own), + master_nodal_buses_dropped=dropped, + ), ) for ax in axes.ravel(): ax.set_facecolor("#fcfcfb") diff --git a/tests/equivalence/hotfixes.yaml b/tests/equivalence/hotfixes.yaml index 0a383c48..6d0c0f73 100644 --- a/tests/equivalence/hotfixes.yaml +++ b/tests/equivalence/hotfixes.yaml @@ -122,7 +122,7 @@ pipelines. Still listed because the pin, not a port, is what closes it. - id: HF-7 - commit: 51e1719 + commit: .inf pr: 794 title: renewable_land_access default null -> reference (NREL reV land-exclusion screening of the GODEEEP CFs) confidence: high @@ -319,7 +319,7 @@ cardinality. - id: HF-20 - commit: 51e1719 + commit: .inf pr: 794 title: config.default.yaml becomes a loaded base layer beneath user configs, scenario configs become sparse overlays, and the merged config is schema-validated at parse time confidence: medium @@ -352,7 +352,7 @@ - id: HF-22 commit: e1f5841 - pr: null + pr: title: build_renewable_profiles._drop_leap_day — Feb 29 dropped from the selection window so a leap weather year yields the standard 8760-hour axis confidence: none ported: true @@ -385,7 +385,7 @@ - id: HF-24 commit: 3b149ae - pr: null + pr: title: build_renewable_profiles keeps NREL caps capacity for substations with zero GODEEEP availability or no intersecting cell, instead of silently dropping it confidence: high ported: false @@ -415,7 +415,7 @@ - id: HF-25 commit: 3b149ae - pr: null + pr: title: "renewable.godeeep_cf_weighting: develop weights substation capacity factors up to the s{simpl} cluster by INSTALLABLE capacity (NREL caps p_nom_max); master weights them by EXISTING p_nom and falls back to an unweighted mean where a cluster has none" confidence: high ported: false @@ -451,17 +451,17 @@ - id: HF-26 commit: a5ed78c - pr: null + pr: title: "attach_renewable_capacities_to_atlite runs at CLUSTER granularity on develop, so existing renewable plants at substations with no profile generator keep their MW; on master the same function runs nodally and drops them" confidence: high ported: false usa_noop: false expect: - - capacity_existing_by_carrier/* - - p_nom_existing_by_zone_carrier/* - - objective - - capacity_opt_by_carrier/* - - dispatch_by_carrier/* + - capacity_existing_by_carrier/* + - p_nom_existing_by_zone_carrier/* + - objective + - capacity_opt_by_carrier/* + - dispatch_by_carrier/* master: >- the assembled network is nodal, so `mapped_values = generators_tech.sub_assignment.map(caps_per_bus).dropna()` in @@ -496,7 +496,7 @@ - id: HF-27 commit: 382535b - pr: null + pr: title: "plant -> bus assignment: a DEVELOP REGRESSION, now FIXED — after simplify-early develop matched plants to the {simpl} cluster CENTROIDS (median 37 km) instead of to substations (median 1.0 km, master's own bus set), putting 6.1 % of existing MW in the wrong ReEDS zone; develop now matches substations and maps through busmap_s{simpl}" confidence: high ported: false @@ -542,7 +542,7 @@ - id: HF-28 commit: 01f742f - pr: null + pr: title: "demand disaggregation key: master splits EFS state demand over NODAL buses keyed on `buses.state`, develop over s{simpl} buses keyed on `buses.reeds_state`; the LAF both sides consume was normalised over a THIRD column (`full_state`), so neither conserved state demand until the renormalisation landed" confidence: high ported: false @@ -589,7 +589,7 @@ - id: HF-29 commit: 2a3adf8 - pr: null + pr: title: "LAF renormalised inside the demand key at consumption time, on BOTH branches (state demand conserved: was +2.28 % master, +1.85 % develop); plus a DC-column fold into Maryland that is skipped when a bus carries the key" confidence: none ported: true diff --git a/tests/equivalence/metrics.py b/tests/equivalence/metrics.py index 1b69f66d..df3a6aad 100644 --- a/tests/equivalence/metrics.py +++ b/tests/equivalence/metrics.py @@ -430,8 +430,7 @@ def aggregate_profile_to_clusters( n_dropped = int((~keep).sum()) if n_dropped: logger.warning( - "aggregate_profile_to_clusters: %d of %d buses are absent from the busmap " - "and were dropped (first few: %s)", + "aggregate_profile_to_clusters: %d of %d buses are absent from the busmap and were dropped (first few: %s)", n_dropped, len(buses), ", ".join(str(b) for b in buses[~keep][:5]), @@ -497,7 +496,7 @@ def _weights(name: str) -> pd.Series: out_ds = xr.Dataset(data_vars, coords=coords, attrs=dict(ds.attrs)) out_ds.attrs["eq_dropped_buses"] = n_dropped out_ds.attrs["eq_zero_weight_clusters"] = n_zero_weight - out_ds.attrs["eq_aggregated_from_buses"] = int(len(buses)) + out_ds.attrs["eq_aggregated_from_buses"] = len(buses) return out_ds diff --git a/tests/equivalence/paths.py b/tests/equivalence/paths.py index 7d97c843..db4ffdea 100644 --- a/tests/equivalence/paths.py +++ b/tests/equivalence/paths.py @@ -39,7 +39,9 @@ # under config/, and build.py copies the shared harness config in there. CONFIGFILE = f"repo_data/config/{_CONFIG_NAME}" BASELINE_CONFIGFILE = f"config/{_CONFIG_NAME}" -CLUSTERS = os.environ.get("EQ_CLUSTERS", "4") # reeds transport: must equal the footprint's ReEDS zone count (western/CA slice 4; usa 134) +CLUSTERS = os.environ.get( + "EQ_CLUSTERS", "4" +) # reeds transport: must equal the footprint's ReEDS zone count (western/CA slice 4; usa 134) LL = "v1.0" @@ -148,7 +150,11 @@ def _profile_horizon_dir() -> str: import yaml cfg_path = os.path.join( - os.path.dirname(__file__), "..", "..", "workflow", CONFIGFILE + os.path.dirname(__file__), + "..", + "..", + "workflow", + CONFIGFILE, ) with open(cfg_path) as f: cfg = yaml.safe_load(f) @@ -156,7 +162,6 @@ def _profile_horizon_dir() -> str: return "" if scenarios[0] == "historical" else f"{HORIZON}/" - @dataclass(frozen=True) class ArtifactPair: """One comparable artifact across the two sides.""" diff --git a/tests/equivalence/plots.py b/tests/equivalence/plots.py index 8cd7e4a8..ebfd0fc5 100644 --- a/tests/equivalence/plots.py +++ b/tests/equivalence/plots.py @@ -875,12 +875,20 @@ def export_all( objective_figure(m.get("objective"), "objective", run_dir) paired_bar( - m.get("capacity_existing_by_carrier"), "existing capacity", "MW", "capacity_existing_by_carrier", run_dir + m.get("capacity_existing_by_carrier"), + "existing capacity", + "MW", + "capacity_existing_by_carrier", + run_dir, ) paired_bar(m.get("capacity_opt_by_carrier"), "optimised capacity", "MW", "capacity_opt_by_carrier", run_dir) paired_bar(m.get("dispatch_by_carrier"), "annual dispatch", "MWh", "dispatch_by_carrier", run_dir) paired_bar( - m.get("capacity_factor_by_carrier"), "realised capacity factor", "-", "capacity_factor_by_carrier", run_dir + m.get("capacity_factor_by_carrier"), + "realised capacity factor", + "-", + "capacity_factor_by_carrier", + run_dir, ) paired_bar(m.get("demand_by_zone"), "demand by zone", "MW", "demand_zones", run_dir) diff --git a/tests/equivalence/test_demand_conservation_diagnostic.py b/tests/equivalence/test_demand_conservation_diagnostic.py index bd89993a..1e4fc63c 100644 --- a/tests/equivalence/test_demand_conservation_diagnostic.py +++ b/tests/equivalence/test_demand_conservation_diagnostic.py @@ -137,16 +137,19 @@ def test_main_writes_a_png_and_its_csv_twin(tmp_path, capsys): make_table(tmp_path / "d.parquet", {"Maryland": 200.0, "Virginia": 150.0}) outdir = tmp_path / "out" - assert main( - [ - "--network", - str(tmp_path / "n.nc"), - "--demand-table", - str(tmp_path / "d.parquet"), - "--outdir", - str(outdir), - ], - ) == 0 + assert ( + main( + [ + "--network", + str(tmp_path / "n.nc"), + "--demand-table", + str(tmp_path / "d.parquet"), + "--outdir", + str(outdir), + ], + ) + == 0 + ) assert (outdir / "demand_conservation.png").exists() written = pd.read_csv(outdir / "demand_conservation.csv", index_col="demand_key") diff --git a/tests/equivalence/test_master_benchmark_branch.py b/tests/equivalence/test_master_benchmark_branch.py index 804e1d31..7a687c12 100644 --- a/tests/equivalence/test_master_benchmark_branch.py +++ b/tests/equivalence/test_master_benchmark_branch.py @@ -174,8 +174,7 @@ def test_ported_commits_name_a_hotfix(commits): hf = c["trailers"].get("Hot-fix", "") if not re.fullmatch(r"HF-\d+", hf): bad.append( - f"{c['sha'][:7]} {c['subject']!r}: Hot-fix={hf!r}, expected 'HF-' " - "(bare integer, not zero-padded)", + f"{c['sha'][:7]} {c['subject']!r}: Hot-fix={hf!r}, expected 'HF-' (bare integer, not zero-padded)", ) assert not bad, "hot-fix trailer violations:\n" + "\n".join(bad) @@ -283,7 +282,9 @@ def _manifest_table_shas(branch: str) -> list[str]: start = text.index("## Commits") end = text.find("\n## ", start + 1) table = text[start:end] if end != -1 else text[start:] - return [m.group(1) for line in table.splitlines() if line.lstrip().startswith("|") for m in _SHA_TOKEN.finditer(line)] + return [ + m.group(1) for line in table.splitlines() if line.lstrip().startswith("|") for m in _SHA_TOKEN.finditer(line) + ] def test_branch_manifest_matches(branch, commits): diff --git a/tests/equivalence/test_nomenclature.py b/tests/equivalence/test_nomenclature.py index 6bf2e71f..14243028 100644 --- a/tests/equivalence/test_nomenclature.py +++ b/tests/equivalence/test_nomenclature.py @@ -53,12 +53,7 @@ def _scanned_files() -> list[Path]: - files = sorted( - p - for ext in ("py", "yaml", "sbatch") - for p in HARNESS.glob(f"*.{ext}") - if p.name != _SELF - ) + files = sorted(p for ext in ("py", "yaml", "sbatch") for p in HARNESS.glob(f"*.{ext}") if p.name != _SELF) files += sorted((REPO / "workflow" / "repo_data" / "config").glob("config.equivalence*.yaml")) files.append(REPO / "CONTEXT.md") return [p for p in files if p.exists()] @@ -85,10 +80,7 @@ def test_no_anchor_nomenclature(): for lineno, line in enumerate(block.splitlines(), start=1): if "anchor" in _ALLOWED.sub("", line).lower(): hits.append(f"pyproject.toml [tool.pytest.ini_options]+{lineno}: {line.strip()}") - assert not hits, ( - "retired 'anchor' nomenclature found; the baseline is master-benchmark:\n" - + "\n".join(hits) - ) + assert not hits, "retired 'anchor' nomenclature found; the baseline is master-benchmark:\n" + "\n".join(hits) @pytest.mark.fast diff --git a/tests/equivalence/test_submit_benchmark.py b/tests/equivalence/test_submit_benchmark.py index f82d288c..2f2591f8 100644 --- a/tests/equivalence/test_submit_benchmark.py +++ b/tests/equivalence/test_submit_benchmark.py @@ -27,7 +27,7 @@ def fake_sbatch(tmp_path: Path) -> Path: f"n=$(( $(grep -c '^ARGS' '{log}' 2>/dev/null || echo 0) + 1 ))\n" f"echo \"ARGS $*\" >> '{log}'\n" f"echo \"ENV EQ_INTERCONNECT=$EQ_INTERCONNECT EQ_UNTIL=$EQ_UNTIL EQ_RUN_ID=$EQ_RUN_ID EXTRA=$EQ_EXTRA_ARGS\" >> '{log}'\n" - 'echo "1000$n"\n' + 'echo "1000$n"\n', ) shim.chmod(shim.stat().st_mode | stat.S_IEXEC) return log diff --git a/tests/static/test_paths.py b/tests/static/test_paths.py index 67e3a60f..91920d57 100644 --- a/tests/static/test_paths.py +++ b/tests/static/test_paths.py @@ -96,9 +96,7 @@ def test_exactly_one_sbatch_driver(): def test_sbatch_uses_the_serc_partition(sbatch_path): """The project's partition is serc (PROJECT.md §4), not normal.""" source = sbatch_path.read_text() - assert re.search(r"^#SBATCH\s+-p\s+serc\b", source, re.MULTILINE), ( - f"{sbatch_path.name} does not request -p serc" - ) + assert re.search(r"^#SBATCH\s+-p\s+serc\b", source, re.MULTILINE), f"{sbatch_path.name} does not request -p serc" assert not re.search(r"^#SBATCH\s+-p\s+normal\b", source, re.MULTILINE), ( f"{sbatch_path.name} still requests -p normal" ) @@ -134,9 +132,5 @@ def test_sbatch_log_directives_are_relative(sbatch_path): an absolute site path is one someone else will not have. Relative means the submit directory, which by construction exists. """ - bad = [ - line.strip() - for line in sbatch_path.read_text().splitlines() - if re.match(r"^#SBATCH\s+-[oe]\s+/", line) - ] + bad = [line.strip() for line in sbatch_path.read_text().splitlines() if re.match(r"^#SBATCH\s+-[oe]\s+/", line)] assert not bad, f"{sbatch_path.name} writes Slurm logs to an absolute path: {bad}" diff --git a/workflow/report_benchmarks.py b/workflow/report_benchmarks.py index e10344bd..4d89eca5 100644 --- a/workflow/report_benchmarks.py +++ b/workflow/report_benchmarks.py @@ -129,7 +129,7 @@ def rec_time(peak_s: float) -> str: agg["peak_time"] = agg.peak_s.map(fmt_hms) print( - f"{'RULE':<44} {'N':>2} {'PEAK_RSS_MB':>11} {'PEAK_TIME':>10} {'LOAD%':>6} {'REC_MEM_MB':>10} {'REC_WALLTIME':>12}" + f"{'RULE':<44} {'N':>2} {'PEAK_RSS_MB':>11} {'PEAK_TIME':>10} {'LOAD%':>6} {'REC_MEM_MB':>10} {'REC_WALLTIME':>12}", ) print("-" * 108) for name, r in agg.iterrows(): diff --git a/workflow/scripts/_helpers.py b/workflow/scripts/_helpers.py index 9e95af8f..00ba3583 100644 --- a/workflow/scripts/_helpers.py +++ b/workflow/scripts/_helpers.py @@ -346,9 +346,9 @@ def mock_snakemake(rulename, **wildcards): from snakemake.script import Snakemake script_dir = Path(__file__).parent.resolve() - assert ( - Path.cwd().resolve() == script_dir - ), f"mock_snakemake has to be run from the repository scripts directory {script_dir}" + assert Path.cwd().resolve() == script_dir, ( + f"mock_snakemake has to be run from the repository scripts directory {script_dir}" + ) os.chdir(script_dir.parent) for p in sm.SNAKEFILE_CHOICES: if os.path.exists(p): @@ -429,9 +429,9 @@ def validate_checksum(file_path, zenodo_url=None, checksum=None): for chunk in iter(lambda: f.read(65536), b""): # 64kb chunks hasher.update(chunk) calculated_checksum = hasher.hexdigest() - assert ( - calculated_checksum == checksum - ), "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule." + assert calculated_checksum == checksum, ( + "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule." + ) def get_checksum_from_zenodo(file_url): diff --git a/workflow/scripts/build_demand.py b/workflow/scripts/build_demand.py index 26683bf5..f8afcc25 100644 --- a/workflow/scripts/build_demand.py +++ b/workflow/scripts/build_demand.py @@ -2117,8 +2117,7 @@ def _log_unallocated_demand(demand: pd.DataFrame, zone_data: pd.DataFrame) -> No if not orphans: return logger.warning( - "No bus carries demand key(s) %s; %.1f MW of mean demand is DROPPED. " - "Per key: %s.", + "No bus carries demand key(s) %s; %.1f MW of mean demand is DROPPED. Per key: %s.", sorted(orphans), sum(orphans.values()), {k: round(v, 1) for k, v in sorted(orphans.items())}, diff --git a/workflow/scripts/build_renewable_profiles.py b/workflow/scripts/build_renewable_profiles.py index c101fa28..2ee99812 100644 --- a/workflow/scripts/build_renewable_profiles.py +++ b/workflow/scripts/build_renewable_profiles.py @@ -529,8 +529,7 @@ def plot_data(data): cache_dir=snakemake.params.mapping_cache_dir, ) logger.info( - f"Cell→substation mapping: {mapping_sub['name'].nunique()} substations, " - f"{len(mapping_sub)} cell rows", + f"Cell→substation mapping: {mapping_sub['name'].nunique()} substations, {len(mapping_sub)} cell rows", ) agg = capacity_weighted_bus_aggregation( ds_cf["capacity_factor"], diff --git a/workflow/scripts/plot_docs_figures.py b/workflow/scripts/plot_docs_figures.py index 8fed63e3..15d723f7 100644 --- a/workflow/scripts/plot_docs_figures.py +++ b/workflow/scripts/plot_docs_figures.py @@ -159,7 +159,12 @@ def _generator_legend(fig, title): handles = [ plt.Line2D([], [], marker="o", linestyle="", color=GEN_COLORS["conventional"], label="conventional generator"), plt.Line2D( - [], [], marker="o", linestyle="", color=GEN_COLORS["renewable"], label="renewable / other generator" + [], + [], + marker="o", + linestyle="", + color=GEN_COLORS["renewable"], + label="renewable / other generator", ), ] fig.legend(handles=handles, loc="lower center", ncol=2, frameon=False, fontsize=9, title=title, title_fontsize=8) @@ -219,8 +224,7 @@ def plot_cluster_suffixes(simpl_path, suffix_paths, shapes_path, out_path, conve ) fig.suptitle( - f"The same {n_simpl_buses}-zone {{simpl}} network clustered to " - f"{len(buses)} zones by each {{clusters}} suffix", + f"The same {n_simpl_buses}-zone {{simpl}} network clustered to {len(buses)} zones by each {{clusters}} suffix", fontsize=11, ) _generator_legend( @@ -280,7 +284,9 @@ def plot_simpl_resolutions(panels, shapes_path, out_path, conventional_carriers= "The same footprint at each {simpl} resolution, clustered with the a suffix (nothing aggregated)", fontsize=11, ) - _generator_legend(fig, "marker area ∝ p_nom (capped); every generator drawn at its {simpl} zone (nothing aggregated)") + _generator_legend( + fig, "marker area ∝ p_nom (capped); every generator drawn at its {simpl} zone (nothing aggregated)" + ) fig.tight_layout(rect=(0, 0.09, 1, 0.92)) fig.savefig(out_path, dpi=150, bbox_inches="tight", facecolor="white") plt.close(fig) diff --git a/workflow/scripts/test/test_cluster_wildcard.py b/workflow/scripts/test/test_cluster_wildcard.py index 05563496..9e49355e 100644 --- a/workflow/scripts/test/test_cluster_wildcard.py +++ b/workflow/scripts/test/test_cluster_wildcard.py @@ -7,7 +7,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from cluster_network import parse_clusters_wildcard # noqa: E402 +from cluster_network import parse_clusters_wildcard ALL = {"onwind", "solar", "CCGT", "coal", "nuclear"} CONV = {"CCGT", "coal", "nuclear"} diff --git a/workflow/scripts/test/test_policy.py b/workflow/scripts/test/test_policy.py index f3949c3a..92dbf60f 100644 --- a/workflow/scripts/test/test_policy.py +++ b/workflow/scripts/test/test_policy.py @@ -568,4 +568,3 @@ def test_add_rps_constraints_zone_without_eligible_gens_does_not_stop_others(pol rps = [c for c in n.model.constraints if c.endswith("_rps_limit")] assert rps, "the non-empty zone lost its constraint" assert not any(c.startswith("GlobalConstraint-CA") for c in rps) - From 50b8830c557615a48e88f94b6204323789ed8fef Mon Sep 17 00:00:00 2001 From: Kamran Date: Mon, 28 Sep 2026 11:20:59 -0700 Subject: [PATCH 3/3] Revert "[pre-commit.ci] auto fixes from pre-commit.com hooks" This reverts commit 190ae9fdf82896fd8f407bd577cd980a25fbed22. --- .gitignore | 1 + tests/equivalence/build.py | 8 +-- tests/equivalence/compare.py | 4 +- tests/equivalence/conftest.py | 5 +- tests/equivalence/context.py | 8 ++- .../diagnostics/p_max_pu_quantiles.py | 63 ++++++------------- tests/equivalence/hotfixes.yaml | 28 ++++----- tests/equivalence/metrics.py | 5 +- tests/equivalence/paths.py | 11 +--- tests/equivalence/plots.py | 12 +--- .../test_demand_conservation_diagnostic.py | 23 +++---- .../test_master_benchmark_branch.py | 7 +-- tests/equivalence/test_nomenclature.py | 12 +++- tests/equivalence/test_submit_benchmark.py | 2 +- tests/static/test_paths.py | 10 ++- workflow/report_benchmarks.py | 2 +- workflow/scripts/_helpers.py | 12 ++-- workflow/scripts/build_demand.py | 3 +- workflow/scripts/build_renewable_profiles.py | 3 +- workflow/scripts/plot_docs_figures.py | 14 ++--- .../scripts/test/test_cluster_wildcard.py | 2 +- workflow/scripts/test/test_policy.py | 1 + 22 files changed, 105 insertions(+), 131 deletions(-) diff --git a/.gitignore b/.gitignore index f7bf973c..2218656c 100644 --- a/.gitignore +++ b/.gitignore @@ -312,3 +312,4 @@ slurm-*.err !workflow/repo_data/** !workflow/geodata_repo/** !natura.tiff + diff --git a/tests/equivalence/build.py b/tests/equivalence/build.py index 980410ae..3ea66184 100644 --- a/tests/equivalence/build.py +++ b/tests/equivalence/build.py @@ -122,7 +122,7 @@ def resolve_baseline_sha() -> str: "create it first (T1 of memory/plans/harness-master-vs-develop.md — " "`git switch -c master-benchmark master`, then port the documented " "commits), or set EQ_BASELINE_REF to an existing ref.\n" - f"git said: {(cp.stderr or cp.stdout).strip()[-500:]}", + f"git said: {(cp.stderr or cp.stdout).strip()[-500:]}" ) return sha @@ -175,7 +175,7 @@ def provision_baseline_worktree() -> Path: f"it points at {_foreign_gitdir(wt)}. This is a leftover from a " "previous checkout location — running git against it would target " "another repository. Remove or rename it, then re-run; refusing to " - "adopt it.", + "adopt it." ) if key in registered: @@ -292,14 +292,14 @@ def assert_clean_checkout(side: str, root: Path) -> list[str]: if os.environ.get("EQ_ALLOW_DIRTY") == "1": log( f"WARNING: {side} checkout {root} has uncommitted changes to tracked files; " - f"building anyway because EQ_ALLOW_DIRTY=1:\n{listing}{more}", + f"building anyway because EQ_ALLOW_DIRTY=1:\n{listing}{more}" ) return dirt raise RuntimeError( f"{side} checkout {root} has uncommitted changes to tracked files, so the sha " f"its manifest records would not describe the code that ran:\n{listing}{more}\n" "Commit or stash them, or set EQ_ALLOW_DIRTY=1 to build anyway and have the " - "manifest record dirty: true.", + "manifest record dirty: true." ) diff --git a/tests/equivalence/compare.py b/tests/equivalence/compare.py index 81332b7c..74293600 100644 --- a/tests/equivalence/compare.py +++ b/tests/equivalence/compare.py @@ -818,8 +818,8 @@ def compare_profiles( "detail": { "hours_compared": int(n), "hours_mismatched": int(bad.sum()), - "len_develop": len(sc), - "len_master": len(sa), + "len_develop": int(len(sc)), + "len_master": int(len(sa)), # Per-hour relative errors. ``worst_rel_pct`` is # dominated by dawn/dusk hours where master is a # few MW, so a fraction of a MW reads as 100 %; diff --git a/tests/equivalence/conftest.py b/tests/equivalence/conftest.py index a21b4b32..19b4d496 100644 --- a/tests/equivalence/conftest.py +++ b/tests/equivalence/conftest.py @@ -71,7 +71,10 @@ def make_network( sns = n.snapshots if solved: gen_p = pd.DataFrame( - {name: np.full(len(sns), 10.0 * (i + 1) * scale) for i, name in enumerate(n.generators.index)}, + { + name: np.full(len(sns), 10.0 * (i + 1) * scale) + for i, name in enumerate(n.generators.index) + }, index=sns, ) n.generators_t["p"] = gen_p diff --git a/tests/equivalence/context.py b/tests/equivalence/context.py index 618d6c20..9e438001 100644 --- a/tests/equivalence/context.py +++ b/tests/equivalence/context.py @@ -362,7 +362,8 @@ def merged_config(side: str) -> dict: if not out.exists(): tail = "\n".join((cp.stderr or cp.stdout).splitlines()[-60:]) raise RuntimeError( - f"could not dump the merged config for side {side!r} from {wf} (snakemake exit {cp.returncode}).\n{tail}", + f"could not dump the merged config for side {side!r} from {wf} " + f"(snakemake exit {cp.returncode}).\n{tail}", ) return json.loads(out.read_text()) @@ -490,7 +491,8 @@ def _is_disabled_block(v: object) -> bool: ), ( "plotting", - "master-only plot axis limits and thresholds; no plotting rule is in the benchmark target chain on either side", + "master-only plot axis limits and thresholds; no plotting rule is in the benchmark target " + "chain on either side", ), # --- opt-gated, and this run's opts do not select them ------------------- ( @@ -550,7 +552,7 @@ def _is_disabled_block(v: object) -> bool: ), ( "run.benchmark_cpuc_horizons", - "read only when run.benchmark_cpuc is true, which is itself develop-only and false (HF-21)", + "read only when run.benchmark_cpuc is true, which is itself develop-only and false " "(HF-21)", ), # --- values that match the other side's inline fallback ------------------ ( diff --git a/tests/equivalence/diagnostics/p_max_pu_quantiles.py b/tests/equivalence/diagnostics/p_max_pu_quantiles.py index 68fda270..4b2cf84c 100644 --- a/tests/equivalence/diagnostics/p_max_pu_quantiles.py +++ b/tests/equivalence/diagnostics/p_max_pu_quantiles.py @@ -28,11 +28,11 @@ import matplotlib matplotlib.use("Agg") -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import pypsa -import xarray as xr +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 +import pypsa # noqa: E402 +import xarray as xr # noqa: E402 REPO = Path(__file__).resolve().parents[3] sys.path.insert(0, str(REPO)) @@ -76,9 +76,7 @@ def master_own_s300(tech: str) -> pd.DataFrame: ts = n.generators_t.p_max_pu cols = [g for g in gens.index if g in ts.columns] log.info("master elec_s300 %s: %d generators, %d with a p_max_pu series", tech, len(gens), len(cols)) - da = xr.DataArray( - ts[cols].to_numpy().T, dims=("bus", "time"), coords={"bus": gens.loc[cols, "bus"].astype(str).values} - ) + da = xr.DataArray(ts[cols].to_numpy().T, dims=("bus", "time"), coords={"bus": gens.loc[cols, "bus"].astype(str).values}) return quantiles_over_time(da) @@ -106,12 +104,7 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da dropped = int(ds_rec.attrs.get("eq_dropped_buses", 0)) log.info( "%s: %d common clusters (develop %d, recon %d, master own %d); master nodal buses absent from busmap: %d", - tech, - len(paired), - len(q_dev), - len(q_rec), - len(q_own), - dropped, + tech, len(paired), len(q_dev), len(q_rec), len(q_own), dropped, ) fig, axes = plt.subplots(3, 3, figsize=(15, 11.5), facecolor="#fcfcfb") @@ -119,8 +112,7 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da f"USA s300 — {tech} p_max_pu per resource cluster, master vs develop\n" f"paired view: master's nodal profile rolled onto develop's {len(paired)} clusters " f"(busmap_s300, p_nom_max-weighted = develop's rule, HF-25; HF-24-dropped substations absent)", - fontsize=12, - color=INK, + fontsize=12, color=INK, ) summary_rows = [] for i, q in enumerate(Q): @@ -149,16 +141,11 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da rmse = float(np.sqrt((d**2).mean())) within = int((d.abs() <= 0.005).sum()) ax.text( - 0.03, - 0.97, + 0.03, 0.97, f"n = {len(d)}\nmax |Δ| = {d.abs().max():.4f}\nRMSE = {rmse:.5f}\n|Δ| ≤ 0.005: {within}/{len(d)}", - transform=ax.transAxes, - va="top", - fontsize=9, - color=INK, + transform=ax.transAxes, va="top", fontsize=9, color=INK, ) - ax.set_xlim(lim) - ax.set_ylim(lim) + ax.set_xlim(lim); ax.set_ylim(lim) ax.set_xlabel(f"develop p{q}", color=INK) ax.set_ylabel(f"master, reconstructed p{q}", color=INK) ax.set_title(f"p{q}: paired per cluster after reconstruction", fontsize=10, color=INK) @@ -170,33 +157,19 @@ def one_tech(tech: str, busmap: pd.Series, zones: pd.Series, out: Path) -> pd.Da ax.axhspan(-0.005, 0.005, color=GRID, alpha=0.6, zorder=0) worst = ds.abs().sort_values(ascending=False).head(3) ax.text( - 0.03, - 0.97, + 0.03, 0.97, "largest |Δ|: " + ", ".join(f"{c} ({paired.loc[c, 'reeds_zone']}) {ds[c]:+.4f}" for c in worst.index), - transform=ax.transAxes, - va="top", - fontsize=8, - color=INK, - wrap=True, + transform=ax.transAxes, va="top", fontsize=8, color=INK, wrap=True, ) ax.set_xlabel("clusters, ranked by Δ", color=MUTED) ax.set_ylabel(f"Δ p{q} = master recon − develop", color=INK) ax.set_title(f"p{q}: residual per cluster (band = ±0.005)", fontsize=10, color=INK) summary_rows.append( - dict( - tech=tech, - quantile=f"p{q}", - n_clusters=len(d), - max_abs_delta=float(d.abs().max()), - rmse=rmse, - median_abs_delta=float(d.abs().median()), - n_within_0p005=within, - develop_median=float(x.median()), - master_recon_median=float(y.median()), - master_own_s300_median=float(q_own[col].median()), - master_own_n=len(q_own), - master_nodal_buses_dropped=dropped, - ), + dict(tech=tech, quantile=f"p{q}", n_clusters=len(d), max_abs_delta=float(d.abs().max()), + rmse=rmse, median_abs_delta=float(d.abs().median()), n_within_0p005=within, + develop_median=float(x.median()), master_recon_median=float(y.median()), + master_own_s300_median=float(q_own[col].median()), master_own_n=len(q_own), + master_nodal_buses_dropped=dropped) ) for ax in axes.ravel(): ax.set_facecolor("#fcfcfb") diff --git a/tests/equivalence/hotfixes.yaml b/tests/equivalence/hotfixes.yaml index 6d0c0f73..0a383c48 100644 --- a/tests/equivalence/hotfixes.yaml +++ b/tests/equivalence/hotfixes.yaml @@ -122,7 +122,7 @@ pipelines. Still listed because the pin, not a port, is what closes it. - id: HF-7 - commit: .inf + commit: 51e1719 pr: 794 title: renewable_land_access default null -> reference (NREL reV land-exclusion screening of the GODEEEP CFs) confidence: high @@ -319,7 +319,7 @@ cardinality. - id: HF-20 - commit: .inf + commit: 51e1719 pr: 794 title: config.default.yaml becomes a loaded base layer beneath user configs, scenario configs become sparse overlays, and the merged config is schema-validated at parse time confidence: medium @@ -352,7 +352,7 @@ - id: HF-22 commit: e1f5841 - pr: + pr: null title: build_renewable_profiles._drop_leap_day — Feb 29 dropped from the selection window so a leap weather year yields the standard 8760-hour axis confidence: none ported: true @@ -385,7 +385,7 @@ - id: HF-24 commit: 3b149ae - pr: + pr: null title: build_renewable_profiles keeps NREL caps capacity for substations with zero GODEEEP availability or no intersecting cell, instead of silently dropping it confidence: high ported: false @@ -415,7 +415,7 @@ - id: HF-25 commit: 3b149ae - pr: + pr: null title: "renewable.godeeep_cf_weighting: develop weights substation capacity factors up to the s{simpl} cluster by INSTALLABLE capacity (NREL caps p_nom_max); master weights them by EXISTING p_nom and falls back to an unweighted mean where a cluster has none" confidence: high ported: false @@ -451,17 +451,17 @@ - id: HF-26 commit: a5ed78c - pr: + pr: null title: "attach_renewable_capacities_to_atlite runs at CLUSTER granularity on develop, so existing renewable plants at substations with no profile generator keep their MW; on master the same function runs nodally and drops them" confidence: high ported: false usa_noop: false expect: - - capacity_existing_by_carrier/* - - p_nom_existing_by_zone_carrier/* - - objective - - capacity_opt_by_carrier/* - - dispatch_by_carrier/* + - capacity_existing_by_carrier/* + - p_nom_existing_by_zone_carrier/* + - objective + - capacity_opt_by_carrier/* + - dispatch_by_carrier/* master: >- the assembled network is nodal, so `mapped_values = generators_tech.sub_assignment.map(caps_per_bus).dropna()` in @@ -496,7 +496,7 @@ - id: HF-27 commit: 382535b - pr: + pr: null title: "plant -> bus assignment: a DEVELOP REGRESSION, now FIXED — after simplify-early develop matched plants to the {simpl} cluster CENTROIDS (median 37 km) instead of to substations (median 1.0 km, master's own bus set), putting 6.1 % of existing MW in the wrong ReEDS zone; develop now matches substations and maps through busmap_s{simpl}" confidence: high ported: false @@ -542,7 +542,7 @@ - id: HF-28 commit: 01f742f - pr: + pr: null title: "demand disaggregation key: master splits EFS state demand over NODAL buses keyed on `buses.state`, develop over s{simpl} buses keyed on `buses.reeds_state`; the LAF both sides consume was normalised over a THIRD column (`full_state`), so neither conserved state demand until the renormalisation landed" confidence: high ported: false @@ -589,7 +589,7 @@ - id: HF-29 commit: 2a3adf8 - pr: + pr: null title: "LAF renormalised inside the demand key at consumption time, on BOTH branches (state demand conserved: was +2.28 % master, +1.85 % develop); plus a DC-column fold into Maryland that is skipped when a bus carries the key" confidence: none ported: true diff --git a/tests/equivalence/metrics.py b/tests/equivalence/metrics.py index df3a6aad..1b69f66d 100644 --- a/tests/equivalence/metrics.py +++ b/tests/equivalence/metrics.py @@ -430,7 +430,8 @@ def aggregate_profile_to_clusters( n_dropped = int((~keep).sum()) if n_dropped: logger.warning( - "aggregate_profile_to_clusters: %d of %d buses are absent from the busmap and were dropped (first few: %s)", + "aggregate_profile_to_clusters: %d of %d buses are absent from the busmap " + "and were dropped (first few: %s)", n_dropped, len(buses), ", ".join(str(b) for b in buses[~keep][:5]), @@ -496,7 +497,7 @@ def _weights(name: str) -> pd.Series: out_ds = xr.Dataset(data_vars, coords=coords, attrs=dict(ds.attrs)) out_ds.attrs["eq_dropped_buses"] = n_dropped out_ds.attrs["eq_zero_weight_clusters"] = n_zero_weight - out_ds.attrs["eq_aggregated_from_buses"] = len(buses) + out_ds.attrs["eq_aggregated_from_buses"] = int(len(buses)) return out_ds diff --git a/tests/equivalence/paths.py b/tests/equivalence/paths.py index db4ffdea..7d97c843 100644 --- a/tests/equivalence/paths.py +++ b/tests/equivalence/paths.py @@ -39,9 +39,7 @@ # under config/, and build.py copies the shared harness config in there. CONFIGFILE = f"repo_data/config/{_CONFIG_NAME}" BASELINE_CONFIGFILE = f"config/{_CONFIG_NAME}" -CLUSTERS = os.environ.get( - "EQ_CLUSTERS", "4" -) # reeds transport: must equal the footprint's ReEDS zone count (western/CA slice 4; usa 134) +CLUSTERS = os.environ.get("EQ_CLUSTERS", "4") # reeds transport: must equal the footprint's ReEDS zone count (western/CA slice 4; usa 134) LL = "v1.0" @@ -150,11 +148,7 @@ def _profile_horizon_dir() -> str: import yaml cfg_path = os.path.join( - os.path.dirname(__file__), - "..", - "..", - "workflow", - CONFIGFILE, + os.path.dirname(__file__), "..", "..", "workflow", CONFIGFILE ) with open(cfg_path) as f: cfg = yaml.safe_load(f) @@ -162,6 +156,7 @@ def _profile_horizon_dir() -> str: return "" if scenarios[0] == "historical" else f"{HORIZON}/" + @dataclass(frozen=True) class ArtifactPair: """One comparable artifact across the two sides.""" diff --git a/tests/equivalence/plots.py b/tests/equivalence/plots.py index ebfd0fc5..8cd7e4a8 100644 --- a/tests/equivalence/plots.py +++ b/tests/equivalence/plots.py @@ -875,20 +875,12 @@ def export_all( objective_figure(m.get("objective"), "objective", run_dir) paired_bar( - m.get("capacity_existing_by_carrier"), - "existing capacity", - "MW", - "capacity_existing_by_carrier", - run_dir, + m.get("capacity_existing_by_carrier"), "existing capacity", "MW", "capacity_existing_by_carrier", run_dir ) paired_bar(m.get("capacity_opt_by_carrier"), "optimised capacity", "MW", "capacity_opt_by_carrier", run_dir) paired_bar(m.get("dispatch_by_carrier"), "annual dispatch", "MWh", "dispatch_by_carrier", run_dir) paired_bar( - m.get("capacity_factor_by_carrier"), - "realised capacity factor", - "-", - "capacity_factor_by_carrier", - run_dir, + m.get("capacity_factor_by_carrier"), "realised capacity factor", "-", "capacity_factor_by_carrier", run_dir ) paired_bar(m.get("demand_by_zone"), "demand by zone", "MW", "demand_zones", run_dir) diff --git a/tests/equivalence/test_demand_conservation_diagnostic.py b/tests/equivalence/test_demand_conservation_diagnostic.py index 1e4fc63c..bd89993a 100644 --- a/tests/equivalence/test_demand_conservation_diagnostic.py +++ b/tests/equivalence/test_demand_conservation_diagnostic.py @@ -137,19 +137,16 @@ def test_main_writes_a_png_and_its_csv_twin(tmp_path, capsys): make_table(tmp_path / "d.parquet", {"Maryland": 200.0, "Virginia": 150.0}) outdir = tmp_path / "out" - assert ( - main( - [ - "--network", - str(tmp_path / "n.nc"), - "--demand-table", - str(tmp_path / "d.parquet"), - "--outdir", - str(outdir), - ], - ) - == 0 - ) + assert main( + [ + "--network", + str(tmp_path / "n.nc"), + "--demand-table", + str(tmp_path / "d.parquet"), + "--outdir", + str(outdir), + ], + ) == 0 assert (outdir / "demand_conservation.png").exists() written = pd.read_csv(outdir / "demand_conservation.csv", index_col="demand_key") diff --git a/tests/equivalence/test_master_benchmark_branch.py b/tests/equivalence/test_master_benchmark_branch.py index 7a687c12..804e1d31 100644 --- a/tests/equivalence/test_master_benchmark_branch.py +++ b/tests/equivalence/test_master_benchmark_branch.py @@ -174,7 +174,8 @@ def test_ported_commits_name_a_hotfix(commits): hf = c["trailers"].get("Hot-fix", "") if not re.fullmatch(r"HF-\d+", hf): bad.append( - f"{c['sha'][:7]} {c['subject']!r}: Hot-fix={hf!r}, expected 'HF-' (bare integer, not zero-padded)", + f"{c['sha'][:7]} {c['subject']!r}: Hot-fix={hf!r}, expected 'HF-' " + "(bare integer, not zero-padded)", ) assert not bad, "hot-fix trailer violations:\n" + "\n".join(bad) @@ -282,9 +283,7 @@ def _manifest_table_shas(branch: str) -> list[str]: start = text.index("## Commits") end = text.find("\n## ", start + 1) table = text[start:end] if end != -1 else text[start:] - return [ - m.group(1) for line in table.splitlines() if line.lstrip().startswith("|") for m in _SHA_TOKEN.finditer(line) - ] + return [m.group(1) for line in table.splitlines() if line.lstrip().startswith("|") for m in _SHA_TOKEN.finditer(line)] def test_branch_manifest_matches(branch, commits): diff --git a/tests/equivalence/test_nomenclature.py b/tests/equivalence/test_nomenclature.py index 14243028..6bf2e71f 100644 --- a/tests/equivalence/test_nomenclature.py +++ b/tests/equivalence/test_nomenclature.py @@ -53,7 +53,12 @@ def _scanned_files() -> list[Path]: - files = sorted(p for ext in ("py", "yaml", "sbatch") for p in HARNESS.glob(f"*.{ext}") if p.name != _SELF) + files = sorted( + p + for ext in ("py", "yaml", "sbatch") + for p in HARNESS.glob(f"*.{ext}") + if p.name != _SELF + ) files += sorted((REPO / "workflow" / "repo_data" / "config").glob("config.equivalence*.yaml")) files.append(REPO / "CONTEXT.md") return [p for p in files if p.exists()] @@ -80,7 +85,10 @@ def test_no_anchor_nomenclature(): for lineno, line in enumerate(block.splitlines(), start=1): if "anchor" in _ALLOWED.sub("", line).lower(): hits.append(f"pyproject.toml [tool.pytest.ini_options]+{lineno}: {line.strip()}") - assert not hits, "retired 'anchor' nomenclature found; the baseline is master-benchmark:\n" + "\n".join(hits) + assert not hits, ( + "retired 'anchor' nomenclature found; the baseline is master-benchmark:\n" + + "\n".join(hits) + ) @pytest.mark.fast diff --git a/tests/equivalence/test_submit_benchmark.py b/tests/equivalence/test_submit_benchmark.py index 2f2591f8..f82d288c 100644 --- a/tests/equivalence/test_submit_benchmark.py +++ b/tests/equivalence/test_submit_benchmark.py @@ -27,7 +27,7 @@ def fake_sbatch(tmp_path: Path) -> Path: f"n=$(( $(grep -c '^ARGS' '{log}' 2>/dev/null || echo 0) + 1 ))\n" f"echo \"ARGS $*\" >> '{log}'\n" f"echo \"ENV EQ_INTERCONNECT=$EQ_INTERCONNECT EQ_UNTIL=$EQ_UNTIL EQ_RUN_ID=$EQ_RUN_ID EXTRA=$EQ_EXTRA_ARGS\" >> '{log}'\n" - 'echo "1000$n"\n', + 'echo "1000$n"\n' ) shim.chmod(shim.stat().st_mode | stat.S_IEXEC) return log diff --git a/tests/static/test_paths.py b/tests/static/test_paths.py index 91920d57..67e3a60f 100644 --- a/tests/static/test_paths.py +++ b/tests/static/test_paths.py @@ -96,7 +96,9 @@ def test_exactly_one_sbatch_driver(): def test_sbatch_uses_the_serc_partition(sbatch_path): """The project's partition is serc (PROJECT.md §4), not normal.""" source = sbatch_path.read_text() - assert re.search(r"^#SBATCH\s+-p\s+serc\b", source, re.MULTILINE), f"{sbatch_path.name} does not request -p serc" + assert re.search(r"^#SBATCH\s+-p\s+serc\b", source, re.MULTILINE), ( + f"{sbatch_path.name} does not request -p serc" + ) assert not re.search(r"^#SBATCH\s+-p\s+normal\b", source, re.MULTILINE), ( f"{sbatch_path.name} still requests -p normal" ) @@ -132,5 +134,9 @@ def test_sbatch_log_directives_are_relative(sbatch_path): an absolute site path is one someone else will not have. Relative means the submit directory, which by construction exists. """ - bad = [line.strip() for line in sbatch_path.read_text().splitlines() if re.match(r"^#SBATCH\s+-[oe]\s+/", line)] + bad = [ + line.strip() + for line in sbatch_path.read_text().splitlines() + if re.match(r"^#SBATCH\s+-[oe]\s+/", line) + ] assert not bad, f"{sbatch_path.name} writes Slurm logs to an absolute path: {bad}" diff --git a/workflow/report_benchmarks.py b/workflow/report_benchmarks.py index 4d89eca5..e10344bd 100644 --- a/workflow/report_benchmarks.py +++ b/workflow/report_benchmarks.py @@ -129,7 +129,7 @@ def rec_time(peak_s: float) -> str: agg["peak_time"] = agg.peak_s.map(fmt_hms) print( - f"{'RULE':<44} {'N':>2} {'PEAK_RSS_MB':>11} {'PEAK_TIME':>10} {'LOAD%':>6} {'REC_MEM_MB':>10} {'REC_WALLTIME':>12}", + f"{'RULE':<44} {'N':>2} {'PEAK_RSS_MB':>11} {'PEAK_TIME':>10} {'LOAD%':>6} {'REC_MEM_MB':>10} {'REC_WALLTIME':>12}" ) print("-" * 108) for name, r in agg.iterrows(): diff --git a/workflow/scripts/_helpers.py b/workflow/scripts/_helpers.py index 00ba3583..9e95af8f 100644 --- a/workflow/scripts/_helpers.py +++ b/workflow/scripts/_helpers.py @@ -346,9 +346,9 @@ def mock_snakemake(rulename, **wildcards): from snakemake.script import Snakemake script_dir = Path(__file__).parent.resolve() - assert Path.cwd().resolve() == script_dir, ( - f"mock_snakemake has to be run from the repository scripts directory {script_dir}" - ) + assert ( + Path.cwd().resolve() == script_dir + ), f"mock_snakemake has to be run from the repository scripts directory {script_dir}" os.chdir(script_dir.parent) for p in sm.SNAKEFILE_CHOICES: if os.path.exists(p): @@ -429,9 +429,9 @@ def validate_checksum(file_path, zenodo_url=None, checksum=None): for chunk in iter(lambda: f.read(65536), b""): # 64kb chunks hasher.update(chunk) calculated_checksum = hasher.hexdigest() - assert calculated_checksum == checksum, ( - "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule." - ) + assert ( + calculated_checksum == checksum + ), "Checksum is invalid. This may be due to an incomplete download. Delete the file and re-execute the rule." def get_checksum_from_zenodo(file_url): diff --git a/workflow/scripts/build_demand.py b/workflow/scripts/build_demand.py index f8afcc25..26683bf5 100644 --- a/workflow/scripts/build_demand.py +++ b/workflow/scripts/build_demand.py @@ -2117,7 +2117,8 @@ def _log_unallocated_demand(demand: pd.DataFrame, zone_data: pd.DataFrame) -> No if not orphans: return logger.warning( - "No bus carries demand key(s) %s; %.1f MW of mean demand is DROPPED. Per key: %s.", + "No bus carries demand key(s) %s; %.1f MW of mean demand is DROPPED. " + "Per key: %s.", sorted(orphans), sum(orphans.values()), {k: round(v, 1) for k, v in sorted(orphans.items())}, diff --git a/workflow/scripts/build_renewable_profiles.py b/workflow/scripts/build_renewable_profiles.py index 2ee99812..c101fa28 100644 --- a/workflow/scripts/build_renewable_profiles.py +++ b/workflow/scripts/build_renewable_profiles.py @@ -529,7 +529,8 @@ def plot_data(data): cache_dir=snakemake.params.mapping_cache_dir, ) logger.info( - f"Cell→substation mapping: {mapping_sub['name'].nunique()} substations, {len(mapping_sub)} cell rows", + f"Cell→substation mapping: {mapping_sub['name'].nunique()} substations, " + f"{len(mapping_sub)} cell rows", ) agg = capacity_weighted_bus_aggregation( ds_cf["capacity_factor"], diff --git a/workflow/scripts/plot_docs_figures.py b/workflow/scripts/plot_docs_figures.py index 15d723f7..8fed63e3 100644 --- a/workflow/scripts/plot_docs_figures.py +++ b/workflow/scripts/plot_docs_figures.py @@ -159,12 +159,7 @@ def _generator_legend(fig, title): handles = [ plt.Line2D([], [], marker="o", linestyle="", color=GEN_COLORS["conventional"], label="conventional generator"), plt.Line2D( - [], - [], - marker="o", - linestyle="", - color=GEN_COLORS["renewable"], - label="renewable / other generator", + [], [], marker="o", linestyle="", color=GEN_COLORS["renewable"], label="renewable / other generator" ), ] fig.legend(handles=handles, loc="lower center", ncol=2, frameon=False, fontsize=9, title=title, title_fontsize=8) @@ -224,7 +219,8 @@ def plot_cluster_suffixes(simpl_path, suffix_paths, shapes_path, out_path, conve ) fig.suptitle( - f"The same {n_simpl_buses}-zone {{simpl}} network clustered to {len(buses)} zones by each {{clusters}} suffix", + f"The same {n_simpl_buses}-zone {{simpl}} network clustered to " + f"{len(buses)} zones by each {{clusters}} suffix", fontsize=11, ) _generator_legend( @@ -284,9 +280,7 @@ def plot_simpl_resolutions(panels, shapes_path, out_path, conventional_carriers= "The same footprint at each {simpl} resolution, clustered with the a suffix (nothing aggregated)", fontsize=11, ) - _generator_legend( - fig, "marker area ∝ p_nom (capped); every generator drawn at its {simpl} zone (nothing aggregated)" - ) + _generator_legend(fig, "marker area ∝ p_nom (capped); every generator drawn at its {simpl} zone (nothing aggregated)") fig.tight_layout(rect=(0, 0.09, 1, 0.92)) fig.savefig(out_path, dpi=150, bbox_inches="tight", facecolor="white") plt.close(fig) diff --git a/workflow/scripts/test/test_cluster_wildcard.py b/workflow/scripts/test/test_cluster_wildcard.py index 9e49355e..05563496 100644 --- a/workflow/scripts/test/test_cluster_wildcard.py +++ b/workflow/scripts/test/test_cluster_wildcard.py @@ -7,7 +7,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from cluster_network import parse_clusters_wildcard +from cluster_network import parse_clusters_wildcard # noqa: E402 ALL = {"onwind", "solar", "CCGT", "coal", "nuclear"} CONV = {"CCGT", "coal", "nuclear"} diff --git a/workflow/scripts/test/test_policy.py b/workflow/scripts/test/test_policy.py index 92dbf60f..f3949c3a 100644 --- a/workflow/scripts/test/test_policy.py +++ b/workflow/scripts/test/test_policy.py @@ -568,3 +568,4 @@ def test_add_rps_constraints_zone_without_eligible_gens_does_not_stop_others(pol rps = [c for c in n.model.constraints if c.endswith("_rps_limit")] assert rps, "the non-empty zone lost its constraint" assert not any(c.startswith("GlobalConstraint-CA") for c in rps) +