diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f5c79a6..a640f294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,81 @@ # Changelog +## [0.3.3] - 2026-07-16 + +Large architecture-review pass over the `alphaDeesp` graph layer: correctness +fixes, performance work, two deep revisions, the interactive-viewer +decomposition, and the fix for the multigraph Dijkstra-weight bug (issue #1). +All changes are unit-tested (372 runnable tests without grid2op) and checked by +adversarial multi-agent verification. Public import paths are unchanged. + +### Bug Fixes + +- **Multigraph Dijkstra weight in the null-flow path search** (issue #1): on the + overflow `MultiDiGraph`, networkx passes a callable weight the `{key: attr}` + parallel-edge view, so `_compute_sssp_paths`'s `attr.get("capacity", 0)` + silently read `0` and routing was hop-cost-only. The routing weight is now + precomputed as an edge attribute (string-weight Dijkstra) with a + `capacity_weighted` switch: `False` (default) reproduces the historical + behaviour **bit-identical** ("bless"); `True` enables capacity-weighted routing + with correct min-parallel capacity and `(u,v)`/`(u,v,key)` promoted matching + ("fix"). Threaded through `add_relevant_null_flow_lines[_all_paths]` so + downstream callers can opt in without patching. +- **`shortest_paths.py`**: removed a dead branch and made the promoted-edge + matching multigraph-correct (was silently weighting every parallel edge as `0`). +- **`OverFlowGraph.__init__` no longer mutates the caller's DataFrame** (copies + it); `rename_nodes` loop variable clarified (behaviour unchanged). +- **`Structured_Overload_Distribution_Graph.get_dispatch_edges_nodes`** guards + the empty-loops case (`red_loops.Path.sum()` no longer raises `TypeError` on a + grid with no loop paths). +- **`to_DiGraph`** defaults a missing edge `capacity` to `0.0` (was `1.0`), which + no longer skews `rank_red_loops`' min-cut. + +### New Features / Deep revisions + +- **`OverFlowGraph` model/renderer split**: new + `graphs/overflow_renderer.py::OverflowGraphRenderer` owns all Graphviz + presentation (penwidth, shapes, tapered styling, compound highlight colours, + plotting); `OverFlowGraph` keeps the semantic model. New + `graphs/edge_roles.py::edge_role_of` (+ `EDGE_ROLE_*`, `OverFlowGraph.edge_role`) + is the single authority mapping an edge's base colour to a semantic role, so no + consumer parses Graphviz colour strings; `highlight_significant_line_loading` + records an authoritative `base_color`. +- **`AlphaDeesp` explicit pipeline**: the ranking pipeline moved to `run()`; + `AlphaDeesp(..., auto_run=True)` (default) preserves the previous behaviour, + `auto_run=False` builds the object without side effects. +- **Lazy `Structured_Overload_Distribution_Graph`**: colour-filtered views, + `red_loops` and `hubs` are `functools.cached_property` computed from a + construction-time snapshot — behaviour-identical and order-independent. + +### Performance + +- `delete_color_edges` accepts a colour **or an iterable of colours**, removing + them in a single graph copy; the structured graph builds its derived views in + single passes. +- `sort_hubs` and the initial-inflow lookup in `AlphaDeesp` are vectorised + (were per-hub / per-edge `iterrows` scans). +- `simulation.create_df` is vectorised (was several `iterrows` passes; + 400-case differential fuzz pins equivalence). +- Null-flow Dijkstra uses a precomputed string weight instead of a per-edge + Python callable. + +### Maintainability + +- `core/interactive_html.py` (976 LOC) split into the `core/interactive_html/` + package (8 focused modules) with the CSS/JS/HTML skeleton externalised under + `assets/` and reassembled byte-exactly at runtime (`package_data` shipped). +- `find_loops` / consolidation path enumeration gained opt-in cutoffs + (`loop_path_cutoff`, default `None` = unbounded = original behaviour). +- New `docs/CODE_REVIEW.md` (full review + downstream-impact analysis on + `Expert_op4grid_recommender` + remaining-work backlog); `CLAUDE.md` refreshed. + +### Tests + +- New suites: `test_edge_roles.py`, `test_overflow_renderer.py`, + `test_simulation_create_df.py`, `test_null_flow_weighting.py`; expanded + `test_graphs_package.py`, `test_overflow_graph.py`, `test_alphadeesp_unit.py`, + `test_shortest_paths.py`, `test_interactive_html.py`. + ## [0.3.2.post4] - 2026-06-17 ### New Features diff --git a/CLAUDE.md b/CLAUDE.md index b11055a8..b7f94d60 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,18 +24,38 @@ alphaDeesp/ ├── Expert_rule_action_verification.py # Rule-checking utilities for proposed actions ├── core/ │ ├── alphadeesp.py # AlphaDeesp algorithm (ranking, topology exploration) -│ ├── graphsAndPaths.py # OverFlowGraph, PowerFlowGraph, Structured_Overload_Distribution_Graph +│ ├── topology_scorer.py # TopologyScorerMixin (score helpers for AlphaDeesp) +│ ├── topo_applicator.py # TopoApplicatorMixin (apply topo vectors to the graph) +│ ├── twin_nodes.py # Twin-node id scheme (busbar-split node encoding) +│ ├── graphsAndPaths.py # Back-compat shim → re-exports from core/graphs/ +│ ├── graphs/ # Graph layer (split out of the old graphsAndPaths.py) +│ │ ├── overflow_graph.py # OverFlowGraph — the overflow *semantic model* +│ │ ├── overflow_renderer.py # OverflowGraphRenderer — Graphviz *presentation* only +│ │ ├── power_flow_graph.py # PowerFlowGraph (base current-state graph) +│ │ ├── structured_overload_graph.py # Structured_Overload_Distribution_Graph +│ │ ├── constrained_path.py # ConstrainedPath value object +│ │ ├── null_flow_graph.py # NullFlowGraphMixin (null-flow line handling) +│ │ ├── null_flow.py # double/un-double null-flow edge helpers +│ │ ├── graph_consolidation.py # GraphConsolidationMixin (disambiguation) +│ │ ├── graph_utils.py # pure networkx helpers (delete_color_edges, ...) +│ │ ├── shortest_paths.py # mandatory/promoted-edge shortest paths +│ │ ├── edge_roles.py # base-colour → semantic role accessor (edge_role_of) +│ │ └── constants.py # default_voltage_colors, ... +│ ├── interactive_html/ # Interactive HTML/SVG overflow viewer (used by other repos) +│ │ ├── {helpers,constants,layers,model,svg,template,render}.py +│ │ └── assets/{viewer.css,viewer.js,template.html} # externalised JS/CSS/skeleton │ ├── simulation.py # Abstract Simulation base class + DataFrame plumbing │ ├── network.py # Network / Substation model objects -│ ├── elements.py # Production, Consumption, OriginLine, ExtremityLine dataclasses +│ ├── elements.py # Production, Consumption, OriginLine, ExtremityLine classes │ ├── printer.py # Graphviz/shell printing helpers │ ├── grid2op/ # Grid2op backend (Grid2opSimulation, Grid2opObservationLoader) -│ └── pypownet/ # Pypownet backend (legacy / optional) +│ └── pypownet/ # Pypownet backend (legacy / deprecated) ├── ressources/ │ ├── config/config.ini # Default runtime config (thresholds, layout, simulator type) │ └── parameters/ # Built-in grids (l2rpn_2019, rte_case14_realistic, custom14, ...) └── tests/ # pytest suite (unit + integration, grid2op + pypownet) -docs/ # Sphinx sources (RST) +docs/ # Sphinx sources (RST) + CODE_REVIEW.md +scripts/code_quality_report.py # Aggregated radon/vulture/ruff/interrogate report getting_started/ # Jupyter tutorial notebooks .circleci/config.yml # CI: pytest on cimg/python:3.12 with graphviz ``` @@ -57,7 +77,7 @@ getting_started/ # Jupyter tutorial notebooks │ expert_operator.expert_operator() │ └────────┬───────────────────────────┘ │ - ├─► OverFlowGraph (graphsAndPaths.py) + ├─► OverFlowGraph (core/graphs/overflow_graph.py) ├─► AlphaDeesp.get_ranked_combinations() └─► sim.compute_new_network_changes() -> end-result DataFrame ``` @@ -66,6 +86,56 @@ Key contract: any new simulator backend must implement the abstract methods of `alphaDeesp/core/simulation.py::Simulation` (get_dataframe, isAntenna, get_substation_elements, compute_new_network_changes, etc.). +### The `graphs/` package (and `OverFlowGraph`) + +`core/graphsAndPaths.py` is now a **backwards-compatible shim** that re-exports +the public surface of the `core/graphs/` package. External code (and other +repos) can keep importing `from alphaDeesp.core.graphsAndPaths import +OverFlowGraph, ...`; new code should import from `alphaDeesp.core.graphs`. + +`OverFlowGraph` is split into a **semantic model** and a **renderer**: + +- `graphs/overflow_graph.py::OverFlowGraph` owns the semantic model — graph + topology, per-edge redispatch magnitude, edge *role* encoded as a base + colour (`black` overload / `blue` negative / `coral` positive / `gray` + insignificant), and the boolean semantic flags consumed downstream + (`is_overload`, `is_monitored`, `on_constrained_path`, `in_red_loop`, + `is_hub`, `is_extra_cut`). The public method signatures are unchanged. +- `graphs/overflow_renderer.py::OverflowGraphRenderer` owns all Graphviz + *presentation* (penwidth scaling, node shapes, tapered swap styling, the + compound `"colour:yellow:colour"` highlight strings and HTML loading + labels, and plotting). It is **stateless** (static methods over a passed-in + graph) so downstream repos can reuse it on any compatible `MultiDiGraph`. + +The public surface of the package is pinned by `tests/test_graphs_package.py` +(`EXPECTED_PUBLIC_NAMES` / `EXPECTED_SUBMODULES`) — update those sets when you +add or move a public symbol. + +**Semantic edge roles.** `graphs/edge_roles.py::edge_role_of(edge_data)` is the +single authority mapping an edge's base colour to a stable role +(`EDGE_ROLE_OVERLOAD/NEGATIVE/POSITIVE/INSIGNIFICANT/NULL_NON_RECONNECTABLE`). +It prefers the `base_color` attribute (recorded by the renderer when it wraps a +colour into a compound `"c:yellow:c"` highlight) and is compound-safe — so no +consumer should ever parse a Graphviz colour string. `OverFlowGraph.edge_role(name)` +is the convenience by-line-name accessor. + +**Lazy structured graph.** `Structured_Overload_Distribution_Graph` computes its +colour-filtered views, `red_loops` and `hubs` as `functools.cached_property` +(constrained path stays eager). `red_loops` uses the constructor *seed* hubs; +the public `find_loops()` re-enumerates with the *detected* hubs (this split is +what makes the lazy properties order-independent while matching the old eager +behaviour). Don't reintroduce eager computation. + +**AlphaDeesp construction.** `AlphaDeesp(..., auto_run=True)` runs the ranking +pipeline in the constructor (default, backwards-compatible). Pass +`auto_run=False` and call `.run()` for staged/testable execution. +`AlphaDeesp_warmStart` is the pre-existing "skip the pipeline" path. + +**Interactive viewer.** `core/interactive_html/` is a package; the CSS/JS/HTML +skeleton are externalised under `assets/` and reassembled at runtime by +`template.html_template()`. Edit the `.css`/`.js` assets directly. The package +is shipped via `package_data` in `setup.py` + `MANIFEST.in`. + ## Common Commands ```bash @@ -120,7 +190,12 @@ Runtime: `Grid2Op`, `lightsim2grid`, `networkx`, `rustworkx`, `pandapower`, Optional: `pypownet>=2.2.0`, `oct2py`, `pypower` (for the legacy backend). -Python: targeted at 3.12 in CI; `setup.py` still advertises 3.6/3.7 (stale). +Python: `setup.py` declares `python_requires=">=3.9"` with classifiers for +3.9–3.12; CI runs on 3.12. Note the `graphs/` package unit tests +(`test_overflow_graph.py`, `test_graphs_package.py`, `test_null_flow.py`, ...) +run **without** grid2op; the grid2op integration suites, `alphadeesp_test.py`, +`test_cli.py` and `test_expert_rules.py` require `grid2op` + `lightsim2grid` +installed. Graphviz **executables** must be on PATH for snapshot/plot mode (not just the Python binding). On Debian/Ubuntu: `apt-get install graphviz`. @@ -134,14 +209,33 @@ Python binding). On Debian/Ubuntu: `apt-get install graphviz`. - `simulatorType = Grid2OP` (exact casing) in config.ini; `Pypownet` and `RTE` are alternative literals checked in `main.py`. - Naming is mixed: public API uses both `snake_case` (`get_dataframe`) and - `camelCase` (`isAntenna`, `isDoubleLine`, `getLinesAtSubAndBusbar`). Preserve - existing names when editing to avoid breaking the abstract contract. -- `core/alphadeesp.py` and `core/network.py` rely on `from elements import *`; - adding new element classes requires keeping that star import working. -- Several modules print directly to stdout; there is no logging framework - wired up. -- Only one type-annotated function exists in the codebase — do not expect mypy - to be useful without first adding annotations. + `camelCase` (`isAntenna`, `isDoubleLine`, `getLinesAtSubAndBusbar`), plus the + PascalCase-with-underscores `Structured_Overload_Distribution_Graph`. + Preserve existing names when editing to avoid breaking the abstract contract + and external importers. +- `core/elements.py` classes are now imported **explicitly** (e.g. + `from alphaDeesp.core.elements import Consumption, Production`); the old + `from elements import *` star imports have been removed. +- Logging: newer modules (`graphs/`, `alphadeesp.py`, `simulation.py`, + `elements.py`, `main.py`) use the `logging` framework; the older backends + (`grid2op/`, `pypownet/`, `network.py`, `printer.py`) still `print()` in + places. Prefer `logging` in new/edited code. +- Typing: the `graphs/` package, `simulation.py`, `elements.py`, `network.py` + and `alphadeesp.py` carry type annotations. CI enforces mypy **strictly** on + `simulation.py` and `elements.py` (permissive elsewhere). Do not reintroduce + the "codebase is untyped" assumption. +- `graphs/graph_utils.delete_color_edges` accepts a single colour **or an + iterable of colours** and removes them in one graph copy — prefer the + multi-colour form to avoid chained full-graph copies. +- `Structured_Overload_Distribution_Graph.find_loops` can bound simple-path + enumeration with `loop_path_cutoff` to avoid hangs on very large grids, but + it is **opt-in**: the default is `None` (unbounded = original behaviour) + because rustworkx `cutoff` counts *nodes* and real grids have loop paths well + beyond any small bound — pass an int only where enumeration is a problem. + Consolidation path enumeration has an analogous opt-in + `DEFAULT_CONSOLIDATION_PATH_CUTOFF` (also `None`). +- `OverFlowGraph` copies the caller's DataFrame in `__init__` (it never + mutates the frame you pass in). ## Branch Policy for Claude Code Sessions diff --git a/MANIFEST.in b/MANIFEST.in index e487f81f..380c58e7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,4 +7,5 @@ include alphaDeesp/ressources/parameters/l2rpn_2019/config.py include alphaDeesp/ressources/parameters/l2rpn_2019/difficulty_levels.json include alphaDeesp/ressources/parameters/l2rpn_2019/grid.json include alphaDeesp/ressources/parameters/l2rpn_2019/grid_layout.json -include alphaDeesp/ressources/parameters/l2rpn_2019/prods_charac.csv \ No newline at end of file +include alphaDeesp/ressources/parameters/l2rpn_2019/prods_charac.csv +include alphaDeesp/core/interactive_html/assets/* diff --git a/alphaDeesp/core/alphadeesp.py b/alphaDeesp/core/alphadeesp.py index fa956bbd..12e5e67a 100755 --- a/alphaDeesp/core/alphadeesp.py +++ b/alphaDeesp/core/alphadeesp.py @@ -34,7 +34,17 @@ def __init__( simulator_data: Optional[Dict[str, Any]] = None, substation_in_cooldown: Optional[List[int]] = None, debug: bool = False, + auto_run: bool = True, ) -> None: + """Build the solver state and (by default) run the ranking pipeline. + + :param auto_run: when ``True`` (default, backwards-compatible) the full + ranking pipeline runs in the constructor. Pass ``False`` to build the + object without side effects and call :meth:`run` explicitly — useful + for testing, staged execution, or introspecting intermediate state. + :class:`AlphaDeesp_warmStart` is the pre-existing "skip the pipeline" + path and is now expressible as ``auto_run=False``. + """ self.bag_of_graphs: Dict[str, Any] = {} self.debug = debug self.boolean_dump_data_to_file = False @@ -47,6 +57,24 @@ def __init__( self.initial_graph = self.g.copy() self.substation_in_cooldown = substation_in_cooldown if substation_in_cooldown is not None else [] + # Pipeline results — populated by run(); initialised empty so the object + # is well-formed (no AttributeError) even when auto_run is False. + self.g_distribution_graph: Any = None + self.rankedLoopBuses: Dict[Any, float] = {} + self.structured_topological_actions: Dict[int, Any] = {} + self.ranked_combinations: List[pd.DataFrame] = [] + + if auto_run: + self.run() + + def run(self) -> "AlphaDeesp": + """Execute the full ranking pipeline and cache the results on ``self``. + + Idempotent-by-recompute: builds the structured distribution graph, ranks + red loops and loop buses, identifies routing buses and scores the + candidate topologies. Returns ``self`` so callers can chain + ``AlphaDeesp(..., auto_run=False).run().get_ranked_combinations()``. + """ self.g_distribution_graph = Structured_Overload_Distribution_Graph(self.g) self.rank_red_loops() @@ -58,6 +86,8 @@ def __init__( if self.boolean_dump_data_to_file: self.ranked_combinations[0].to_csv("./result_ranked_combinations.csv", index=True) + return self + def get_ranked_combinations(self) -> List[pd.DataFrame]: return self.ranked_combinations @@ -162,19 +192,23 @@ def rank_topologies( return pd.DataFrame(columns=["score", "topology", "node"], data=scores_data) def sort_hubs(self, hubs: Optional[List[Any]]) -> Optional[pd.DataFrame]: - """Sort hubs by largest absolute incident delta-flow; None if no hubs.""" + """Sort hubs by largest absolute incident delta-flow; None if no hubs. + + Vectorised: the total absolute delta-flow entering (``idx_ex``) and + leaving (``idx_or``) every node is computed once with two group-sums + instead of re-scanning the whole DataFrame for each hub. + """ if not hubs: return None - flows = [] - for node in hubs: - ingoing, outgoing = [], [] - for _, row in self.df.iterrows(): - if row["idx_or"] == node: - outgoing.append(abs(row["delta_flows"])) - if row["idx_ex"] == node: - ingoing.append(abs(row["delta_flows"])) - flows.append(max(sum(ingoing), sum(outgoing))) + abs_delta = self.df["delta_flows"].abs() + out_sum = abs_delta.groupby(self.df["idx_or"]).sum() + in_sum = abs_delta.groupby(self.df["idx_ex"]).sum() + + flows = [ + max(float(out_sum.get(node, 0.0)), float(in_sum.get(node, 0.0))) + for node in hubs + ] df = pd.DataFrame({"hubs": hubs, "max_flows": flows}) df.sort_values("max_flows", ascending=False, inplace=True) @@ -200,9 +234,15 @@ def identify_routing_buses(self) -> Dict[int, Any]: def rank_loop_buses( self, graph: nx.MultiDiGraph, df_initial_flows: pd.DataFrame ) -> Dict[Any, float]: - """Score each intermediate bus of every red loop by (non_red_inflow + local_production) * red_inflow_delta.""" + """Score each intermediate bus of every red loop by (non_red_inflow + local_production) * red_inflow_delta. + + The per-``(source, target)`` initial-inflow lookup is built once from + ``df_initial_flows`` instead of re-scanning the flow arrays for every + non-red inflow edge of every candidate bus (was O(buses x edges x rows)). + """ color_attrs = nx.get_edge_attributes(graph, "color") label_attrs = nx.get_edge_attributes(graph, "label") + inflow_lookup = self._build_inflow_lookup(df_initial_flows) strength_by_bus: Dict[Any, float] = {} red_loops = self.g_distribution_graph.get_loops() @@ -211,7 +251,8 @@ def rank_loop_buses( if bus == loop.Source or bus == loop.Target: continue strength_by_bus[bus] = self._bus_loop_strength( - bus, df_initial_flows, color_attrs, label_attrs) + bus, df_initial_flows, color_attrs, label_attrs, + inflow_lookup=inflow_lookup) return strength_by_bus def _bus_loop_strength( @@ -220,8 +261,14 @@ def _bus_loop_strength( df_initial_flows: pd.DataFrame, color_attrs: Dict[Any, Any], label_attrs: Dict[Any, Any], + inflow_lookup: Optional[Dict[Any, float]] = None, ) -> float: - """Compute the red-loop strength measure for a single intermediate bus.""" + """Compute the red-loop strength measure for a single intermediate bus. + + When *inflow_lookup* is supplied (see :meth:`_build_inflow_lookup`) the + non-red initial inflow is read in O(1); otherwise it falls back to the + linear :meth:`_initial_inflow_between` scan (kept for direct callers). + """ red_delta_in = 0.0 non_red_in = 0.0 for edge in self.g.in_edges(bus, keys=True): @@ -229,10 +276,36 @@ def _bus_loop_strength( red_delta_in += float(label_attrs[edge]) else: other = edge[0] if edge[0] != bus else edge[1] - non_red_in += self._initial_inflow_between(df_initial_flows, other, bus) + if inflow_lookup is not None: + non_red_in += inflow_lookup.get((other, bus), 0.0) + else: + non_red_in += self._initial_inflow_between(df_initial_flows, other, bus) total_in = non_red_in + self._local_production_at_bus(bus) return total_in * red_delta_in + @staticmethod + def _build_inflow_lookup(df_initial_flows: pd.DataFrame) -> Dict[Any, float]: + """Precompute ``(source, target) -> |init_flow|`` once for the whole grid. + + Mirrors the first-match, sign-aware orientation logic of + :meth:`_initial_inflow_between`: a row carries power from ``idx_or`` to + ``idx_ex`` when ``init_flow >= 0`` and the reverse when ``init_flow <= + 0`` (a zero-flow row registers both orientations at ``0``). The first + row (in DataFrame order) producing a given key wins, matching the + original linear-scan semantics exactly. + """ + lookup: Dict[Any, float] = {} + or_arr = df_initial_flows["idx_or"].to_numpy() + ex_arr = df_initial_flows["idx_ex"].to_numpy() + fl_arr = df_initial_flows["init_flows"].to_numpy() + for o, e, f in zip(or_arr, ex_arr, fl_arr): + abs_f = float(np.abs(f)) + if f >= 0: + lookup.setdefault((o, e), abs_f) + if f <= 0: + lookup.setdefault((e, o), abs_f) + return lookup + def _local_production_at_bus(self, bus: Any) -> float: """Sum production values attached to *bus* (0 if none).""" total = 0.0 @@ -279,10 +352,15 @@ def rank_red_loops(self) -> None: red_loops["min_cut_edges"] = cut_sets def to_DiGraph(self, gM: nx.MultiDiGraph) -> nx.DiGraph: - """Flatten a MultiDiGraph to a DiGraph by summing parallel edge capacities.""" + """Flatten a MultiDiGraph to a DiGraph by summing parallel edge capacities. + + A genuinely missing ``capacity`` contributes ``0.0`` (a neutral edge in + the min-cut used by :meth:`rank_red_loops`), not a spurious unit weight + that would skew the cut. + """ G = nx.DiGraph() for u, v, _, data in gM.edges(data=True, keys=True): - w = data.get("capacity", 1.0) + w = data.get("capacity", 0.0) if G.has_edge(u, v): G[u][v]["capacity"] += w else: @@ -305,7 +383,13 @@ def filter_constrained_path(self, path_to_filter: Any) -> List[Any]: class AlphaDeesp_warmStart(AlphaDeesp): - """Skip the expensive pipeline; caller supplies a pre-built distribution graph.""" + """Skip the expensive pipeline; caller supplies a pre-built distribution graph. + + Equivalent in spirit to ``AlphaDeesp(..., auto_run=False)`` with the + distribution graph injected, but kept as a distinct class (external code — + e.g. the recommender — imports it directly) and deliberately avoids the base + constructor's ``initial_graph = g.copy()`` so warm starts stay cheap. + """ def __init__( self, @@ -320,3 +404,8 @@ def __init__( self.g = g self.g_distribution_graph = g_distribution_graph self.simulator_data = simulator_data + # Well-formedness: the pipeline hasn't run, but the result attributes + # exist (empty) so accessing them never raises AttributeError. + self.rankedLoopBuses: Dict[Any, float] = {} + self.structured_topological_actions: Dict[int, Any] = {} + self.ranked_combinations: List[pd.DataFrame] = [] diff --git a/alphaDeesp/core/graphs/__init__.py b/alphaDeesp/core/graphs/__init__.py index dae6fa7b..17f9adf2 100644 --- a/alphaDeesp/core/graphs/__init__.py +++ b/alphaDeesp/core/graphs/__init__.py @@ -6,6 +6,16 @@ """ from alphaDeesp.core.graphs.constants import default_voltage_colors +from alphaDeesp.core.graphs.edge_roles import ( + EDGE_ROLE_INSIGNIFICANT, + EDGE_ROLE_NEGATIVE, + EDGE_ROLE_NULL_NON_RECONNECTABLE, + EDGE_ROLE_OVERLOAD, + EDGE_ROLE_POSITIVE, + EDGE_ROLE_UNKNOWN, + base_color_of, + edge_role_of, +) from alphaDeesp.core.graphs.graph_utils import ( all_simple_edge_paths_multi, delete_color_edges, @@ -28,14 +38,24 @@ from alphaDeesp.core.graphs.structured_overload_graph import ( Structured_Overload_Distribution_Graph, ) +from alphaDeesp.core.graphs.overflow_renderer import OverflowGraphRenderer from alphaDeesp.core.graphs.overflow_graph import OverFlowGraph __all__ = [ "default_voltage_colors", "PowerFlowGraph", "OverFlowGraph", + "OverflowGraphRenderer", "ConstrainedPath", "Structured_Overload_Distribution_Graph", + "edge_role_of", + "base_color_of", + "EDGE_ROLE_OVERLOAD", + "EDGE_ROLE_NEGATIVE", + "EDGE_ROLE_POSITIVE", + "EDGE_ROLE_INSIGNIFICANT", + "EDGE_ROLE_NULL_NON_RECONNECTABLE", + "EDGE_ROLE_UNKNOWN", "from_edges_get_nodes", "delete_color_edges", "nodepath_to_edgepath", diff --git a/alphaDeesp/core/graphs/edge_roles.py b/alphaDeesp/core/graphs/edge_roles.py new file mode 100644 index 00000000..071c25b2 --- /dev/null +++ b/alphaDeesp/core/graphs/edge_roles.py @@ -0,0 +1,60 @@ +"""Semantic edge roles for overflow graphs. + +The overflow *model* (:class:`~alphaDeesp.core.graphs.overflow_graph.OverFlowGraph`) +encodes each edge's role as a **base colour**. This module is the single +authority that maps that base colour to a stable semantic *role* — so +consumers (this package, the interactive viewer, downstream repos) never have +to parse Graphviz ``color`` strings, including the compound +``"colour:yellow:colour"`` highlight form produced by the renderer. + +The model stays authoritative: the renderer *derives* the displayed colour +from the role/base colour, and when it wraps a base colour into a compound +highlight string it records the untouched base under the ``base_color`` edge +attribute. :func:`base_color_of` therefore prefers ``base_color`` and only +falls back to (compound-safe) parsing of ``color`` for graphs produced by +older code or another repository. +""" + +from typing import Any, Dict + +# Semantic roles — stable identifiers, independent of the rendered colour. +EDGE_ROLE_OVERLOAD = "overload" # base colour black +EDGE_ROLE_NEGATIVE = "negative" # base colour blue +EDGE_ROLE_POSITIVE = "positive" # base colour coral +EDGE_ROLE_INSIGNIFICANT = "insignificant" # base colour gray +EDGE_ROLE_NULL_NON_RECONNECTABLE = "null_non_reconnectable" # base colour dimgray +EDGE_ROLE_UNKNOWN = "unknown" + +_BASE_COLOR_TO_ROLE: Dict[str, str] = { + "black": EDGE_ROLE_OVERLOAD, + "blue": EDGE_ROLE_NEGATIVE, + "coral": EDGE_ROLE_POSITIVE, + "gray": EDGE_ROLE_INSIGNIFICANT, + "grey": EDGE_ROLE_INSIGNIFICANT, + "dimgray": EDGE_ROLE_NULL_NON_RECONNECTABLE, + "dimgrey": EDGE_ROLE_NULL_NON_RECONNECTABLE, +} + + +def base_color_of(edge_data: Dict[str, Any]) -> str: + """Return an edge's authoritative base colour (lower-cased). + + Prefers the stable ``base_color`` attribute; otherwise falls back to the + ``color`` attribute, stripping any compound ``"c:yellow:c"`` wrapper. Returns + ``""`` when neither is a usable string. + """ + colour = edge_data.get("base_color") or edge_data.get("color", "") + if not isinstance(colour, str): + return "" + # Compound Graphviz colours look like '"black:yellow:black"'. + return colour.split(":", 1)[0].strip().strip('"').lower() + + +def edge_role_of(edge_data: Dict[str, Any]) -> str: + """Return the semantic role of an edge from its authoritative base colour. + + :param edge_data: a NetworkX edge attribute dict (e.g. ``g.edges[u, v, k]``). + :returns: one of the ``EDGE_ROLE_*`` constants (``EDGE_ROLE_UNKNOWN`` if the + base colour is unrecognised). + """ + return _BASE_COLOR_TO_ROLE.get(base_color_of(edge_data), EDGE_ROLE_UNKNOWN) diff --git a/alphaDeesp/core/graphs/graph_consolidation.py b/alphaDeesp/core/graphs/graph_consolidation.py index 134d47ac..c7e5e0e7 100644 --- a/alphaDeesp/core/graphs/graph_consolidation.py +++ b/alphaDeesp/core/graphs/graph_consolidation.py @@ -24,6 +24,15 @@ logger = logging.getLogger(__name__) +# Optional bound on the number of *edges* in a simple path enumerated during +# consolidation (networkx ``cutoff`` counts edges, unlike rustworkx which +# counts nodes). Enumerating all simple paths is combinatorial and can hang on +# large grids. Like the loop-path cutoff this is **OFF by default** (``None`` +# == unbounded == original behaviour) so it can never silently prune a +# legitimate consolidation path; pass an int to opt into a bound on grids +# where enumeration is a problem. +DEFAULT_CONSOLIDATION_PATH_CUTOFF = None + class GraphConsolidationMixin: """Graph consolidation and flow-direction helpers; mixed into OverFlowGraph.""" @@ -52,10 +61,11 @@ def consolidate_constrained_path( self._recolor_ambiguous_as_blue(g_c, sources) def _recolor_ambiguous_as_blue( - self, g_c: nx.MultiDiGraph, sources: Iterable[Any] + self, g_c: nx.MultiDiGraph, sources: Iterable[Any], + cutoff: Optional[int] = DEFAULT_CONSOLIDATION_PATH_CUTOFF, ) -> None: """Recolour non-{blue, black} edges on cycles within g_c to blue on self.g.""" - paths = list(all_simple_edge_paths_multi(g_c, sources, sources)) + paths = list(all_simple_edge_paths_multi(g_c, sources, sources, cutoff=cutoff)) if not paths: return colors = nx.get_edge_attributes(g_c, 'color') @@ -118,6 +128,7 @@ def consolidate_loop_path( hub_sources: Iterable[Any], hub_targets: Iterable[Any], ignore_null_edges: bool = True, + cutoff: Optional[int] = DEFAULT_CONSOLIDATION_PATH_CUTOFF, ) -> None: """Recolour gray edges on loop paths between hubs to coral.""" all_edges_to_recolor = [] @@ -129,7 +140,7 @@ def consolidate_loop_path( [e for e, cap in init_capacity.items() if cap == 0.]) for source, target in zip(hub_sources, hub_targets): - for path in nx.all_simple_edge_paths(g_without_blue, source, target): + for path in nx.all_simple_edge_paths(g_without_blue, source, target, cutoff=cutoff): all_edges_to_recolor += path all_edges_to_recolor = set(all_edges_to_recolor) diff --git a/alphaDeesp/core/graphs/graph_utils.py b/alphaDeesp/core/graphs/graph_utils.py index d7c782c0..97cf610c 100644 --- a/alphaDeesp/core/graphs/graph_utils.py +++ b/alphaDeesp/core/graphs/graph_utils.py @@ -6,7 +6,7 @@ """ import logging -from typing import Any, Iterable, List, Optional, Tuple +from typing import Any, Iterable, List, Optional, Union import networkx as nx @@ -29,11 +29,17 @@ def from_edges_get_nodes(edges: Iterable[Any], amont_or_aval: str, constrained_e else: raise ValueError("Error in function from_edges_get_nodes") -def delete_color_edges(_g: nx.MultiDiGraph, edge_color: str) -> nx.MultiDiGraph: +def delete_color_edges(_g: nx.MultiDiGraph, edge_color: Union[str, Iterable[str]]) -> nx.MultiDiGraph: """ - Returns a copy of a graph without edges of a given color. Gray for instance, with values below a threshold of significance + Returns a copy of a graph without edges of the given colour(s). - From a given node, get blue edges (with negative overflow redispatch) that are above this node + A single colour (``"gray"``) or an iterable of colours + (``("gray", "dimgray")``) may be passed. Passing several colours at once + removes them in a *single* graph copy, which is markedly cheaper than + chaining calls (each chained call would copy the whole graph again). + The result is identical to removing the colours one after another: an + edge survives iff its colour is not in the requested set, and isolated + nodes are pruned once at the end. Parameters ---------- @@ -41,29 +47,27 @@ def delete_color_edges(_g: nx.MultiDiGraph, edge_color: str) -> nx.MultiDiGraph: _g: :class:`nx:MultiDiGraph` an overflow redispatch networkx graph - edge_color: ``str`` - color of edges to delete from graoh + edge_color: ``str`` or iterable of ``str`` + colour(s) of edges to delete from the graph Returns ---------- res: :class:`nx:MultiDiGraph` - the graph without edges for the targeted color + the graph without edges for the targeted colour(s) """ + colors_to_delete = {edge_color} if isinstance(edge_color, str) else set(edge_color) g = _g.copy() - TargetColor_edges = [] - i = 1 - for u, v,idx, color in g.edges(data="color",keys=True): - if color == edge_color: - TargetColor_edges.append((i, (u, v,idx))) - i += 1 - - # delete from graph gray edges - # this extracts the (u,v) from pos_edges - if TargetColor_edges: - g.remove_edges_from(list(zip(*TargetColor_edges))[1]) + target_edges = [ + (u, v, idx) + for u, v, idx, color in g.edges(keys=True, data="color") + if color in colors_to_delete + ] + + if target_edges: + g.remove_edges_from(target_edges) g.remove_nodes_from(list(nx.isolates(g))) return g diff --git a/alphaDeesp/core/graphs/null_flow_graph.py b/alphaDeesp/core/graphs/null_flow_graph.py index eddb5658..c659adef 100644 --- a/alphaDeesp/core/graphs/null_flow_graph.py +++ b/alphaDeesp/core/graphs/null_flow_graph.py @@ -14,7 +14,6 @@ import networkx as nx from alphaDeesp.core.graphs.graph_utils import ( - all_simple_edge_paths_multi, find_multidigraph_edges_by_name, nodepath_to_edgepath, ) @@ -57,8 +56,16 @@ def add_relevant_null_flow_lines_all_paths( structured_graph: Any, non_connected_lines: List[Any], non_reconnectable_lines: List[Any] = [], + capacity_weighted: bool = False, ) -> None: - """Apply null-flow logic for all four target-path strategies.""" + """Apply null-flow logic for all four target-path strategies. + + ``capacity_weighted`` selects the null-flow path-search routing weight — + see :meth:`_compute_sssp_paths` (issue #1). ``False`` (default) is the + "bless" mode: hop-cost-only routing, bit-identical to the historical + behaviour. ``True`` is the capacity-weighted "fix" mode; downstream + callers (e.g. the recommender) opt into it here. + """ non_connected_lines = self._setup_null_flow_styles(non_connected_lines, non_reconnectable_lines) structural_info = self._structural_info_for_null_flow(structured_graph) @@ -68,7 +75,8 @@ def add_relevant_null_flow_lines_all_paths( structured_graph, non_connected_lines, non_reconnectable_lines, target_path=target_path, _skip_style_setup=True, - _structural_info=structural_info) + _structural_info=structural_info, + capacity_weighted=capacity_weighted) def add_relevant_null_flow_lines( self, @@ -80,8 +88,13 @@ def add_relevant_null_flow_lines( max_null_flow_path_length: int = 7, _skip_style_setup: bool = False, _structural_info: Optional[Dict[str, Any]] = None, + capacity_weighted: bool = False, ) -> None: - """Make null-flow edges bidirectional and recolour relevant ones.""" + """Make null-flow edges bidirectional and recolour relevant ones. + + ``capacity_weighted`` is forwarded to the path search — see + :meth:`_compute_sssp_paths` (issue #1 "bless or fix"). + """ if not _skip_style_setup: non_connected_lines = self._setup_null_flow_styles( non_connected_lines, non_reconnectable_lines) @@ -109,6 +122,7 @@ def add_relevant_null_flow_lines( edges_non_reconnectable_lines, depth_reconnectable_edges_search, max_null_flow_path_length, + capacity_weighted, ) self._apply_null_flow_recoloring( @@ -196,6 +210,7 @@ def _detect_edges_for_target_path( edges_non_reconnectable_lines: Set[Any], depth_reconnectable_edges_search: int, max_null_flow_path_length: int, + capacity_weighted: bool = False, ) -> Tuple[Set[Any], Set[Any]]: """Per-component dispatch to detect_edges_to_keep for the chosen strategy.""" node_red_paths = structural_info["node_red_paths"] @@ -210,7 +225,8 @@ def _run(g_c: Any, sources: Any, targets: Any) -> None: g_c, sources, targets, edges_non_connected_lines, edges_non_reconnectable_lines, depth_edges_search=depth_reconnectable_edges_search, - max_null_flow_path_length=max_null_flow_path_length) + max_null_flow_path_length=max_null_flow_path_length, + capacity_weighted=capacity_weighted) edges_to_keep.update(keep) edges_non_reconnectable.update(non_rec) @@ -314,6 +330,7 @@ def detect_edges_to_keep( non_reconnectable_edges: List[Any] = [], depth_edges_search: int = 2, max_null_flow_path_length: int = 7, + capacity_weighted: bool = False, ) -> Tuple[Set[Any], Set[Any]]: """Detect edges of interest on short paths between source and target nodes.""" prepared = self._prepare_detect_edges_inputs( @@ -322,7 +339,8 @@ def detect_edges_to_keep( if prepared is None: return set(), set() - sssp_paths_cache = self._compute_sssp_paths(g_c, prepared, edges_of_interest) + sssp_paths_cache = self._compute_sssp_paths( + g_c, prepared, edges_of_interest, capacity_weighted=capacity_weighted) paths_of_interest = self._collect_paths_of_interest( g_c, prepared, sssp_paths_cache, max_null_flow_path_length) return self._classify_paths_by_reconnectability(prepared, paths_of_interest) @@ -429,19 +447,60 @@ def _compute_sssp_paths( g_c: nx.MultiDiGraph, prepared: Dict[str, Any], edges_of_interest: Set[Any], + capacity_weighted: bool = False, ) -> Dict[Any, Any]: - """Run single-source Dijkstra per source with an incentivised weight function.""" + """Single-source Dijkstra per source over a *precomputed* edge weight. + + The routing weight is materialised once as an edge attribute and Dijkstra + is run with a **string** weight instead of a per-edge-relaxation Python + callable — the callable is networkx's slowest weight mode and this search + is the dominant non-load-flow cost of the overflow-graph build at national + scale (see issue #1). + + Two weighting modes (issue #1, "bless or fix"): + + * ``capacity_weighted=False`` (**default — "Option A", bless**): reproduces + the historical *effective* behaviour on the overflow ``MultiDiGraph``. + networkx hands a callable weight the ``{key: attr}`` parallel-edge view, + so the old ``attr.get("capacity", 0)`` silently read ``0`` and the + ``(u, v)`` promoted test never matched the ``(u, v, key)`` set — routing + was a *uniform hop cost*. We keep exactly that (weight ≡ hop cost) but + make it explicit and fast. Output is bit-identical to the pre-refactor + code. + * ``capacity_weighted=True`` (**"Option B", fix**): capacity-weighted + routing as originally intended — the minimum capacity across parallel + edges dominates (``capacity * HUGE + hop``), with correct multigraph + promoted matching (``(u, v)`` or the exact ``(u, v, key)``). **This + changes routing** and should be validated on reference cases; callers + opt in via ``add_relevant_null_flow_lines[_all_paths](..., + capacity_weighted=True)``. + """ HUGE_MULTIPLIER = 1_000_000_000 NORMAL_HOP_COST = 100 PROMOTED_HOP_COST = 33 - promoted_set = set(edges_of_interest) - - def incentivized_weight(u: Any, v: Any, attr: Dict[str, Any]) -> float: - real_weight = attr.get("capacity", 0) - if real_weight < 0: - raise ValueError("Negative weights not allowed.") - hop_cost = PROMOTED_HOP_COST if (u, v) in promoted_set else NORMAL_HOP_COST - return (real_weight * HUGE_MULTIPLIER) + hop_cost + WEIGHT_ATTR = "_nf_weight" + + promoted_edges = set(edges_of_interest) # (u, v, key) tuples + promoted_pairs = {(e[0], e[1]) for e in promoted_edges} # (u, v) pairs + + # One O(E) pass: materialise the routing weight on every edge so Dijkstra + # can use a fast string weight (min over parallel edges for a multigraph). + weights: Dict[Any, float] = {} + for u, v, key, data in g_c.edges(keys=True, data=True): + if capacity_weighted: + capacity = data.get("capacity", 0) + if capacity < 0: + raise ValueError("Negative weights not allowed.") + promoted = (u, v) in promoted_pairs or (u, v, key) in promoted_edges + base = capacity * HUGE_MULTIPLIER + else: + # Bless: capacity ignored (as the old callable did on a multigraph) + # and the historical 2-tuple-vs-3-tuple promoted test, which never + # engaged — i.e. a uniform hop weight. Bit-identical to before. + promoted = (u, v) in promoted_edges + base = 0 + weights[(u, v, key)] = base + (PROMOTED_HOP_COST if promoted else NORMAL_HOP_COST) + nx.set_edge_attributes(g_c, weights, WEIGHT_ATTR) bfs_cache = prepared["bfs_cache"] targets_with_bfs = prepared["targets_with_bfs"] @@ -456,8 +515,15 @@ def incentivized_weight(u: Any, v: Any, attr: Dict[str, Any]) -> float: continue try: sssp_paths_cache[source_node] = nx.single_source_dijkstra_path( - g_c, source_node, weight=incentivized_weight) - except Exception: + g_c, source_node, weight=WEIGHT_ATTR) + except (nx.NetworkXException, ValueError) as exc: + # Expected failure modes: source absent from the component + # (NetworkXException) or a negative capacity reaching the + # incentivised weight fn (ValueError). Anything else is a real + # bug and is left to propagate rather than silently masked. + logger.warning( + "single_source_dijkstra_path failed at node %s: %s; " + "treating as no reachable paths.", source_node, exc) sssp_paths_cache[source_node] = {} return sssp_paths_cache diff --git a/alphaDeesp/core/graphs/overflow_graph.py b/alphaDeesp/core/graphs/overflow_graph.py index d2931e2a..488f0d85 100644 --- a/alphaDeesp/core/graphs/overflow_graph.py +++ b/alphaDeesp/core/graphs/overflow_graph.py @@ -1,38 +1,45 @@ -"""OverFlowGraph: coloured overflow-redispatch graph. +"""OverFlowGraph: the coloured overflow-redispatch *semantic model*. Subclasses :class:`PowerFlowGraph`, :class:`NullFlowGraphMixin`, and :class:`GraphConsolidationMixin`. The null-flow and consolidation logic live in the mixins to keep per-file complexity within A-grade bounds. + +Responsibility split +-------------------- +``OverFlowGraph`` owns the **semantic model** of the overflow: the graph +topology, per-edge redispatch magnitudes, the *role* of each edge encoded as +a base colour (``black`` overload / ``blue`` negative / ``coral`` positive / +``gray`` insignificant), and the boolean semantic flags consumed downstream +(``is_overload``, ``is_monitored``, ``on_constrained_path``, ``in_red_loop``, +``is_hub``, ``is_extra_cut``). + +Everything that is *purely Graphviz presentation* — penwidth scaling, node +shapes, tapered swap styling, the compound ``"colour:yellow:colour"`` +highlight strings and the HTML loading labels, and the actual plotting — +lives in :class:`~alphaDeesp.core.graphs.overflow_renderer.OverflowGraphRenderer`. +The methods below keep their public signatures and delegate the rendering to +that stateless renderer, so downstream repositories importing ``OverFlowGraph`` +are unaffected while the two concerns stay cleanly separated. """ import logging -from math import fabs -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple import networkx as nx -import numpy as np import pandas as pd -from alphaDeesp.core.printer import Printer from alphaDeesp.core.graphs.power_flow_graph import PowerFlowGraph from alphaDeesp.core.graphs.null_flow_graph import NullFlowGraphMixin from alphaDeesp.core.graphs.graph_consolidation import GraphConsolidationMixin from alphaDeesp.core.graphs.graph_utils import delete_color_edges +from alphaDeesp.core.graphs.overflow_renderer import OverflowGraphRenderer +from alphaDeesp.core.graphs.edge_roles import EDGE_ROLE_POSITIVE, edge_role_of logger = logging.getLogger(__name__) -# Penwidth thresholds used by build_edges_from_df. -# The floor is dynamic: at least the width equivalent to 1 MW of flow -# (``scaling_factor`` applied to 1.0), and at least 10 % of the largest -# rendered penwidth, so low / zero-flow edges (reconnectable, -# non-reconnectable, null-flow) remain visible without zooming. -_TARGET_MAX_PENWIDTH = 15.0 -_MIN_PENWIDTH_FLOW_MW = 1.0 -_MIN_PENWIDTH_FRACTION = 0.10 - class OverFlowGraph(NullFlowGraphMixin, GraphConsolidationMixin, PowerFlowGraph): - """A coloured graph of grid overflow redispatch.""" + """A coloured semantic graph of grid overflow redispatch.""" def __init__( self, @@ -43,13 +50,17 @@ def __init__( float_precision: str = "%.2f", extra_lines_to_cut: Optional[Iterable[int]] = None, ) -> None: - if "line_name" not in df_overflow.columns: - df_overflow["line_name"] = [ + # Work on a copy so the caller's DataFrame is never mutated (this + # class adds a ``line_name`` column and ``rename_nodes`` rewrites the + # endpoint columns — surprising side effects for an external caller). + df = df_overflow.copy() + if "line_name" not in df.columns: + df["line_name"] = [ str(idx_or) + "_" + str(idx_ex) + "_" + str(i) - for i, (idx_or, idx_ex) in df_overflow[["idx_or", "idx_ex"]].iterrows() + for i, (idx_or, idx_ex) in df[["idx_or", "idx_ex"]].iterrows() ] - self.df = df_overflow + self.df = df # Subset of ``lines_to_cut`` that the caller wants the cut-analysis # to treat like overloads (so they get the same black/constrained # styling and feed the structured-overload graph the same way) but @@ -77,12 +88,8 @@ def build_graph(self) -> None: def build_edges_from_df(self, g: nx.MultiDiGraph, lines_to_cut: List[int]) -> None: """Add one coloured edge per row of self.df to g.""" - max_abs_flow = self.df["delta_flows"].abs().max() - scaling_factor = _TARGET_MAX_PENWIDTH / max_abs_flow if max_abs_flow > 0 else 1.0 - min_penwidth = max( - _MIN_PENWIDTH_FLOW_MW * scaling_factor, - _MIN_PENWIDTH_FRACTION * _TARGET_MAX_PENWIDTH, - ) + scaling_factor, min_penwidth = OverflowGraphRenderer.penwidth_scaling( + self.df["delta_flows"]) cols = ("idx_or", "idx_ex", "delta_flows", "gray_edges", "line_name") # Operator-selected extras must NOT be coloured black: black is the @@ -129,7 +136,8 @@ def _add_overflow_edge( ) -> None: """Add a single styled overflow edge to g.""" fp = self.float_precision - penwidth = max(float(fp % (fabs(reported_flow) * scaling_factor)), min_penwidth) + penwidth = OverflowGraphRenderer.edge_penwidth( + reported_flow, scaling_factor, min_penwidth, fp) attrs = { "capacity": float(fp % reported_flow), "label": fp % reported_flow, @@ -176,12 +184,13 @@ def set_hubs_shape(self, hubs: Iterable[Any], shape_hub: str = "circle") -> None layer toggle stays consistent regardless of which other tagging method (``tag_constrained_path`` / ``collapse_red_loops``) has already run. + + The node *shape* itself (the Graphviz presentation) is applied by + :meth:`OverflowGraphRenderer.set_hub_shapes`; only the semantic flags + are owned here. """ - dict_shapes = {node: "oval" for node in self.g.nodes} hubs_set = set(hubs) - for hub in hubs_set: - dict_shapes[hub] = shape_hub - nx.set_node_attributes(self.g, dict_shapes, "shape") + OverflowGraphRenderer.set_hub_shapes(self.g, hubs_set, shape_hub) nx.set_node_attributes( self.g, {node: (node in hubs_set) for node in self.g.nodes}, "is_hub" ) @@ -195,10 +204,7 @@ def set_hubs_shape(self, hubs: Iterable[Any], shape_hub: str = "circle") -> None def highlight_swapped_flows(self, lines_swapped: List[Any]) -> None: """Draw lines whose flow direction has swapped in a tapered style.""" - edge_names = nx.get_edge_attributes(self.g, "name") - swapped_edges = [edge for edge, name in edge_names.items() if name in lines_swapped] - for attr_name, value in (("style", "tapered"), ("dir", "both"), ("arrowtail", "none")): - nx.set_edge_attributes(self.g, {edge: value for edge in swapped_edges}, attr_name) + OverflowGraphRenderer.highlight_swapped_flows(self.g, lines_swapped) def highlight_significant_line_loading(self, dict_line_loading: Dict[Any, Any]) -> None: """Augment edge labels with loading rates for monitored lines. @@ -214,6 +220,9 @@ def highlight_significant_line_loading(self, dict_line_loading: Dict[Any, Any]) that are overloaded contingency lines (current colour was ``black`` before the highlight). Overloads are therefore a subset of low-margin lines, not a disjoint category. + + The semantic flags are owned here; the label / compound-colour + *formatting* is delegated to :class:`OverflowGraphRenderer`. """ edge_names = nx.get_edge_attributes(self.g, "name") edge_colors = nx.get_edge_attributes(self.g, "color") @@ -224,6 +233,7 @@ def highlight_significant_line_loading(self, dict_line_loading: Dict[Any, Any]) is_overload_attrs: Dict[Any, bool] = {} is_monitored_attrs: Dict[Any, bool] = {} + base_color_attrs: Dict[Any, Any] = {} for edge, edge_name in edge_names.items(): if edge_name not in dict_line_loading: @@ -243,22 +253,32 @@ def highlight_significant_line_loading(self, dict_line_loading: Dict[Any, Any]) if not is_extra: is_monitored_attrs[edge] = True if current_edge_color == "black": - edge_x_labels[edge] = f'< {current_x_label}
{before}% → {after}%>' + edge_x_labels[edge] = OverflowGraphRenderer.overload_label( + current_x_label, before, after) is_overload_attrs[edge] = True else: - edge_x_labels[edge] = f'< {current_x_label}
{before}% → {after}% >' - edge_colors[edge] = f'"{current_edge_color}:yellow:{current_edge_color}"' + edge_x_labels[edge] = OverflowGraphRenderer.low_margin_label( + current_x_label, before, after) + # Wrapping the base colour into a compound "c:yellow:c" string is + # a *rendering* step; record the untouched base colour so the + # model stays authoritative and ``edge_role_of`` never has to + # parse the compound (see :mod:`alphaDeesp.core.graphs.edge_roles`). + base_color_attrs[edge] = current_edge_color + edge_colors[edge] = OverflowGraphRenderer.highlight_color(current_edge_color) else: # Extras keep their natural flow colour; only the - # ``before → 0%`` annotation surfaces the cut so the + # ``before → after`` annotation surfaces the cut so the # operator sees how their choice materialises. - edge_x_labels[edge] = f'< {current_x_label}
{before}% → {after}% >' + edge_x_labels[edge] = OverflowGraphRenderer.low_margin_label( + current_x_label, before, after) label_font_color[edge] = color_label_highlight nx.set_edge_attributes(self.g, edge_x_labels, "label") nx.set_edge_attributes(self.g, label_font_color, "fontcolor") nx.set_edge_attributes(self.g, edge_colors, "color") + if base_color_attrs: + nx.set_edge_attributes(self.g, base_color_attrs, "base_color") if is_overload_attrs: nx.set_edge_attributes(self.g, is_overload_attrs, "is_overload") if is_monitored_attrs: @@ -274,26 +294,39 @@ def plot( save_folder: str = "", without_gray_edges: bool = False, ) -> Any: - printer = Printer(save_folder) - g = self.g - - if without_gray_edges: - layout_dict = {n: c for n, c in zip(g.nodes, layout)} if layout is not None else None - g = delete_color_edges(g, "gray") - if layout_dict is not None: - layout = [layout_dict[node] for node in g.nodes] - - kwargs = dict(rescale_factor=rescale_factor, fontsize=fontsize, - node_thickness=node_thickness, name="g_overflow_print") - if save_folder == "": - return printer.plot_graphviz(g, layout, allow_overlap=allow_overlap, **kwargs) - printer.display_geo(g, layout, **kwargs) + """Render the graph via :class:`OverflowGraphRenderer`.""" + return OverflowGraphRenderer.plot( + self.g, layout, + rescale_factor=rescale_factor, + allow_overlap=allow_overlap, + fontsize=fontsize, + node_thickness=node_thickness, + save_folder=save_folder, + without_gray_edges=without_gray_edges, + ) + + def edge_role(self, name: Any) -> Optional[str]: + """Return the semantic role of the first edge carrying line ``name``. + + Reads the authoritative base colour (see + :func:`~alphaDeesp.core.graphs.edge_roles.edge_role_of`) so callers + never parse the rendered — possibly compound ``"c:yellow:c"`` — ``color`` + string. Returns one of the ``EDGE_ROLE_*`` constants, or ``None`` when no + edge carries that name. + + A physical line may appear as two directed edges (e.g. a ``blue`` and a + ``coral`` direction); this returns the first match. When the direction + matters, iterate edges and call ``edge_role_of`` per edge instead. + """ + for _, _, data in self.g.edges(data=True): + if data.get("name") == name: + return edge_role_of(data) return None def rename_nodes(self, mapping: Dict[Any, Any]) -> None: self.g = nx.relabel_nodes(self.g, mapping, copy=True) self.df["idx_or"] = [mapping[idx_or] for idx_or in self.df["idx_or"]] - self.df["idx_ex"] = [mapping[idx_or] for idx_or in self.df["idx_ex"]] + self.df["idx_ex"] = [mapping[idx_ex] for idx_ex in self.df["idx_ex"]] def collapse_red_loops(self) -> None: """Collapse purely-coral, non-hub nodes to point shapes. @@ -303,24 +336,9 @@ def collapse_red_loops(self) -> None: no longer derived from this collapse — it is set explicitly by :meth:`tag_red_loops` from the recommender's ``get_dispatch_edges_nodes(only_loop_paths=True)`` source-of- - truth list. + truth list. The collapse itself is delegated to the renderer. """ - shapes = nx.get_node_attributes(self.g, "shape") - peripheries = nx.get_node_attributes(self.g, "peripheries") - edge_colors = nx.get_edge_attributes(self.g, "color") - edge_styles = nx.get_edge_attributes(self.g, "style") - - nodes_to_collapse = {} - for node in self.g.nodes: - if shapes.get(node) != "oval": - continue - if node in peripheries and peripheries[node] >= 2: - continue - all_edges = list(self.g.in_edges(node, keys=True)) + list(self.g.out_edges(node, keys=True)) - if all_edges and self._all_edges_coral_no_dash(all_edges, edge_colors, edge_styles): - nodes_to_collapse[node] = "point" - - nx.set_node_attributes(self.g, nodes_to_collapse, "shape") + OverflowGraphRenderer.collapse_red_loops(self.g) def tag_red_loops( self, @@ -381,22 +399,20 @@ def tag_constrained_path( is on the constrained path; including the coral counterpart would surface positive-overflow edges in the layer toggle and confuse the operator. + + The coral test reads the edge's semantic *role* via + :func:`~alphaDeesp.core.graphs.edge_roles.edge_role_of` (which prefers + the authoritative ``base_color`` and is compound-colour safe) rather + than parsing the rendered ``color`` string here. """ if lines_constrained_path: wanted = set(lines_constrained_path) edge_names = nx.get_edge_attributes(self.g, "name") - edge_colors = nx.get_edge_attributes(self.g, "color") edge_attrs: Dict[Any, bool] = {} for edge, name in edge_names.items(): if name not in wanted: continue - color = edge_colors.get(edge, "") - base_color = ( - color.split(":", 1)[0].strip().strip('"').lower() - if isinstance(color, str) - else "" - ) - if base_color == "coral": + if edge_role_of(self.g.edges[edge]) == EDGE_ROLE_POSITIVE: continue edge_attrs[edge] = True if edge_attrs: @@ -415,10 +431,11 @@ def _all_edges_coral_no_dash( edge_colors: Dict[Any, str], edge_styles: Dict[Any, str], ) -> bool: - """Return True when all edges are coral and none are dashed/dotted.""" - for edge in all_edges: - if edge_colors.get(edge) != "coral": - return False - if edge_styles.get(edge, "") in ("dashed", "dotted"): - return False - return True + """Return True when all edges are coral and none are dashed/dotted. + + Retained for backwards compatibility (tests and external callers + reference ``OverFlowGraph._all_edges_coral_no_dash``); the + implementation lives on :class:`OverflowGraphRenderer`. + """ + return OverflowGraphRenderer._all_edges_coral_no_dash( + all_edges, edge_colors, edge_styles) diff --git a/alphaDeesp/core/graphs/overflow_renderer.py b/alphaDeesp/core/graphs/overflow_renderer.py new file mode 100644 index 00000000..db0aabcf --- /dev/null +++ b/alphaDeesp/core/graphs/overflow_renderer.py @@ -0,0 +1,202 @@ +"""OverflowGraphRenderer: Graphviz presentation for the overflow model. + +This module isolates every *Graphviz-specific rendering* concern that used +to live inside :class:`~alphaDeesp.core.graphs.overflow_graph.OverFlowGraph`: + +* penwidth scaling (edge thickness proportional to redispatch magnitude), +* node shapes (hub markers, collapsing pure-loop nodes to points), +* tapered styling for flow-direction swaps, +* the compound ``"colour:yellow:colour"`` highlight strings and the HTML + ``before% → after%`` loading labels, +* the actual plotting via :class:`~alphaDeesp.core.printer.Printer`. + +Keeping it separate draws a clean line between the **semantic model** — +edge *roles* encoded as base colours (black overload / blue negative / coral +positive / gray insignificant) plus boolean flags (``is_overload``, +``on_constrained_path`` …) owned by ``OverFlowGraph`` — and its **rendering**. +Downstream repositories that build their own semantic overflow graph can +reuse this renderer directly, and the model can be reasoned about (and +tested) without pulling in any Graphviz vocabulary. + +All methods are **static** and operate on a passed-in graph, so the renderer +carries no state and can be applied to any compatible ``MultiDiGraph``. +""" + +import logging +from math import fabs +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import networkx as nx + +from alphaDeesp.core.printer import Printer +from alphaDeesp.core.graphs.graph_utils import delete_color_edges + +logger = logging.getLogger(__name__) + +# Penwidth thresholds used when stamping edge thickness. +# The floor is dynamic: at least the width equivalent to 1 MW of flow +# (``scaling_factor`` applied to 1.0), and at least 10 % of the largest +# rendered penwidth, so low / zero-flow edges (reconnectable, +# non-reconnectable, null-flow) remain visible without zooming. +_TARGET_MAX_PENWIDTH = 15.0 +_MIN_PENWIDTH_FLOW_MW = 1.0 +_MIN_PENWIDTH_FRACTION = 0.10 + + +class OverflowGraphRenderer: + """Stateless Graphviz renderer for overflow semantic graphs.""" + + # ------------------------------------------------------------------ + # Penwidth (edge thickness) + # ------------------------------------------------------------------ + + @staticmethod + def penwidth_scaling(delta_flows: Any) -> Tuple[float, float]: + """Return ``(scaling_factor, min_penwidth)`` for a set of delta flows. + + ``scaling_factor`` maps the largest absolute flow onto + ``_TARGET_MAX_PENWIDTH``; ``min_penwidth`` is the visibility floor. + """ + max_abs_flow = abs(delta_flows).max() if len(delta_flows) else 0.0 + scaling_factor = ( + _TARGET_MAX_PENWIDTH / max_abs_flow if max_abs_flow > 0 else 1.0 + ) + min_penwidth = max( + _MIN_PENWIDTH_FLOW_MW * scaling_factor, + _MIN_PENWIDTH_FRACTION * _TARGET_MAX_PENWIDTH, + ) + return scaling_factor, min_penwidth + + @staticmethod + def edge_penwidth( + reported_flow: float, + scaling_factor: float, + min_penwidth: float, + float_precision: str, + ) -> float: + """Penwidth for a single edge, clamped to the visibility floor.""" + return max( + float(float_precision % (fabs(reported_flow) * scaling_factor)), + min_penwidth, + ) + + # ------------------------------------------------------------------ + # Node shapes + # ------------------------------------------------------------------ + + @staticmethod + def set_hub_shapes( + g: nx.MultiDiGraph, hubs: Iterable[Any], shape_hub: str = "circle" + ) -> None: + """Give every node an ``oval`` shape and hub nodes ``shape_hub``. + + This is the *visual* half of ``OverFlowGraph.set_hubs_shape``; the + semantic ``is_hub`` / ``on_constrained_path`` / ``in_red_loop`` flags + are stamped by the model. + """ + dict_shapes = {node: "oval" for node in g.nodes} + for hub in set(hubs): + dict_shapes[hub] = shape_hub + nx.set_node_attributes(g, dict_shapes, "shape") + + @staticmethod + def collapse_red_loops(g: nx.MultiDiGraph) -> None: + """Collapse purely-coral, non-hub oval nodes to ``point`` shapes. + + Purely a visual heuristic (point markers vs ovals); it sets no + semantic attribute. + """ + shapes = nx.get_node_attributes(g, "shape") + peripheries = nx.get_node_attributes(g, "peripheries") + edge_colors = nx.get_edge_attributes(g, "color") + edge_styles = nx.get_edge_attributes(g, "style") + + nodes_to_collapse = {} + for node in g.nodes: + if shapes.get(node) != "oval": + continue + if node in peripheries and peripheries[node] >= 2: + continue + all_edges = list(g.in_edges(node, keys=True)) + list(g.out_edges(node, keys=True)) + if all_edges and OverflowGraphRenderer._all_edges_coral_no_dash( + all_edges, edge_colors, edge_styles + ): + nodes_to_collapse[node] = "point" + + nx.set_node_attributes(g, nodes_to_collapse, "shape") + + @staticmethod + def _all_edges_coral_no_dash( + all_edges: List[Any], + edge_colors: Dict[Any, str], + edge_styles: Dict[Any, str], + ) -> bool: + """Return True when all edges are coral and none are dashed/dotted.""" + for edge in all_edges: + if edge_colors.get(edge) != "coral": + return False + if edge_styles.get(edge, "") in ("dashed", "dotted"): + return False + return True + + # ------------------------------------------------------------------ + # Flow-direction swap styling + # ------------------------------------------------------------------ + + @staticmethod + def highlight_swapped_flows(g: nx.MultiDiGraph, lines_swapped: List[Any]) -> None: + """Draw lines whose flow direction has swapped in a tapered style.""" + edge_names = nx.get_edge_attributes(g, "name") + swapped_edges = [edge for edge, name in edge_names.items() if name in lines_swapped] + for attr_name, value in (("style", "tapered"), ("dir", "both"), ("arrowtail", "none")): + nx.set_edge_attributes(g, {edge: value for edge in swapped_edges}, attr_name) + + # ------------------------------------------------------------------ + # Loading annotations (compound colour + HTML label) + # ------------------------------------------------------------------ + + @staticmethod + def overload_label(current_x_label: Any, before: Any, after: Any) -> str: + """HTML label for an overloaded edge (``before%`` emphasised).""" + return f'< {current_x_label}
{before}% → {after}%>' + + @staticmethod + def low_margin_label(current_x_label: Any, before: Any, after: Any) -> str: + """HTML label for a low-margin / extra-cut edge (``after%`` emphasised).""" + return f'< {current_x_label}
{before}% → {after}% >' + + @staticmethod + def highlight_color(current_edge_color: Any) -> str: + """Compound Graphviz colour that yellow-tints an edge's base colour.""" + return f'"{current_edge_color}:yellow:{current_edge_color}"' + + # ------------------------------------------------------------------ + # Plotting + # ------------------------------------------------------------------ + + @staticmethod + def plot( + g: nx.MultiDiGraph, + layout: Optional[List[Any]], + rescale_factor: Optional[float] = None, + allow_overlap: bool = True, + fontsize: Optional[int] = None, + node_thickness: int = 3, + save_folder: str = "", + without_gray_edges: bool = False, + ) -> Any: + """Render *g* with Graphviz, optionally dropping gray edges first.""" + printer = Printer(save_folder) + + if without_gray_edges: + layout_dict = {n: c for n, c in zip(g.nodes, layout)} if layout is not None else None + g = delete_color_edges(g, "gray") + if layout_dict is not None: + layout = [layout_dict[node] for node in g.nodes] + + kwargs = dict(rescale_factor=rescale_factor, fontsize=fontsize, + node_thickness=node_thickness, name="g_overflow_print") + if save_folder == "": + return printer.plot_graphviz(g, layout, allow_overlap=allow_overlap, **kwargs) + printer.display_geo(g, layout, **kwargs) + return None diff --git a/alphaDeesp/core/graphs/shortest_paths.py b/alphaDeesp/core/graphs/shortest_paths.py index 9e1fa293..ebae38fc 100644 --- a/alphaDeesp/core/graphs/shortest_paths.py +++ b/alphaDeesp/core/graphs/shortest_paths.py @@ -5,11 +5,52 @@ favour promoted edges, then minimise hop count. """ -from typing import Any, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple import networkx as nx +def _make_incentivized_weight( + G: Any, + weight_attr: str, + huge_multiplier: float, + normal_hop_cost: float, + promoted_hop_cost: float, + promoted_set: Set[Any], +) -> Any: + """Build a Dijkstra weight fn ``(physical_weight * huge) + hop_cost``. + + Physical weight dominates; ``hop_cost`` is ``promoted_hop_cost`` on a promoted + edge, else ``normal_hop_cost`` — so among equal-weight paths the one using + more promoted edges (then fewer hops) wins. + + Handles **both** graph kinds correctly. For a ``MultiDiGraph`` networkx + passes the ``{key: attr}`` view of the parallel edges, so we take the minimum + physical weight across them and treat the connection as promoted when either + ``(u, v)`` or any ``(u, v, key)`` is in ``promoted_set``. For a plain graph + the single attribute dict is used directly. (The historical closures called + ``attr.get(weight_attr)`` unconditionally, which silently read ``0`` for + every multigraph edge.) + """ + is_multi = G.is_multigraph() + + def incentivized_weight(u: Any, v: Any, attr: Dict[Any, Any]) -> float: + if is_multi: + weights = [d.get(weight_attr, 0) for d in attr.values()] + real_weight = min(weights) if weights else 0 + promoted = (u, v) in promoted_set or any( + (u, v, key) in promoted_set for key in attr.keys()) + else: + real_weight = attr.get(weight_attr, 0) + promoted = (u, v) in promoted_set + if real_weight < 0: + raise ValueError("Dijkstra does not accept negative weights.") + hop_cost = promoted_hop_cost if promoted else normal_hop_cost + return (real_weight * huge_multiplier) + hop_cost + + return incentivized_weight + + def shortest_path_min_weight_then_hops(G: Any, source: Any, target: Any, mandatory_edge: Tuple[Any, ...], weight_attr: str = "weight") -> Tuple[Optional[List[Any]], float]: """ Finds the path that: @@ -21,15 +62,11 @@ def shortest_path_min_weight_then_hops(G: Any, source: Any, target: Any, mandato # Must be larger than the max possible number of edges in a path (e.g., number of nodes). MULTIPLIER = 1_000_000 - # Define the custom weight function for Dijkstra - # Returns: (Actual_Weight * 1,000,000) + 1 - def composite_weight(u, v, attr): - # Handle MultiDiGraph: attr might be the inner dict or we might be iterating keys - # nx.dijkstra_path passes the edge attribute dictionary directly - w = attr.get(weight_attr, 0) # Default to 0 if no weight - if w < 0: - raise ValueError("Dijkstra does not accept negative weights.") - return (w * MULTIPLIER) + 1 + # Weight fn: (Actual_Weight * 1,000,000) + 1. No promotion here, so the hop + # cost is a constant 1 (pure min-weight then min-hops). + composite_weight = _make_incentivized_weight( + G, weight_attr, MULTIPLIER, + normal_hop_cost=1, promoted_hop_cost=1, promoted_set=set()) # Unpack mandatory edge u, v = mandatory_edge[0], mandatory_edge[1] @@ -96,27 +133,11 @@ def shortest_path_mandatory_and_promoted(G: Any, source: Any, target: Any, manda NORMAL_HOP_COST = 100 PROMOTED_HOP_COST = 1 - # Optimization: Set for O(1) lookup - promoted_set = set(promoted_edges) - - # --- 1. Define the Custom Weight Function --- - def incentivized_weight(u, v, attr): - # A. Physical Cost - real_weight = attr.get(weight_attr, 0) - if real_weight < 0: - raise ValueError("Dijkstra does not accept negative weights.") - - # B. Preference Cost - is_promoted = (u, v) in promoted_set - - # Note: For MultiDiGraph, strict key checking would require iterating G[u][v] - # or checking if ANY parallel edge is promoted. - # Here we assume if the connection (u,v) is promoted, we take the bonus. - - hop_cost = PROMOTED_HOP_COST if is_promoted else NORMAL_HOP_COST - - # Formula: (Physical_Weight * HUGE) + Preference_Cost - return (real_weight * HUGE_MULTIPLIER) + hop_cost + # --- 1. Custom weight fn (multigraph-correct promoted matching) --- + incentivized_weight = _make_incentivized_weight( + G, weight_attr, HUGE_MULTIPLIER, + normal_hop_cost=NORMAL_HOP_COST, promoted_hop_cost=PROMOTED_HOP_COST, + promoted_set=set(promoted_edges)) # --- 2. Decompose the Problem --- u_mand, v_mand = mandatory_edge[0], mandatory_edge[1] @@ -178,41 +199,11 @@ def shortest_path_with_promoted_edges(G: Any, source: Any, target: Any, promoted NORMAL_HOP_COST = 100 PROMOTED_HOP_COST = 33 - # 1. Optimize Lookup: Convert list to set for O(1) checking - # We handle both (u,v) and (u,v,key) formats - promoted_set = set(promoted_edges) - - # 2. Define the Custom Weight Function - def incentivized_weight(u, v, attr): - # --- A. Physical Cost --- - real_weight = attr.get(weight_attr, 0) - if real_weight < 0: - raise ValueError("Negative weights not allowed.") - - # --- B. Preference Cost --- - # Check if this edge is promoted - # (MultiGraph keys are not passed to this function in all NX versions, - # but 'attr' usually contains them or we check connectivity) - - is_promoted = False - - # Check 1: Is the specific (u, v) pair in the set? - if (u, v) in promoted_set: - is_promoted = True - # Check 2: If MultiGraph, is the specific key in the set? - elif G.is_multigraph(): - # In some NX versions, 'attr' might not have the key directly if iterated strictly. - # But usually we can infer or pass keys. - # If your promoted_edges has keys (u, v, k), we need to match carefully. - # For simplicity here: if (u, v) is promoted, we treat all parallel lines as promoted - # UNLESS you specifically require key matching. - pass - - # Apply costs - hop_cost = PROMOTED_HOP_COST if is_promoted else NORMAL_HOP_COST - - # Formula: (Weight * HUGE) + Hop_Cost - return (real_weight * HUGE_MULTIPLIER) + hop_cost + # 1. Weight fn: multigraph-correct, matches promoted (u,v) or (u,v,key). + incentivized_weight = _make_incentivized_weight( + G, weight_attr, HUGE_MULTIPLIER, + normal_hop_cost=NORMAL_HOP_COST, promoted_hop_cost=PROMOTED_HOP_COST, + promoted_set=set(promoted_edges)) # 3. Run Dijkstra with the Custom Weight try: diff --git a/alphaDeesp/core/graphs/structured_overload_graph.py b/alphaDeesp/core/graphs/structured_overload_graph.py index 325489a9..a98493f4 100644 --- a/alphaDeesp/core/graphs/structured_overload_graph.py +++ b/alphaDeesp/core/graphs/structured_overload_graph.py @@ -3,6 +3,7 @@ """ import logging +from functools import cached_property from typing import Any, List, Optional, Tuple import networkx as nx @@ -18,12 +19,27 @@ logger = logging.getLogger(__name__) +# Optional bound on the number of *nodes* in a loop path enumerated by +# :meth:`find_loops` (rustworkx ``cutoff`` counts nodes). Enumerating all +# simple paths between every pair of candidate hubs is combinatorial and can +# hang on very large grids. The bound is **OFF by default** (``None`` == +# unbounded == the original behaviour): a too-small cutoff silently drops +# legitimate long loops — real RTE zone grids have loop paths well beyond 10 +# nodes, and an emptied ``find_loops`` then breaks downstream consumers. Pass +# an int to opt into a bound only on grids where enumeration is a problem. +DEFAULT_LOOP_PATH_CUTOFF = None + class Structured_Overload_Distribution_Graph: """ Staring from a raw overload distribution graph with color edges, this class identifies the underlying path structure in terms of constrained path, loop paths and hub nodes """ - def __init__(self, g: nx.MultiDiGraph, possible_hubs: Optional[List[Any]] = None) -> None: + def __init__( + self, + g: nx.MultiDiGraph, + possible_hubs: Optional[List[Any]] = None, + loop_path_cutoff: Optional[int] = DEFAULT_LOOP_PATH_CUTOFF, + ) -> None: """ Parameters ---------- @@ -31,25 +47,99 @@ def __init__(self, g: nx.MultiDiGraph, possible_hubs: Optional[List[Any]] = None g: :class:`nx:MultiDiGraph` a raw graph from OverflowGraph + possible_hubs: list, optional + a pre-computed subset of hub candidates (e.g. when consolidating a + previously built overflow graph) + + loop_path_cutoff: int, optional + optional maximum number of *nodes* in a loop path enumerated by + :meth:`find_loops`. Defaults to :data:`DEFAULT_LOOP_PATH_CUTOFF` + (``None`` == unbounded == the original behaviour). Pass an int only + to bound enumeration on grids where it would otherwise hang; too + small a value silently drops legitimate long loops. + """ - self.g_init=g - self.g_without_pos_edges = delete_color_edges(self.g_init, "coral") #graph without loop path that have positive/red-coloured weight edges - self.g_only_blue_components = delete_color_edges(self.g_without_pos_edges, "gray") - self.g_only_blue_components = delete_color_edges(self.g_only_blue_components, "dimgray")#also delete those edges of non reconnectable lines that we would want to visualize but is not an operational path in the structured path - - self.g_without_constrained_edge = delete_color_edges(self.g_init, "black") - self.g_without_gray_and_c_edge = delete_color_edges(self.g_without_constrained_edge, "gray") - self.g_without_gray_and_c_edge = delete_color_edges(self.g_without_gray_and_c_edge, "dimgray") - self.g_only_red_components = delete_color_edges(self.g_without_gray_and_c_edge, "blue")#graph with only loop path that have positive/red-coloured weight edges - - self.constrained_path= self.find_constrained_path() #constrained path that contains the constrained edges and their connected component of blue edges - self.type=""# - if possible_hubs is not None:#in case we already have a subset of candidates, for instance when we already built a first Overload graph and are consolidating it - self.hubs=possible_hubs - else: - self.hubs=[] - self.red_loops = self.find_loops() #parallel path to the constrained path on which flow can be rerouted - self.hubs = self.find_hubs() #specific nodes at substations connecting loop paths to constrained path. This is where flow can be most easily rerouted + self.g_init = g + self.loop_path_cutoff = loop_path_cutoff + # Snapshot the graph at construction. The colour-filtered views are lazy + # (below) but MUST reflect the graph *as it was when this object was + # built* — the historical eager ``__init__`` copied every view at + # construction, so a later mutation of the caller's graph (e.g. + # ``consolidate_graph`` removing the ignored lines from the shared + # ``OverFlowGraph.g`` before it re-reads this object) did not leak into + # the views. Deferring the copies to first access would read the mutated + # graph instead; freezing a single snapshot here preserves the exact + # snapshot semantics while keeping the views lazy. ``g_init`` itself stays + # a live reference — ``get_constrained_edges_nodes`` reads names off it and + # historically saw the live graph, so that alias is deliberately kept. + self._g_snapshot = g.copy() + # Caller-supplied hub seeds influence *loop enumeration* only (see the + # ``red_loops`` property); the *detected* hubs are exposed via ``hubs`` / + # ``get_hubs``. Historically this seed was stored in ``self.hubs`` until + # ``find_hubs`` overwrote it — capturing it separately makes the lazy + # properties order-independent while preserving that exact behaviour. + self._possible_hubs = list(possible_hubs) if possible_hubs is not None else [] + self.type = "" + + # The colour-filtered views, red loops and hubs are lazy *cached + # properties* (computed once on first access, then memoised): a consumer + # that needs only a subset — or that never consolidates — does not pay + # for the rest, and the consolidation loop's repeated rebuilds compute + # only what each iteration touches. The constrained path is validated + # eagerly: it is cheap and preserves the construction-time "is this a + # valid overflow graph" check that callers relied on. + self.constrained_path = self.find_constrained_path() + + # ------------------------------------------------------------------ + # Lazy colour-filtered views (pure functions of the construction-time + # snapshot; cached). Each removes the *union* of the listed colours in a + # single graph copy. + # ------------------------------------------------------------------ + + @cached_property + def g_without_pos_edges(self) -> nx.MultiDiGraph: + """Overflow graph without coral (positive / loop) edges.""" + return delete_color_edges(self._g_snapshot, "coral") + + @cached_property + def g_only_blue_components(self) -> nx.MultiDiGraph: + """Only the blue constrained-path components (drops coral / gray / dimgray). + + ``dimgray`` (non-reconnectable null-flow lines that we still visualise) + is removed too: they are not an operational path in the structured path. + """ + return delete_color_edges(self._g_snapshot, ("coral", "gray", "dimgray")) + + @cached_property + def g_without_constrained_edge(self) -> nx.MultiDiGraph: + """Overflow graph without the black (overloaded) edges.""" + return delete_color_edges(self._g_snapshot, "black") + + @cached_property + def g_without_gray_and_c_edge(self) -> nx.MultiDiGraph: + """Only the coloured redispatch edges (drops black / gray / dimgray).""" + return delete_color_edges(self._g_snapshot, ("black", "gray", "dimgray")) + + @cached_property + def g_only_red_components(self) -> nx.MultiDiGraph: + """Only the coral (positive / loop) redispatch edges.""" + return delete_color_edges(self._g_snapshot, ("black", "gray", "dimgray", "blue")) + + @cached_property + def red_loops(self) -> pd.DataFrame: + """Parallel (loop) paths, enumerated with the caller-supplied seed hubs. + + Mirrors the historical eager ``self.red_loops = find_loops()`` which ran + *before* hub detection — so it uses ``_possible_hubs`` (the seed), not the + detected ``hubs``. The public :meth:`find_loops` re-enumerates with the + detected hubs when called after construction (as consolidation does). + """ + return self._find_loops(self._possible_hubs) + + @cached_property + def hubs(self) -> List[Any]: + """Detected hub nodes (memoised).""" + return self.find_hubs() def get_amont_blue_edges(self, g: nx.MultiDiGraph, node: Any) -> List[Any]: """ @@ -119,11 +209,9 @@ def find_hubs(self) -> List[Any]: g = self.g_without_constrained_edge hubs = [] - if self.constrained_path is not None: - logger.debug("In get_hubs(): constrained_path = %s", self.constrained_path) - else: - e_amont, constrained_edge, e_aval = self.get_constrained_path() - self.constrained_path = ConstrainedPath(e_amont, constrained_edge, e_aval) + # ``constrained_path`` is validated eagerly in ``__init__``, so it is + # always available here. + logger.debug("In find_hubs(): constrained_path = %s", self.constrained_path) # for nodes in aval, if node has RED inputs (ie incoming flows) then it is a hub for node in self.constrained_path.n_aval(): @@ -148,6 +236,15 @@ def get_hubs(self) -> List[Any]: return self.hubs def find_loops(self) -> pd.DataFrame: + """Enumerate loop paths using the *currently detected* hubs (``self.hubs``). + + The cached :attr:`red_loops` uses the caller-supplied seed hubs instead + (see its docstring); consolidation re-enumerates through this method + after the hubs have been detected. + """ + return self._find_loops(self.hubs) + + def _find_loops(self, hubs: List[Any]) -> pd.DataFrame: """This function returns all parallel paths. After discussing with Antoine, start with the most "en Aval" node, and walk in reverse for loops and parallel path returns a dict with all data @@ -167,8 +264,8 @@ def find_loops(self) -> pd.DataFrame: # print("==================== In function get_loops ====================") g = self.g_only_red_components c_path_n = self.constrained_path.full_n_constrained_path() - if len(self.hubs)!=0:#already some insights of possible hubs - c_path_n=self.hubs + if len(hubs)!=0:#already some insights of possible hubs + c_path_n=hubs # --- 1. PRE-PROCESSING (Rustworkx) --- # Convert NetworkX graph to Rustworkx for 50x speedup @@ -192,9 +289,12 @@ def find_loops(self) -> pd.DataFrame: s_idx = node_map[src_name] t_idx = node_map[tgt_name] - # Rustworkx: Find all simple paths (FAST) - # cutoff=10 is crucial to prevent hanging on large grids - paths_indices = rx.all_simple_paths(rx_graph, s_idx, t_idx, min_depth=1)#, cutoff=10) + # Rustworkx: Find all simple paths (FAST). + # ``cutoff`` (max nodes per path) is crucial to prevent + # hanging on large grids; see ``loop_path_cutoff``. rustworkx + # treats ``cutoff=None`` as "no bound". + paths_indices = rx.all_simple_paths( + rx_graph, s_idx, t_idx, min_depth=1, cutoff=self.loop_path_cutoff) # Convert Indices -> Names # We extend the main list directly @@ -232,12 +332,11 @@ def get_loops(self) -> pd.DataFrame: return self.red_loops def find_constrained_path(self) -> "ConstrainedPath": - """Find and return the constrained path + """Find and return the constrained path. - Returns - ---------- - - res: :class:`ConstrainedPath` + Returns + ------- + ConstrainedPath a constrained path object """ constrained_edge = None @@ -254,16 +353,14 @@ def get_constrained_path(self) -> "ConstrainedPath": return self.constrained_path def get_constrained_edges_nodes(self) -> Tuple[List[Any], List[Any], List[Any], List[Any]]: - """ - This function identifies the constrained path within the distribution graph. - - Parameters: - g_distribution_graph (Structured_Overload_Distribution_Graph): The structured overload distribution graph. + """Identify the constrained path within the distribution graph. - Returns: - tuple: A tuple containing two lists: - - edges_constrained_path: List of edges that are part of the constrained path. - - nodes_constrained_path: List of nodes that are part of the constrained path. + Returns + ------- + tuple + ``(edges_constrained_path, nodes_constrained_path, other_blue_edges, + other_blue_nodes)`` — the line names and nodes on the constrained + path, plus the blue edges/nodes that are *not* on it. """ constrained_path_object = self.constrained_path#self.find_constrained_path() nodes_constrained_path = constrained_path_object.full_n_constrained_path() @@ -291,23 +388,32 @@ def get_constrained_edges_nodes(self) -> Tuple[List[Any], List[Any], List[Any], return list(set(edges_constrained_path)), nodes_constrained_path, other_blue_edges, other_blue_nodes def get_dispatch_edges_nodes(self, only_loop_paths: bool = True) -> Tuple[List[Any], List[Any]]: - """ - This function identifies the dispatch path within the distribution graph. + """Identify the dispatch (loop) path within the distribution graph. - Parameters: - g_distribution_graph (Structured_Overload_Distribution_Graph): The structured overload distribution graph. + Parameters + ---------- + only_loop_paths : bool + when True (default) restrict to nodes that lie on a detected red-loop + path; otherwise use every node of the red-component graph. - Returns: - tuple: A tuple containing two lists: - - lines_redispatch: List of lines that are part of the dispatch path. - - list_nodes_dispatch_path: List of nodes that are part of the dispatch path. + Returns + ------- + tuple + ``(lines_redispatch, list_nodes_dispatch_path)`` — the line names and + nodes that make up the dispatch path. """ lines_redispatch=[] list_nodes_dispatch_path=[] g_red = self.g_only_red_components if only_loop_paths: - list_nodes_dispatch_path = list(set(self.red_loops.Path.sum()))#list(set(self.find_loops()["Path"].sum())) + # ``Series.sum()`` on an empty ``Path`` column returns the scalar + # 0 (not an empty list), so guard the no-loop case explicitly to + # avoid ``set(0)`` -> "int object is not iterable". ``sum(paths, + # [])`` concatenates the per-loop node lists and yields [] when + # there are no loops. + paths = self.red_loops.Path + list_nodes_dispatch_path = list(set(sum(paths, []))) if len(paths) else [] else: list_nodes_dispatch_path=list(g_red.nodes) diff --git a/alphaDeesp/core/graphsAndPaths.py b/alphaDeesp/core/graphsAndPaths.py index d9be4748..92001d3e 100755 --- a/alphaDeesp/core/graphsAndPaths.py +++ b/alphaDeesp/core/graphsAndPaths.py @@ -11,14 +11,23 @@ """ from alphaDeesp.core.graphs import ( # noqa: F401 + EDGE_ROLE_INSIGNIFICANT, + EDGE_ROLE_NEGATIVE, + EDGE_ROLE_NULL_NON_RECONNECTABLE, + EDGE_ROLE_OVERLOAD, + EDGE_ROLE_POSITIVE, + EDGE_ROLE_UNKNOWN, ConstrainedPath, OverFlowGraph, + OverflowGraphRenderer, PowerFlowGraph, Structured_Overload_Distribution_Graph, add_double_edges_null_redispatch, all_simple_edge_paths_multi, + base_color_of, default_voltage_colors, delete_color_edges, + edge_role_of, find_multidigraph_edges_by_name, from_edges_get_nodes, incident_edges, @@ -33,8 +42,17 @@ "default_voltage_colors", "PowerFlowGraph", "OverFlowGraph", + "OverflowGraphRenderer", "ConstrainedPath", "Structured_Overload_Distribution_Graph", + "edge_role_of", + "base_color_of", + "EDGE_ROLE_OVERLOAD", + "EDGE_ROLE_NEGATIVE", + "EDGE_ROLE_POSITIVE", + "EDGE_ROLE_INSIGNIFICANT", + "EDGE_ROLE_NULL_NON_RECONNECTABLE", + "EDGE_ROLE_UNKNOWN", "from_edges_get_nodes", "delete_color_edges", "nodepath_to_edgepath", diff --git a/alphaDeesp/core/interactive_html.py b/alphaDeesp/core/interactive_html.py deleted file mode 100644 index 8ca0d1c0..00000000 --- a/alphaDeesp/core/interactive_html.py +++ /dev/null @@ -1,976 +0,0 @@ -# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) -# See AUTHORS.txt -# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. -# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, -# you can obtain one at http://mozilla.org/MPL/2.0/. -# SPDX-License-Identifier: MPL-2.0 -# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids - -"""Build an interactive HTML viewer around a Graphviz-rendered SVG. - -The viewer keeps the exact dot-computed positions and bezier-curved edges -(it reuses the SVG produced by Graphviz verbatim) and layers JS-driven -interactions on top: pan/zoom, hover tooltip, click-to-highlight -neighborhood, search, layer toggles by edge color/style. - -The companion JSON (``dot -Tjson`` flavour) is parsed in Python so the -embedded data structure exposed to the browser is a flat node/edge model -with a pre-computed adjacency map — interactions stay O(1) at runtime. -""" - -from __future__ import annotations - -import html as html_mod -import json -import logging -import re -from typing import Any, Dict, List, Optional, Tuple - -import pydot - -logger = logging.getLogger(__name__) - -# Section names rendered as

headers in the sidebar layer list. -# Layers carry their section in the model so the JS just groups by it. -_SECTION_STRUCTURAL = "Structural Paths" -_SECTION_PROPERTIES = "Individual entities properties" -_SECTION_FLOWS = "Flow redispatch values" - -# Edge color → human-readable layer label. Restricted to the three flow -# polarities (positive / negative / null). The historical "black" / -# "gray" / "darkred" buckets are dropped because they are redundant -# with the explicit semantic flags (is_overload / is_monitored) or -# carry no operational meaning on their own. -_LAYER_LABELS: Dict[str, str] = { - "coral": "Positive", - "blue": "Negative", - "dimgray": "Null", -} - -# Edge style → layer label (orthogonal to color). -_STYLE_LAYERS: Dict[str, str] = { - "dotted": "Non-reconnectable", - "dashed": "Reconnectable", - "tapered": "Swapped flow", -} - -# Source-of-truth attribute layers — values produced upstream by -# alphaDeesp / expert_op4grid_recommender as explicit boolean flags on -# nodes and/or edges. The viewer scans for them and exposes a layer -# toggle for each. Defining them here (rather than guessing from edge -# colours / shapes) keeps the layer list semantically stable when the -# visual palette evolves. -# -# Each entry: -# key — `data-attr-*` flag scanned on node and edge groups -# label — human-readable sidebar label -# swatch — special-case identifier consumed by the JS -# template to render an inline SVG glyph (no colour -# chip — these layers cut across the colour palette) -# scope — "node", "edge", or "both" -_SEMANTIC_LAYERS: List[Dict[str, str]] = [ - {"key": "on_constrained_path", "label": "Constrained path", "swatch": "constrained-path", "scope": "both"}, - {"key": "in_red_loop", "label": "Red-loop paths", "swatch": "red-loop", "scope": "both"}, - {"key": "is_overload", "label": "Overloads", "swatch": "overload", "scope": "edge"}, - {"key": "is_monitored", "label": "Low margin lines", "swatch": "monitored", "scope": "edge"}, - # Operator-supplied extras (ExpertAgent's `additionalLinesToCut`): - # cut in the analysis like overloads but rendered with their - # natural flow colour and excluded from the Overloads / - # Low margin lines layers. Surfaced as a dedicated layer so the - # operator can still see how their choice materialised. - {"key": "is_extra_cut", "label": "Extra lines to prevent flow increase", "swatch": "extra-cut", "scope": "edge"}, - {"key": "is_hub", "label": "Hubs", "swatch": "diamond", "scope": "node"}, -] - -# Threshold below which a node's ``value`` (prod − load, in MW) is -# treated as "no real prod/load here". Build-time conventions in the -# upstream simulators tag every node with ``prod_or_load="load"`` and -# ``value="0.0"`` even when no consumption exists, so a strict -# ``prod_or_load == "load"`` test would flood the layer with empty -# nodes. The 1 MW floor matches operator practice. -_PROD_LOAD_VALUE_FLOOR_MW = 1.0 - -# Per-kind config for the value-based node layers. Matched against the -# ``prod_or_load`` attribute set by ``build_nodes`` upstream -# (alphaDeesp/core/graphs/power_flow_graph.py and the simulator-specific -# build_nodes_v2 helpers). Each entry produces a single layer in the -# "Individual entities properties" section, populated only with the -# nodes whose absolute ``value`` clears ``_PROD_LOAD_VALUE_FLOOR_MW``. -_VALUE_NODE_LAYERS: List[Dict[str, str]] = [ - {"key": "prod", "label": "Production nodes", "swatch": "prod-node"}, - {"key": "load", "label": "Consumption nodes", "swatch": "load-node"}, -] - -# Per-layer-key section assignment. The JS renders one ``

`` per -# section in the order the sections are first encountered. -_LAYER_SECTIONS: Dict[str, str] = { - # Structural paths — multi-edge structures. - "semantic:on_constrained_path": _SECTION_STRUCTURAL, - "semantic:in_red_loop": _SECTION_STRUCTURAL, - # Individual entities properties — per-edge / per-node flags. - "semantic:is_overload": _SECTION_PROPERTIES, - "semantic:is_monitored": _SECTION_PROPERTIES, - "semantic:is_extra_cut": _SECTION_PROPERTIES, - "semantic:is_hub": _SECTION_PROPERTIES, - "style:dashed": _SECTION_PROPERTIES, - "style:dotted": _SECTION_PROPERTIES, - "style:tapered": _SECTION_PROPERTIES, - # Value-based node layers — see _VALUE_NODE_LAYERS / build_nodes. - "node:prod": _SECTION_PROPERTIES, - "node:load": _SECTION_PROPERTIES, - # Flow polarity buckets. - "color:coral": _SECTION_FLOWS, - "color:blue": _SECTION_FLOWS, - "color:dimgray": _SECTION_FLOWS, -} - -# Render order: sections appear top-to-bottom in this order; layers -# within a section appear in the order the model emits them. -_SECTION_ORDER: List[str] = [ - _SECTION_STRUCTURAL, - _SECTION_PROPERTIES, - _SECTION_FLOWS, -] - - -def _decode_title(text: str) -> str: - """Graphviz HTML-escapes node/edge titles in SVG (``A->B``).""" - return html_mod.unescape(text or "") - - -def _split_edge_title(title: str) -> Tuple[str, str]: - """Edge titles are ``"->["`` (digraph) or ``"--"``.""" - title = _decode_title(title) - for sep in ("->", "--"): - if sep in title: - src, dst = title.split(sep, 1) - return src.strip(), dst.strip() - return title, "" - - -def _color_to_layer_key(color: str) -> Optional[str]: - """Map a (possibly compound or hex) color to a known layer key.""" - if not color: - return None - base = color.split(":", 1)[0].strip().strip('"').lower() - if base in _LAYER_LABELS: - return base - return None - - -def _normalize_attrs(raw: Dict[str, Any]) -> Dict[str, Any]: - """Strip Graphviz-internal _draw_/_ldraw_ keys and quoted strings.""" - out: Dict[str, Any] = {} - for k, v in raw.items(): - if k.startswith("_") or k in ("nodes", "edges", "objects", "subgraphs"): - continue - if isinstance(v, str): - v = v.strip().strip('"') - out[k] = v - return out - - -def _model_from_dot_json(dot_json: bytes) -> Dict[str, Any]: - """Parse ``dot -Tjson`` into a flat node/edge model + adjacency map.""" - data = json.loads(dot_json.decode("utf-8")) - objects: List[Dict[str, Any]] = data.get("objects", []) - raw_edges: List[Dict[str, Any]] = data.get("edges", []) - - nodes: List[Dict[str, Any]] = [] - name_by_gvid: Dict[int, str] = {} - for i, obj in enumerate(objects): - if "nodes" in obj or "subgraphs" in obj: - # cluster/subgraph entry — skip in v1 - continue - name = obj.get("name", f"node{i}") - name_by_gvid[obj.get("_gvid", i)] = name - nodes.append({ - "name": name, - "attrs": _normalize_attrs(obj), - }) - - edges: List[Dict[str, Any]] = [] - adjacency: Dict[str, List[Dict[str, str]]] = {n["name"]: [] for n in nodes} - for j, edge in enumerate(raw_edges): - src = name_by_gvid.get(edge.get("tail")) - dst = name_by_gvid.get(edge.get("head")) - if src is None or dst is None: - continue - attrs = _normalize_attrs(edge) - edges.append({ - "id": f"edge{j + 1}", # matches Graphviz SVG id naming - "source": src, - "target": dst, - "attrs": attrs, - }) - adjacency.setdefault(src, []).append({"node": dst, "edge": f"edge{j + 1}"}) - adjacency.setdefault(dst, []).append({"node": src, "edge": f"edge{j + 1}"}) - - layers = _build_layer_index(edges, nodes) - return { - "nodes": nodes, - "edges": edges, - "adjacency": adjacency, - "layers": layers, - } - - -def _is_truthy_flag(value: Any) -> bool: - """Check whether a graph attribute represents a True boolean flag. - - Boolean attributes round-trip through pydot/graphviz/dot-json as - string ``"True"``. We accept the native Python ``True`` for - in-process callers and the string form for the JSON path. - """ - if value is True: - return True - if isinstance(value, str): - return value.strip().lower() == "true" - return False - - -def _build_layer_index( - edges: List[Dict[str, Any]], - nodes: List[Dict[str, Any]] | None = None, -) -> List[Dict[str, Any]]: - """Group edges & nodes by colour / style / semantic flag so the UI - can offer toggles. Each layer carries both ``nodes`` and ``edges`` - id lists (either may be empty). - """ - by_color: Dict[str, List[str]] = {} - by_style: Dict[str, List[str]] = {} - edge_id_lookup = {e["id"]: e for e in edges} - for e in edges: - color_key = _color_to_layer_key(e["attrs"].get("color", "")) - if color_key: - by_color.setdefault(color_key, []).append(e["id"]) - style = (e["attrs"].get("style") or "").lower() - if style in _STYLE_LAYERS: - by_style.setdefault(style, []).append(e["id"]) - - # Semantic flags scanned on both nodes and edges. Only emit a layer - # entry if at least one element carries the flag — otherwise the - # checkbox would be useless and noise. - semantic_buckets: Dict[str, Dict[str, List[str]]] = { - cfg["key"]: {"nodes": [], "edges": []} for cfg in _SEMANTIC_LAYERS - } - if nodes: - for n in nodes: - for cfg in _SEMANTIC_LAYERS: - if cfg["scope"] in ("node", "both") and _is_truthy_flag( - n["attrs"].get(cfg["key"]) - ): - semantic_buckets[cfg["key"]]["nodes"].append(n["name"]) - for e in edges: - for cfg in _SEMANTIC_LAYERS: - if cfg["scope"] in ("edge", "both") and _is_truthy_flag( - e["attrs"].get(cfg["key"]) - ): - semantic_buckets[cfg["key"]]["edges"].append(e["id"]) - - # For each colour / style layer, the endpoint nodes of every - # claimed edge are also added to the layer so toggling, e.g., - # "Positive overflow" alone keeps the substations the coral edges - # connect visible (the operator can still read the topology around - # the highlighted edges instead of seeing them float in dimmed - # space). We dedupe while preserving first-seen order. - edge_id_to_endpoints: Dict[str, Tuple[str, str]] = { - e["id"]: (e["source"], e["target"]) for e in edges - } - - def _endpoint_nodes(edge_ids: List[str]) -> List[str]: - seen: Dict[str, None] = {} - for eid in edge_ids: - ends = edge_id_to_endpoints.get(eid) - if not ends: - continue - for n in ends: - if n not in seen: - seen[n] = None - return list(seen.keys()) - - # Edge-only semantic layers (Overloads, Low margin lines) carry - # their edges' endpoints too — same UX rationale as colour/style - # layers: when the operator ticks "Overloads" alone the affected - # substations stay visible. - _EDGE_ONLY_SEMANTIC_KEYS = { - cfg["key"] for cfg in _SEMANTIC_LAYERS if cfg["scope"] == "edge" - } - - def _merge_dedup(base: List[str], extra: List[str]) -> List[str]: - seen: Dict[str, None] = {n: None for n in base} - for n in extra: - if n not in seen: - seen[n] = None - return list(seen.keys()) - - raw_layers: List[Dict[str, Any]] = [] - for key, ids in by_color.items(): - raw_layers.append({ - "key": f"color:{key}", - "label": _LAYER_LABELS[key], - "swatch": key, - "nodes": _endpoint_nodes(ids), - "edges": ids, - }) - for key, ids in by_style.items(): - raw_layers.append({ - "key": f"style:{key}", - "label": _STYLE_LAYERS[key], - "swatch": "", - "nodes": _endpoint_nodes(ids), - "edges": ids, - }) - for cfg in _SEMANTIC_LAYERS: - bucket = semantic_buckets[cfg["key"]] - if not bucket["nodes"] and not bucket["edges"]: - continue - nodes_for_layer = bucket["nodes"] - if cfg["key"] in _EDGE_ONLY_SEMANTIC_KEYS: - nodes_for_layer = _merge_dedup( - nodes_for_layer, _endpoint_nodes(bucket["edges"]) - ) - raw_layers.append({ - "key": f"semantic:{cfg['key']}", - "label": cfg["label"], - "swatch": cfg["swatch"], - "nodes": nodes_for_layer, - "edges": bucket["edges"], - }) - - # Value-based node layers (Production / Consumption). Built from - # the ``prod_or_load`` attribute upstream tagged on every node by - # ``build_nodes`` — see _VALUE_NODE_LAYERS. The white-coloured - # zero-balance nodes carry ``prod_or_load="load"`` AND - # ``value="0.0"`` upstream by convention; the 1 MW floor filters - # them out so the operator's "Consumption nodes" toggle doesn't - # also tag every passive substation. - if nodes: - value_buckets: Dict[str, List[str]] = { - cfg["key"]: [] for cfg in _VALUE_NODE_LAYERS - } - for n in nodes: - kind = n["attrs"].get("prod_or_load") - if kind not in value_buckets: - continue - try: - magnitude = abs(float(n["attrs"].get("value", "0"))) - except (TypeError, ValueError): - continue - if magnitude < _PROD_LOAD_VALUE_FLOOR_MW: - continue - value_buckets[kind].append(n["name"]) - for cfg in _VALUE_NODE_LAYERS: - ids = value_buckets[cfg["key"]] - if not ids: - continue - raw_layers.append({ - "key": f"node:{cfg['key']}", - "label": cfg["label"], - "swatch": cfg["swatch"], - "nodes": ids, - "edges": [], - }) - - # Drop layers without a section assignment (e.g. ``color:black``, - # ``color:gray``, ``color:darkred`` — historically redundant - # buckets). Then group by section in the canonical order so the - # JS can render them with section headers. - sectioned: Dict[str, List[Dict[str, Any]]] = {s: [] for s in _SECTION_ORDER} - for layer in raw_layers: - section = _LAYER_SECTIONS.get(layer["key"]) - if section is None: - continue - layer["section"] = section - sectioned.setdefault(section, []).append(layer) - - layers: List[Dict[str, Any]] = [] - for section in _SECTION_ORDER: - layers.extend(sectioned.get(section, [])) - # Silence unused-var warning; lookup retained for future hover xref. - del edge_id_lookup - return layers - - -def _inject_svg_data_attrs(svg_bytes: bytes, model: Dict[str, Any]) -> str: - """Annotate Graphviz SVG nodes/edges with stable data-* attributes. - - Graphviz emits ``NAME``; we - rely on the title to look up our model entries and append data-* - attributes (which the JS uses for selectors and tooltips). - """ - svg = svg_bytes.decode("utf-8") - edge_by_id = {e["id"]: e for e in model["edges"]} - node_by_name = {n["name"]: n for n in model["nodes"]} - - def _attrs_to_data(prefix: str, attrs: Dict[str, Any]) -> str: - out: List[str] = [] - for k, v in attrs.items(): - safe_v = html_mod.escape(str(v), quote=True) - out.append(f' data-{prefix}-{k}="{safe_v}"') - return "".join(out) - - def _node_repl(match: re.Match) -> str: - gid = match.group(1) - title = match.group(2) - name = _decode_title(title) - node = node_by_name.get(name) - if not node: - return match.group(0) - data = _attrs_to_data("attr", node["attrs"]) - return ( - f'{title}' - ) - - def _edge_repl(match: re.Match) -> str: - gid = match.group(1) - title = match.group(2) - edge = edge_by_id.get(gid) - if not edge: - return match.group(0) - src, dst = edge["source"], edge["target"] - layers: List[str] = [] - color_key = _color_to_layer_key(edge["attrs"].get("color", "")) - if color_key: - layers.append(f"color:{color_key}") - style = (edge["attrs"].get("style") or "").lower() - if style in _STYLE_LAYERS: - layers.append(f"style:{style}") - data = _attrs_to_data("attr", edge["attrs"]) - layer_attr = f' data-layers="{html_mod.escape(" ".join(layers), quote=True)}"' if layers else "" - return ( - f'{title}' - ) - - svg = re.sub( - r'\s*([^<]*)', - _node_repl, - svg, - ) - svg = re.sub( - r'\s*([^<]*)', - _edge_repl, - svg, - ) - return svg - - -# Self-contained HTML template — keeps the SVG inline so the file can be -# opened directly from disk (no web server, no CDN). The JS is a tiny -# ~150-line bundle: pan/zoom, hover tooltip, click-highlight, search, -# layer toggles. All state is driven by CSS classes for cheap toggling. -_HTML_TEMPLATE = """ - - - -__TITLE__ - - - -
- -
-
- - - -
- __SVG__ -
-
-
- - - -""" - - -def _align_edge_ids_with_svg(svg_bytes: bytes, model: Dict[str, Any]) -> Dict[str, Any]: - """Re-key edges in ``model`` so their ``id`` field matches the SVG's - ```` for the SAME (src, dst) endpoints. - - Background - ---------- - Graphviz emits edge IDs ``edgeN`` in **two independent orderings** for - the SVG and the JSON outputs of the same graph. ``_model_from_dot_json`` - assigns IDs by JSON-edge index but the SVG element with the same - ``edgeN`` id often refers to a different edge (different (src, dst) - pair). The downstream JS dim layer queries SVG elements **by id** — - so a mismatch makes the wrong edges dim/highlight when a layer - toggle is flipped (this is exactly the user-reported confusion - SSV.OP7→GROSNP7 ↔ SSV.OP7→CREYSP7 / CHALOP6→CPVANP6 ↔ - CHALOP6→CHALOP3 in the small-grid scenario). - - Fix - --- - Walk the SVG, parse each edge's ```` to extract its true - (src, dst), and greedily pair it with a JSON-side edge of matching - endpoints. Each JSON edge is consumed at most once (parallel edges - are paired in their relative order, which is stable across SVG and - JSON). The model's edge IDs are updated in place; adjacency and - layer membership lists are remapped through the same dict. - - Returns the updated model. - """ - svg = svg_bytes.decode("utf-8") - # Walk SVG edges in document order: ``<g id="edgeN" class="edge"> - # <title>SRC->DST``. Graphviz HTML-escapes the title. - pattern = re.compile( - r'\s*([^<]*)' - ) - svg_edges_in_order: List[Tuple[str, str, str]] = [] - for m in pattern.finditer(svg): - gid = m.group(1) - src, dst = _split_edge_title(m.group(2)) - svg_edges_in_order.append((gid, src, dst)) - - # Build a per-(src, dst) FIFO of JSON edges keeping their original order. - json_edges = model["edges"] - by_pair: Dict[Tuple[str, str], List[int]] = {} - for i, e in enumerate(json_edges): - by_pair.setdefault((e["source"], e["target"]), []).append(i) - - # Greedily match each SVG edge to the next un-consumed JSON edge of the - # same endpoints. The remap dict translates "old (JSON-order) id" → "new - # (SVG-order) id". - remap: Dict[str, str] = {} - for svg_id, s, t in svg_edges_in_order: - candidates = by_pair.get((s, t)) or by_pair.get((t, s)) - if not candidates: - continue - json_idx = candidates.pop(0) - old_id = json_edges[json_idx]["id"] - if old_id == svg_id: - continue - remap[old_id] = svg_id - - if not remap: - return model - - # Apply the remap. ``remap`` may contain swaps (a→b and b→a). To avoid - # collisions we materialise the new IDs through a fresh dict in two - # passes: first relabel each JSON edge to its SVG-aligned id, then walk - # adjacency / layers to substitute the references. - edge_id_lookup = {e["id"]: e for e in json_edges} - for old_id, new_id in remap.items(): - edge = edge_id_lookup[old_id] - edge["id"] = new_id - - # Adjacency entries reference edge ids by string — apply the same - # substitution there. - for entries in model.get("adjacency", {}).values(): - for entry in entries: - if entry.get("edge") in remap: - entry["edge"] = remap[entry["edge"]] - - # Layer membership lists use the same string ids. - for layer in model.get("layers", []): - layer["edges"] = [remap.get(eid, eid) for eid in layer.get("edges", [])] - - return model - - -def build_interactive_html( - pydot_graph: pydot.Graph, - prog: Any = "dot", - title: str = "ExpertOp4Grid — interactive overflow graph", -) -> str: - """Render ``pydot_graph`` to interactive HTML. - - Returns the HTML string. Caller decides where to write it. - """ - svg_bytes = pydot_graph.create(prog=prog, format="svg") - json_bytes = pydot_graph.create(prog=prog, format="json") - model = _model_from_dot_json(json_bytes) - # Align JSON edge ids with the SVG element ids — graphviz emits the - # two orderings independently and the downstream JS toggles SVG - # elements by id, so a mismatch silently dims the wrong edges. - model = _align_edge_ids_with_svg(svg_bytes, model) - annotated_svg = _inject_svg_data_attrs(svg_bytes, model) - return ( - _HTML_TEMPLATE - .replace("__TITLE__", html_mod.escape(title)) - .replace("__SVG__", annotated_svg) - .replace("__MODEL_JSON__", json.dumps(model)) - ) diff --git a/alphaDeesp/core/interactive_html/__init__.py b/alphaDeesp/core/interactive_html/__init__.py new file mode 100644 index 00000000..6c5373aa --- /dev/null +++ b/alphaDeesp/core/interactive_html/__init__.py @@ -0,0 +1,56 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Build an interactive HTML viewer around a Graphviz-rendered SVG. + +The viewer keeps the exact dot-computed positions and bezier-curved edges +(it reuses the SVG produced by Graphviz verbatim) and layers JS-driven +interactions on top: pan/zoom, hover tooltip, click-to-highlight +neighborhood, search, layer toggles by edge color/style. + +The companion JSON (``dot -Tjson`` flavour) is parsed in Python so the +embedded data structure exposed to the browser is a flat node/edge model +with a pre-computed adjacency map — interactions stay O(1) at runtime. + +This package was split out of the former single ``interactive_html.py`` module; +the CSS/JS/HTML skeleton now live as externalised assets under ``assets/``. +The public surface (and the internal helpers referenced by tests) is +re-exported here so ``from alphaDeesp.core.interactive_html import ...`` +keeps working unchanged.""" + +from alphaDeesp.core.interactive_html.helpers import ( + _color_to_layer_key, + _decode_title, + _is_truthy_flag, + _normalize_attrs, + _split_edge_title, +) +from alphaDeesp.core.interactive_html.layers import _build_layer_index +from alphaDeesp.core.interactive_html.model import _model_from_dot_json +from alphaDeesp.core.interactive_html.svg import ( + _align_edge_ids_with_svg, + _inject_svg_data_attrs, +) +from alphaDeesp.core.interactive_html.render import build_interactive_html + +# The public entry point plus the internal helpers that were importable from +# the former single module (kept for backwards compatibility and referenced by +# tests). Listing them in ``__all__`` also marks them as intentional re-exports +# for static analysers. +__all__ = [ + "build_interactive_html", + "_build_layer_index", + "_color_to_layer_key", + "_decode_title", + "_is_truthy_flag", + "_normalize_attrs", + "_split_edge_title", + "_model_from_dot_json", + "_align_edge_ids_with_svg", + "_inject_svg_data_attrs", +] diff --git a/alphaDeesp/core/interactive_html/assets/template.html b/alphaDeesp/core/interactive_html/assets/template.html new file mode 100644 index 00000000..d6de0191 --- /dev/null +++ b/alphaDeesp/core/interactive_html/assets/template.html @@ -0,0 +1,43 @@ + + + + +__TITLE__ + + + +
+ +
+
+ + + +
+ __SVG__ +
+
+
+ + + diff --git a/alphaDeesp/core/interactive_html/assets/viewer.css b/alphaDeesp/core/interactive_html/assets/viewer.css new file mode 100644 index 00000000..5285ce6e --- /dev/null +++ b/alphaDeesp/core/interactive_html/assets/viewer.css @@ -0,0 +1,48 @@ + + :root { + --bg: #fafafa; --panel: #ffffff; --border: #d0d7de; + --muted: #6b7280; --accent: #0969da; --dim-opacity: 0.12; + } + * { box-sizing: border-box; } + html, body { height: 100%; margin: 0; font-family: -apple-system, "Segoe UI", Helvetica, Arial, sans-serif; background: var(--bg); color: #111; } + #app { display: flex; height: 100vh; } + #sidebar { + width: 280px; flex-shrink: 0; padding: 12px 14px; overflow-y: auto; + background: var(--panel); border-right: 1px solid var(--border); font-size: 13px; + } + #sidebar h1 { font-size: 14px; margin: 0 0 6px; } + #sidebar h2 { font-size: 12px; text-transform: uppercase; color: var(--muted); margin: 14px 0 6px; letter-spacing: 0.04em; } + #sidebar .layer-section-header { font-size: 11px; text-transform: uppercase; color: var(--muted); margin: 10px 0 4px; letter-spacing: 0.03em; font-weight: 600; border-top: 1px solid var(--border); padding-top: 6px; } + #sidebar .layer-section-header:first-of-type { border-top: none; padding-top: 0; margin-top: 4px; } + #sidebar input[type=text] { width: 100%; padding: 6px 8px; border: 1px solid var(--border); border-radius: 4px; font-size: 12px; } + #sidebar label { display: flex; align-items: center; gap: 6px; padding: 3px 0; cursor: pointer; } + #sidebar label .swatch { width: 12px; height: 12px; border-radius: 2px; border: 1px solid #ccc; flex-shrink: 0; display: inline-flex; align-items: center; justify-content: center; } + #sidebar label .swatch svg { width: 10px; height: 10px; display: block; } + #sidebar .layer-controls { display: flex; gap: 8px; font-size: 11px; margin-bottom: 4px; } + #sidebar .layer-controls button { background: transparent; border: none; padding: 2px 4px; cursor: pointer; color: var(--accent); font-size: 11px; text-decoration: underline; } + #sidebar .layer-controls button:hover { background: #eef; } + #sidebar .hint { color: var(--muted); font-size: 11px; line-height: 1.5; margin-top: 6px; } + #info { font-family: ui-monospace, "SFMono-Regular", Menlo, monospace; font-size: 11px; white-space: pre-wrap; word-break: break-word; background: #f6f8fa; border-radius: 4px; padding: 8px; min-height: 60px; } + #stage { flex: 1; position: relative; overflow: hidden; background: var(--bg); } + #stage svg { display: block; width: 100%; height: 100%; cursor: grab; } + #stage svg.dragging { cursor: grabbing; } + #stage .toolbar { position: absolute; top: 10px; right: 10px; display: flex; gap: 4px; background: var(--panel); border: 1px solid var(--border); border-radius: 4px; padding: 2px; } + #stage .toolbar button { border: none; background: transparent; padding: 4px 8px; cursor: pointer; font-size: 14px; font-weight: bold; color: #333; } + #stage .toolbar button:hover { background: #eef; } + #tooltip { position: absolute; pointer-events: none; background: rgba(20,20,20,0.92); color: #fff; padding: 6px 8px; border-radius: 4px; font-size: 11px; line-height: 1.4; max-width: 260px; z-index: 10; display: none; } + #tooltip .key { color: #9ad; } + /* Interaction states applied via classes on the root */ + .graph.has-selection .node:not(.hl):not(.selected), + .graph.has-selection .edge:not(.hl) { opacity: var(--dim-opacity); } + .graph .node.selected ellipse, + .graph .node.selected polygon, + .graph .node.selected circle { stroke-width: 5px; } + /* Unchecked layers stay rendered but recede to a near-transparent + light-grey so the user retains spatial context. Applied to both + nodes and edges. Pointer-events kept enabled so hover/click still + works on dimmed elements. */ + .graph .layer-off { opacity: 0.12; } + .graph.has-search .node:not(.match) { opacity: var(--dim-opacity); } + .graph .node.match ellipse, + .graph .node.match polygon, + .graph .node.match circle { stroke: #0969da !important; stroke-width: 4px; } diff --git a/alphaDeesp/core/interactive_html/assets/viewer.js b/alphaDeesp/core/interactive_html/assets/viewer.js new file mode 100644 index 00000000..db564c42 --- /dev/null +++ b/alphaDeesp/core/interactive_html/assets/viewer.js @@ -0,0 +1,309 @@ + +const MODEL = __MODEL_JSON__; +(function(){ + const svg = document.querySelector('#stage svg'); + const root = svg.querySelector('g.graph') || svg.querySelector('g'); + if (root && !root.classList.contains('graph')) root.classList.add('graph'); + + // ---- Pan & zoom: native viewBox manipulation (~30 lines, no deps) ---- + let vb = (svg.getAttribute('viewBox') || '').split(/\s+/).map(Number); + if (vb.length !== 4) { const r = svg.getBBox(); vb = [r.x, r.y, r.width, r.height]; } + const initial = vb.slice(); + function setVB() { svg.setAttribute('viewBox', vb.join(' ')); } + function clientToSvg(x, y) { + const rect = svg.getBoundingClientRect(); + return { x: vb[0] + (x - rect.left) / rect.width * vb[2], y: vb[1] + (y - rect.top) / rect.height * vb[3] }; + } + svg.addEventListener('wheel', (e) => { + e.preventDefault(); + const k = e.deltaY < 0 ? 0.85 : 1.18; + const p = clientToSvg(e.clientX, e.clientY); + vb[0] = p.x - (p.x - vb[0]) * k; + vb[1] = p.y - (p.y - vb[1]) * k; + vb[2] *= k; vb[3] *= k; + setVB(); + }, { passive: false }); + let dragStart = null; + svg.addEventListener('mousedown', (e) => { if (e.button !== 0) return; dragStart = { x: e.clientX, y: e.clientY, vb: vb.slice() }; svg.classList.add('dragging'); }); + window.addEventListener('mouseup', () => { dragStart = null; svg.classList.remove('dragging'); }); + window.addEventListener('mousemove', (e) => { + if (!dragStart) return; + const rect = svg.getBoundingClientRect(); + vb[0] = dragStart.vb[0] - (e.clientX - dragStart.x) / rect.width * vb[2]; + vb[1] = dragStart.vb[1] - (e.clientY - dragStart.y) / rect.height * vb[3]; + setVB(); + }); + document.getElementById('zoom-in').onclick = () => zoomBy(0.8); + document.getElementById('zoom-out').onclick = () => zoomBy(1.25); + document.getElementById('zoom-reset').onclick = () => { vb = initial.slice(); setVB(); }; + function zoomBy(k) { + const cx = vb[0] + vb[2]/2, cy = vb[1] + vb[3]/2; + vb[0] = cx - (cx - vb[0]) * k; vb[1] = cy - (cy - vb[1]) * k; + vb[2] *= k; vb[3] *= k; setVB(); + } + + // ---- Adjacency lookup, hover, click-highlight ---- + const tooltip = document.getElementById('tooltip'); + const info = document.getElementById('info'); + function fmtAttrs(prefix, el, skip) { + const out = []; + const skipSet = skip ? new Set(skip) : null; + for (const a of el.attributes) { + if (a.name.startsWith('data-' + prefix + '-')) { + const k = a.name.slice(('data-' + prefix + '-').length); + if (skipSet && skipSet.has(k)) continue; + out.push('' + k + ': ' + a.value); + } + } + return out.join('
'); + } + // Display name for a node: the readable ``label`` (e.g. a voltage-level + // name such as "Saucats 400kV") when present, else the stable node id + // (``data-name``). The id is preserved as the node identity for + // selection / adjacency / double-click resolution; ``label`` only + // changes what the operator reads. + function nodeDisplayName(el) { + if (!el) return ''; + const label = el.getAttribute('data-attr-label'); + // Graphviz stores an *unset* label as the placeholder "\N" (meaning + // "node name"); any backslash escape (\N, \G, …) is not a readable + // name, so fall back to the stable id (data-name) in that case. + if (label && label.indexOf('\\') === -1) return label; + return el.getAttribute('data-name') || ''; + } + // Tooltip / selection header for a node: readable name in bold, with the + // raw id shown underneath only when it differs from the readable name. + function nodeHeaderHtml(el) { + const disp = nodeDisplayName(el); + const id = el.getAttribute('data-name') || ''; + let head = '' + escapeHtml(disp) + ''; + if (id && id !== disp) head += '
id: ' + escapeHtml(id); + return head; + } + function showTooltip(e, html) { + tooltip.innerHTML = html; + tooltip.style.display = 'block'; + const r = svg.getBoundingClientRect(); + tooltip.style.left = (e.clientX - r.left + 12) + 'px'; + tooltip.style.top = (e.clientY - r.top + 12) + 'px'; + } + function hideTooltip() { tooltip.style.display = 'none'; } + + function clearSelection() { + root.classList.remove('has-selection'); + root.querySelectorAll('.hl, .selected').forEach(el => el.classList.remove('hl', 'selected')); + info.textContent = 'Nothing selected.'; + } + function selectNode(name) { + clearSelection(); + const node = root.querySelector('.node[data-name="' + cssEscape(name) + '"]'); + if (!node) return; + root.classList.add('has-selection'); + node.classList.add('selected', 'hl'); + const neighbours = MODEL.adjacency[name] || []; + for (const n of neighbours) { + const nb = root.querySelector('.node[data-name="' + cssEscape(n.node) + '"]'); + if (nb) nb.classList.add('hl'); + const ed = document.getElementById(n.edge); + if (ed) ed.classList.add('hl'); + } + info.innerHTML = nodeHeaderHtml(node) + '
' + fmtAttrs('attr', node, ['label']) + + '

degree: ' + neighbours.length; + } + function cssEscape(s) { return (window.CSS && CSS.escape) ? CSS.escape(s) : s.replace(/(["\\])/g, '\\$1'); } + function escapeHtml(s) { return s.replace(/[&<>]/g, c => ({'&':'&','<':'<','>':'>'}[c])); } + + root.addEventListener('mouseover', (e) => { + const g = e.target.closest('.node, .edge'); if (!g) return; + if (g.classList.contains('node')) { + showTooltip(e, nodeHeaderHtml(g) + '
' + fmtAttrs('attr', g, ['label'])); + } else { + const lbl = g.querySelector('text'); const name = g.getAttribute('data-attr-name') || ''; + showTooltip(e, '' + escapeHtml(g.getAttribute('data-source')) + ' → ' + escapeHtml(g.getAttribute('data-target')) + '' + + (name ? '
' + escapeHtml(name) : '') + + '
' + fmtAttrs('attr', g)); + } + }); + root.addEventListener('mousemove', (e) => { + if (tooltip.style.display === 'block') { + const r = svg.getBoundingClientRect(); + tooltip.style.left = (e.clientX - r.left + 12) + 'px'; + tooltip.style.top = (e.clientY - r.top + 12) + 'px'; + } + }); + root.addEventListener('mouseout', hideTooltip); + root.addEventListener('click', (e) => { + const g = e.target.closest('.node'); if (!g) return; + e.stopPropagation(); + selectNode(g.getAttribute('data-name')); + }); + svg.addEventListener('click', (e) => { if (e.target === svg || e.target === root) clearSelection(); }); + document.addEventListener('keydown', (e) => { if (e.key === 'Escape') { clearSelection(); document.getElementById('search').value = ''; applySearch(); }}); + + // ---- Search ---- + function applySearch() { + const q = document.getElementById('search').value.trim().toLowerCase(); + root.querySelectorAll('.node.match').forEach(n => n.classList.remove('match')); + if (!q) { root.classList.remove('has-search'); return; } + root.classList.add('has-search'); + let count = 0; + root.querySelectorAll('.node').forEach(n => { + // Match against both the stable id (data-name) and the resolved + // readable display name (e.g. a voltage-level name), so operators can + // find a node by either spelling. nodeDisplayName ignores the + // graphviz "\N" placeholder, so label-less nodes match on their id. + const id = (n.getAttribute('data-name') || '').toLowerCase(); + const disp = nodeDisplayName(n).toLowerCase(); + if (id.indexOf(q) !== -1 || (disp && disp.indexOf(q) !== -1)) { + n.classList.add('match'); count++; + } + }); + } + document.getElementById('search').addEventListener('input', applySearch); + + // ---- Layer toggles ---- + // Membership-based dim model: every node and edge knows which + // layers claim it. An element is **visible** iff at least one of + // its claiming layers is currently checked. Elements with no + // memberships at all are dimmed whenever any layer toggle differs + // from the default "all checked" state — that matches the + // user-facing intent that ticking a single layer focuses the view + // on that layer only and recedes everything else. + const layersEl = document.getElementById('layers'); + function swatchInner(swatch) { + if (!swatch) return ''; + const COLORED = {coral:1, blue:1, black:1, gray:1, dimgray:1, darkred:1, red:1, green:1}; + if (COLORED[swatch]) return ''; + if (swatch === 'diamond') return ''; + if (swatch === 'red-loop') return ''; + if (swatch === 'constrained-path') return ''; + if (swatch === 'overload') return ''; + if (swatch === 'monitored') return ''; + if (swatch === 'extra-cut') return ''; + // Match the upstream node fillcolors set in build_nodes: + // prod (prod_minus_load > 0) → coral + // load (prod_minus_load < 0) → lightblue + if (swatch === 'prod-node') return ''; + if (swatch === 'load-node') return ''; + return ''; + } + function swatchStyle(swatch) { + const COLORED = {coral:1, blue:1, black:1, gray:1, dimgray:1, darkred:1, red:1, green:1}; + if (COLORED[swatch]) return 'background:' + swatch; + return 'background:#fff'; + } + + // Build per-element layer membership maps once. These are consulted + // by `applyAllLayers()` on every checkbox change so an element + // claimed by multiple layers never gets stuck in `layer-off` + // because an unrelated checkbox flipped the wrong way. + const nodeMemberships = new Map(); // data-name -> Array + const edgeMemberships = new Map(); // edge id -> Array + for (let i = 0; i < MODEL.layers.length; i++) { + const layer = MODEL.layers[i]; + for (const n of (layer.nodes || [])) { + const arr = nodeMemberships.get(n); + if (arr) arr.push(i); else nodeMemberships.set(n, [i]); + } + for (const e of (layer.edges || [])) { + const arr = edgeMemberships.get(e); + if (arr) arr.push(i); else edgeMemberships.set(e, [i]); + } + } + + const layerCheckboxes = []; + let lastSection = null; + for (const layer of MODEL.layers) { + if (layer.section && layer.section !== lastSection) { + const header = document.createElement('h3'); + header.className = 'layer-section-header'; + header.textContent = layer.section; + layersEl.appendChild(header); + lastSection = layer.section; + } + const id = 'layer-' + layer.key.replace(/[^a-z0-9]/gi, '-'); + const wrap = document.createElement('label'); + const total = (layer.nodes ? layer.nodes.length : 0) + (layer.edges ? layer.edges.length : 0); + wrap.innerHTML = '' + + '' + swatchInner(layer.swatch) + '' + + '' + escapeHtml(layer.label) + ' (' + total + ')'; + layersEl.appendChild(wrap); + const cb = wrap.querySelector('input'); + layerCheckboxes.push(cb); + cb.addEventListener('change', (e) => { + applyAllLayers(); + window.parent.postMessage({ + type: 'cs4g:overflow-layer-toggled', + key: layer.key, + label: layer.label, + visible: e.target.checked + }, '*'); + }); + } + + function applyAllLayers() { + const checkedSet = new Set(); + let allChecked = true; + for (let i = 0; i < layerCheckboxes.length; i++) { + if (layerCheckboxes[i].checked) checkedSet.add(i); + else allChecked = false; + } + function shouldDim(memberships) { + if (allChecked) return false; + if (!memberships || memberships.length === 0) return true; + for (const idx of memberships) { + if (checkedSet.has(idx)) return false; + } + return true; + } + root.querySelectorAll('.node').forEach((el) => { + const name = el.getAttribute('data-name'); + el.classList.toggle('layer-off', shouldDim(nodeMemberships.get(name))); + }); + root.querySelectorAll('.edge').forEach((el) => { + const id = el.getAttribute('id'); + el.classList.toggle('layer-off', shouldDim(edgeMemberships.get(id))); + }); + } + // Expose for testability and external triggers (e.g. parent app + // requesting a full repaint after dynamic content updates). + window.__cs4gApplyAllLayers = applyAllLayers; + + function setAllLayers(visible) { + let changed = false; + for (let i = 0; i < layerCheckboxes.length; i++) { + if (layerCheckboxes[i].checked !== visible) { + layerCheckboxes[i].checked = visible; + changed = true; + } + } + if (changed) applyAllLayers(); + window.parent.postMessage({ + type: 'cs4g:overflow-select-all-layers', + visible: visible + }, '*'); + } + document.getElementById('layers-select-all').addEventListener('click', () => setAllLayers(true)); + document.getElementById('layers-select-none').addEventListener('click', () => setAllLayers(false)); + + // Double-click on a graph node bubbles up to the parent window, + // which is responsible for opening the corresponding Single-Line + // Diagram view. The node `data-name` is the voltage-level (or + // substation, depending on backend) identifier the parent will + // resolve to a SLD endpoint. + root.addEventListener('dblclick', (ev) => { + const g = ev.target.closest('.node'); + if (!g) return; + ev.preventDefault(); + ev.stopPropagation(); + const name = g.getAttribute('data-name') || ''; + if (!name) return; + window.parent.postMessage({ + type: 'cs4g:overflow-node-double-clicked', + name: name + }, '*'); + }); + + document.getElementById('stats').textContent = + MODEL.nodes.length + ' nodes, ' + MODEL.edges.length + ' edges'; +})(); diff --git a/alphaDeesp/core/interactive_html/constants.py b/alphaDeesp/core/interactive_html/constants.py new file mode 100644 index 00000000..0b84cd0a --- /dev/null +++ b/alphaDeesp/core/interactive_html/constants.py @@ -0,0 +1,113 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Layer / section constants for the interactive HTML viewer.""" + +from typing import Dict, List + +# Section names rendered as

headers in the sidebar layer list. +# Layers carry their section in the model so the JS just groups by it. +_SECTION_STRUCTURAL = "Structural Paths" +_SECTION_PROPERTIES = "Individual entities properties" +_SECTION_FLOWS = "Flow redispatch values" + +# Edge color → human-readable layer label. Restricted to the three flow +# polarities (positive / negative / null). The historical "black" / +# "gray" / "darkred" buckets are dropped because they are redundant +# with the explicit semantic flags (is_overload / is_monitored) or +# carry no operational meaning on their own. +_LAYER_LABELS: Dict[str, str] = { + "coral": "Positive", + "blue": "Negative", + "dimgray": "Null", +} + +# Edge style → layer label (orthogonal to color). +_STYLE_LAYERS: Dict[str, str] = { + "dotted": "Non-reconnectable", + "dashed": "Reconnectable", + "tapered": "Swapped flow", +} + +# Source-of-truth attribute layers — values produced upstream by +# alphaDeesp / expert_op4grid_recommender as explicit boolean flags on +# nodes and/or edges. The viewer scans for them and exposes a layer +# toggle for each. Defining them here (rather than guessing from edge +# colours / shapes) keeps the layer list semantically stable when the +# visual palette evolves. +# +# Each entry: +# key — `data-attr-*` flag scanned on node and edge groups +# label — human-readable sidebar label +# swatch — special-case identifier consumed by the JS +# template to render an inline SVG glyph (no colour +# chip — these layers cut across the colour palette) +# scope — "node", "edge", or "both" +_SEMANTIC_LAYERS: List[Dict[str, str]] = [ + {"key": "on_constrained_path", "label": "Constrained path", "swatch": "constrained-path", "scope": "both"}, + {"key": "in_red_loop", "label": "Red-loop paths", "swatch": "red-loop", "scope": "both"}, + {"key": "is_overload", "label": "Overloads", "swatch": "overload", "scope": "edge"}, + {"key": "is_monitored", "label": "Low margin lines", "swatch": "monitored", "scope": "edge"}, + # Operator-supplied extras (ExpertAgent's `additionalLinesToCut`): + # cut in the analysis like overloads but rendered with their + # natural flow colour and excluded from the Overloads / + # Low margin lines layers. Surfaced as a dedicated layer so the + # operator can still see how their choice materialised. + {"key": "is_extra_cut", "label": "Extra lines to prevent flow increase", "swatch": "extra-cut", "scope": "edge"}, + {"key": "is_hub", "label": "Hubs", "swatch": "diamond", "scope": "node"}, +] + +# Threshold below which a node's ``value`` (prod − load, in MW) is +# treated as "no real prod/load here". Build-time conventions in the +# upstream simulators tag every node with ``prod_or_load="load"`` and +# ``value="0.0"`` even when no consumption exists, so a strict +# ``prod_or_load == "load"`` test would flood the layer with empty +# nodes. The 1 MW floor matches operator practice. +_PROD_LOAD_VALUE_FLOOR_MW = 1.0 + +# Per-kind config for the value-based node layers. Matched against the +# ``prod_or_load`` attribute set by ``build_nodes`` upstream +# (alphaDeesp/core/graphs/power_flow_graph.py and the simulator-specific +# build_nodes_v2 helpers). Each entry produces a single layer in the +# "Individual entities properties" section, populated only with the +# nodes whose absolute ``value`` clears ``_PROD_LOAD_VALUE_FLOOR_MW``. +_VALUE_NODE_LAYERS: List[Dict[str, str]] = [ + {"key": "prod", "label": "Production nodes", "swatch": "prod-node"}, + {"key": "load", "label": "Consumption nodes", "swatch": "load-node"}, +] + +# Per-layer-key section assignment. The JS renders one ``

`` per +# section in the order the sections are first encountered. +_LAYER_SECTIONS: Dict[str, str] = { + # Structural paths — multi-edge structures. + "semantic:on_constrained_path": _SECTION_STRUCTURAL, + "semantic:in_red_loop": _SECTION_STRUCTURAL, + # Individual entities properties — per-edge / per-node flags. + "semantic:is_overload": _SECTION_PROPERTIES, + "semantic:is_monitored": _SECTION_PROPERTIES, + "semantic:is_extra_cut": _SECTION_PROPERTIES, + "semantic:is_hub": _SECTION_PROPERTIES, + "style:dashed": _SECTION_PROPERTIES, + "style:dotted": _SECTION_PROPERTIES, + "style:tapered": _SECTION_PROPERTIES, + # Value-based node layers — see _VALUE_NODE_LAYERS / build_nodes. + "node:prod": _SECTION_PROPERTIES, + "node:load": _SECTION_PROPERTIES, + # Flow polarity buckets. + "color:coral": _SECTION_FLOWS, + "color:blue": _SECTION_FLOWS, + "color:dimgray": _SECTION_FLOWS, +} + +# Render order: sections appear top-to-bottom in this order; layers +# within a section appear in the order the model emits them. +_SECTION_ORDER: List[str] = [ + _SECTION_STRUCTURAL, + _SECTION_PROPERTIES, + _SECTION_FLOWS, +] diff --git a/alphaDeesp/core/interactive_html/helpers.py b/alphaDeesp/core/interactive_html/helpers.py new file mode 100644 index 00000000..af1cdd03 --- /dev/null +++ b/alphaDeesp/core/interactive_html/helpers.py @@ -0,0 +1,68 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Pure helpers for the interactive viewer: title/colour parsing and +attribute normalisation. No graph or asset dependencies.""" + +from __future__ import annotations + +import html as html_mod +from typing import Any, Dict, Optional, Tuple + +from alphaDeesp.core.interactive_html.constants import _LAYER_LABELS + + +def _decode_title(text: str) -> str: + """Graphviz HTML-escapes node/edge titles in SVG (``A->B``).""" + return html_mod.unescape(text or "") + + +def _split_edge_title(title: str) -> Tuple[str, str]: + """Edge titles are ``"->["`` (digraph) or ``"--"``.""" + title = _decode_title(title) + for sep in ("->", "--"): + if sep in title: + src, dst = title.split(sep, 1) + return src.strip(), dst.strip() + return title, "" + + +def _color_to_layer_key(color: str) -> Optional[str]: + """Map a (possibly compound or hex) color to a known layer key.""" + if not color: + return None + base = color.split(":", 1)[0].strip().strip('"').lower() + if base in _LAYER_LABELS: + return base + return None + + +def _normalize_attrs(raw: Dict[str, Any]) -> Dict[str, Any]: + """Strip Graphviz-internal _draw_/_ldraw_ keys and quoted strings.""" + out: Dict[str, Any] = {} + for k, v in raw.items(): + if k.startswith("_") or k in ("nodes", "edges", "objects", "subgraphs"): + continue + if isinstance(v, str): + v = v.strip().strip('"') + out[k] = v + return out + + +def _is_truthy_flag(value: Any) -> bool: + """Check whether a graph attribute represents a True boolean flag. + + Boolean attributes round-trip through pydot/graphviz/dot-json as + string ``"True"``. We accept the native Python ``True`` for + in-process callers and the string form for the JSON path. + """ + if value is True: + return True + if isinstance(value, str): + return value.strip().lower() == "true" + return False diff --git a/alphaDeesp/core/interactive_html/layers.py b/alphaDeesp/core/interactive_html/layers.py new file mode 100644 index 00000000..b39e3eed --- /dev/null +++ b/alphaDeesp/core/interactive_html/layers.py @@ -0,0 +1,191 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Layer-index construction: group edges/nodes by colour, style and +semantic flag so the viewer can offer toggles.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Tuple + +from alphaDeesp.core.interactive_html.constants import ( + _LAYER_LABELS, + _LAYER_SECTIONS, + _PROD_LOAD_VALUE_FLOOR_MW, + _SECTION_ORDER, + _SEMANTIC_LAYERS, + _STYLE_LAYERS, + _VALUE_NODE_LAYERS, +) +from alphaDeesp.core.interactive_html.helpers import ( + _color_to_layer_key, + _is_truthy_flag, +) + + +def _build_layer_index( + edges: List[Dict[str, Any]], + nodes: List[Dict[str, Any]] | None = None, +) -> List[Dict[str, Any]]: + """Group edges & nodes by colour / style / semantic flag so the UI + can offer toggles. Each layer carries both ``nodes`` and ``edges`` + id lists (either may be empty). + """ + by_color: Dict[str, List[str]] = {} + by_style: Dict[str, List[str]] = {} + edge_id_lookup = {e["id"]: e for e in edges} + for e in edges: + color_key = _color_to_layer_key(e["attrs"].get("color", "")) + if color_key: + by_color.setdefault(color_key, []).append(e["id"]) + style = (e["attrs"].get("style") or "").lower() + if style in _STYLE_LAYERS: + by_style.setdefault(style, []).append(e["id"]) + + # Semantic flags scanned on both nodes and edges. Only emit a layer + # entry if at least one element carries the flag — otherwise the + # checkbox would be useless and noise. + semantic_buckets: Dict[str, Dict[str, List[str]]] = { + cfg["key"]: {"nodes": [], "edges": []} for cfg in _SEMANTIC_LAYERS + } + if nodes: + for n in nodes: + for cfg in _SEMANTIC_LAYERS: + if cfg["scope"] in ("node", "both") and _is_truthy_flag( + n["attrs"].get(cfg["key"]) + ): + semantic_buckets[cfg["key"]]["nodes"].append(n["name"]) + for e in edges: + for cfg in _SEMANTIC_LAYERS: + if cfg["scope"] in ("edge", "both") and _is_truthy_flag( + e["attrs"].get(cfg["key"]) + ): + semantic_buckets[cfg["key"]]["edges"].append(e["id"]) + + # For each colour / style layer, the endpoint nodes of every + # claimed edge are also added to the layer so toggling, e.g., + # "Positive overflow" alone keeps the substations the coral edges + # connect visible (the operator can still read the topology around + # the highlighted edges instead of seeing them float in dimmed + # space). We dedupe while preserving first-seen order. + edge_id_to_endpoints: Dict[str, Tuple[str, str]] = { + e["id"]: (e["source"], e["target"]) for e in edges + } + + def _endpoint_nodes(edge_ids: List[str]) -> List[str]: + seen: Dict[str, None] = {} + for eid in edge_ids: + ends = edge_id_to_endpoints.get(eid) + if not ends: + continue + for n in ends: + if n not in seen: + seen[n] = None + return list(seen.keys()) + + # Edge-only semantic layers (Overloads, Low margin lines) carry + # their edges' endpoints too — same UX rationale as colour/style + # layers: when the operator ticks "Overloads" alone the affected + # substations stay visible. + _EDGE_ONLY_SEMANTIC_KEYS = { + cfg["key"] for cfg in _SEMANTIC_LAYERS if cfg["scope"] == "edge" + } + + def _merge_dedup(base: List[str], extra: List[str]) -> List[str]: + seen: Dict[str, None] = {n: None for n in base} + for n in extra: + if n not in seen: + seen[n] = None + return list(seen.keys()) + + raw_layers: List[Dict[str, Any]] = [] + for key, ids in by_color.items(): + raw_layers.append({ + "key": f"color:{key}", + "label": _LAYER_LABELS[key], + "swatch": key, + "nodes": _endpoint_nodes(ids), + "edges": ids, + }) + for key, ids in by_style.items(): + raw_layers.append({ + "key": f"style:{key}", + "label": _STYLE_LAYERS[key], + "swatch": "", + "nodes": _endpoint_nodes(ids), + "edges": ids, + }) + for cfg in _SEMANTIC_LAYERS: + bucket = semantic_buckets[cfg["key"]] + if not bucket["nodes"] and not bucket["edges"]: + continue + nodes_for_layer = bucket["nodes"] + if cfg["key"] in _EDGE_ONLY_SEMANTIC_KEYS: + nodes_for_layer = _merge_dedup( + nodes_for_layer, _endpoint_nodes(bucket["edges"]) + ) + raw_layers.append({ + "key": f"semantic:{cfg['key']}", + "label": cfg["label"], + "swatch": cfg["swatch"], + "nodes": nodes_for_layer, + "edges": bucket["edges"], + }) + + # Value-based node layers (Production / Consumption). Built from + # the ``prod_or_load`` attribute upstream tagged on every node by + # ``build_nodes`` — see _VALUE_NODE_LAYERS. The white-coloured + # zero-balance nodes carry ``prod_or_load="load"`` AND + # ``value="0.0"`` upstream by convention; the 1 MW floor filters + # them out so the operator's "Consumption nodes" toggle doesn't + # also tag every passive substation. + if nodes: + value_buckets: Dict[str, List[str]] = { + cfg["key"]: [] for cfg in _VALUE_NODE_LAYERS + } + for n in nodes: + kind = n["attrs"].get("prod_or_load") + if kind not in value_buckets: + continue + try: + magnitude = abs(float(n["attrs"].get("value", "0"))) + except (TypeError, ValueError): + continue + if magnitude < _PROD_LOAD_VALUE_FLOOR_MW: + continue + value_buckets[kind].append(n["name"]) + for cfg in _VALUE_NODE_LAYERS: + ids = value_buckets[cfg["key"]] + if not ids: + continue + raw_layers.append({ + "key": f"node:{cfg['key']}", + "label": cfg["label"], + "swatch": cfg["swatch"], + "nodes": ids, + "edges": [], + }) + + # Drop layers without a section assignment (e.g. ``color:black``, + # ``color:gray``, ``color:darkred`` — historically redundant + # buckets). Then group by section in the canonical order so the + # JS can render them with section headers. + sectioned: Dict[str, List[Dict[str, Any]]] = {s: [] for s in _SECTION_ORDER} + for layer in raw_layers: + section = _LAYER_SECTIONS.get(layer["key"]) + if section is None: + continue + layer["section"] = section + sectioned.setdefault(section, []).append(layer) + + layers: List[Dict[str, Any]] = [] + for section in _SECTION_ORDER: + layers.extend(sectioned.get(section, [])) + # Silence unused-var warning; lookup retained for future hover xref. + del edge_id_lookup + return layers diff --git a/alphaDeesp/core/interactive_html/model.py b/alphaDeesp/core/interactive_html/model.py new file mode 100644 index 00000000..392047a6 --- /dev/null +++ b/alphaDeesp/core/interactive_html/model.py @@ -0,0 +1,62 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Parse ``dot -Tjson`` into a flat node/edge model + adjacency map.""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List + +from alphaDeesp.core.interactive_html.helpers import _normalize_attrs +from alphaDeesp.core.interactive_html.layers import _build_layer_index + + +def _model_from_dot_json(dot_json: bytes) -> Dict[str, Any]: + """Parse ``dot -Tjson`` into a flat node/edge model + adjacency map.""" + data = json.loads(dot_json.decode("utf-8")) + objects: List[Dict[str, Any]] = data.get("objects", []) + raw_edges: List[Dict[str, Any]] = data.get("edges", []) + + nodes: List[Dict[str, Any]] = [] + name_by_gvid: Dict[int, str] = {} + for i, obj in enumerate(objects): + if "nodes" in obj or "subgraphs" in obj: + # cluster/subgraph entry — skip in v1 + continue + name = obj.get("name", f"node{i}") + name_by_gvid[obj.get("_gvid", i)] = name + nodes.append({ + "name": name, + "attrs": _normalize_attrs(obj), + }) + + edges: List[Dict[str, Any]] = [] + adjacency: Dict[str, List[Dict[str, str]]] = {n["name"]: [] for n in nodes} + for j, edge in enumerate(raw_edges): + src = name_by_gvid.get(edge.get("tail")) + dst = name_by_gvid.get(edge.get("head")) + if src is None or dst is None: + continue + attrs = _normalize_attrs(edge) + edges.append({ + "id": f"edge{j + 1}", # matches Graphviz SVG id naming + "source": src, + "target": dst, + "attrs": attrs, + }) + adjacency.setdefault(src, []).append({"node": dst, "edge": f"edge{j + 1}"}) + adjacency.setdefault(dst, []).append({"node": src, "edge": f"edge{j + 1}"}) + + layers = _build_layer_index(edges, nodes) + return { + "nodes": nodes, + "edges": edges, + "adjacency": adjacency, + "layers": layers, + } diff --git a/alphaDeesp/core/interactive_html/render.py b/alphaDeesp/core/interactive_html/render.py new file mode 100644 index 00000000..c8ac00be --- /dev/null +++ b/alphaDeesp/core/interactive_html/render.py @@ -0,0 +1,49 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Assemble the interactive viewer HTML from a pydot graph.""" + +from __future__ import annotations + +import html as html_mod +import json +from typing import Any + +import pydot + +from alphaDeesp.core.interactive_html.model import _model_from_dot_json +from alphaDeesp.core.interactive_html.svg import ( + _align_edge_ids_with_svg, + _inject_svg_data_attrs, +) +from alphaDeesp.core.interactive_html.template import html_template + + +def build_interactive_html( + pydot_graph: pydot.Graph, + prog: Any = "dot", + title: str = "ExpertOp4Grid — interactive overflow graph", +) -> str: + """Render ``pydot_graph`` to interactive HTML. + + Returns the HTML string. Caller decides where to write it. + """ + svg_bytes = pydot_graph.create(prog=prog, format="svg") + json_bytes = pydot_graph.create(prog=prog, format="json") + model = _model_from_dot_json(json_bytes) + # Align JSON edge ids with the SVG element ids — graphviz emits the + # two orderings independently and the downstream JS toggles SVG + # elements by id, so a mismatch silently dims the wrong edges. + model = _align_edge_ids_with_svg(svg_bytes, model) + annotated_svg = _inject_svg_data_attrs(svg_bytes, model) + return ( + html_template() + .replace("__TITLE__", html_mod.escape(title)) + .replace("__SVG__", annotated_svg) + .replace("__MODEL_JSON__", json.dumps(model)) + ) diff --git a/alphaDeesp/core/interactive_html/svg.py b/alphaDeesp/core/interactive_html/svg.py new file mode 100644 index 00000000..3370dc77 --- /dev/null +++ b/alphaDeesp/core/interactive_html/svg.py @@ -0,0 +1,175 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Annotate the Graphviz SVG with stable ``data-*`` attributes and align +JSON edge ids with the SVG element ids.""" + +from __future__ import annotations + +import html as html_mod +import re +from typing import Any, Dict, List, Tuple + +from alphaDeesp.core.interactive_html.constants import _STYLE_LAYERS +from alphaDeesp.core.interactive_html.helpers import ( + _color_to_layer_key, + _decode_title, + _split_edge_title, +) + + +def _inject_svg_data_attrs(svg_bytes: bytes, model: Dict[str, Any]) -> str: + """Annotate Graphviz SVG nodes/edges with stable data-* attributes. + + Graphviz emits ``NAME``; we + rely on the title to look up our model entries and append data-* + attributes (which the JS uses for selectors and tooltips). + """ + svg = svg_bytes.decode("utf-8") + edge_by_id = {e["id"]: e for e in model["edges"]} + node_by_name = {n["name"]: n for n in model["nodes"]} + + def _attrs_to_data(prefix: str, attrs: Dict[str, Any]) -> str: + out: List[str] = [] + for k, v in attrs.items(): + safe_v = html_mod.escape(str(v), quote=True) + out.append(f' data-{prefix}-{k}="{safe_v}"') + return "".join(out) + + def _node_repl(match: re.Match) -> str: + gid = match.group(1) + title = match.group(2) + name = _decode_title(title) + node = node_by_name.get(name) + if not node: + return match.group(0) + data = _attrs_to_data("attr", node["attrs"]) + return ( + f'{title}' + ) + + def _edge_repl(match: re.Match) -> str: + gid = match.group(1) + title = match.group(2) + edge = edge_by_id.get(gid) + if not edge: + return match.group(0) + src, dst = edge["source"], edge["target"] + layers: List[str] = [] + color_key = _color_to_layer_key(edge["attrs"].get("color", "")) + if color_key: + layers.append(f"color:{color_key}") + style = (edge["attrs"].get("style") or "").lower() + if style in _STYLE_LAYERS: + layers.append(f"style:{style}") + data = _attrs_to_data("attr", edge["attrs"]) + layer_attr = f' data-layers="{html_mod.escape(" ".join(layers), quote=True)}"' if layers else "" + return ( + f'{title}' + ) + + svg = re.sub( + r'\s*([^<]*)', + _node_repl, + svg, + ) + svg = re.sub( + r'\s*([^<]*)', + _edge_repl, + svg, + ) + return svg + + +def _align_edge_ids_with_svg(svg_bytes: bytes, model: Dict[str, Any]) -> Dict[str, Any]: + """Re-key edges in ``model`` so their ``id`` field matches the SVG's + ```` for the SAME (src, dst) endpoints. + + Background + ---------- + Graphviz emits edge IDs ``edgeN`` in **two independent orderings** for + the SVG and the JSON outputs of the same graph. ``_model_from_dot_json`` + assigns IDs by JSON-edge index but the SVG element with the same + ``edgeN`` id often refers to a different edge (different (src, dst) + pair). The downstream JS dim layer queries SVG elements **by id** — + so a mismatch makes the wrong edges dim/highlight when a layer + toggle is flipped (this is exactly the user-reported confusion + SSV.OP7→GROSNP7 ↔ SSV.OP7→CREYSP7 / CHALOP6→CPVANP6 ↔ + CHALOP6→CHALOP3 in the small-grid scenario). + + Fix + --- + Walk the SVG, parse each edge's ```` to extract its true + (src, dst), and greedily pair it with a JSON-side edge of matching + endpoints. Each JSON edge is consumed at most once (parallel edges + are paired in their relative order, which is stable across SVG and + JSON). The model's edge IDs are updated in place; adjacency and + layer membership lists are remapped through the same dict. + + Returns the updated model. + """ + svg = svg_bytes.decode("utf-8") + # Walk SVG edges in document order: ``<g id="edgeN" class="edge"> + # <title>SRC->DST``. Graphviz HTML-escapes the title. + pattern = re.compile( + r'\s*([^<]*)' + ) + svg_edges_in_order: List[Tuple[str, str, str]] = [] + for m in pattern.finditer(svg): + gid = m.group(1) + src, dst = _split_edge_title(m.group(2)) + svg_edges_in_order.append((gid, src, dst)) + + # Build a per-(src, dst) FIFO of JSON edges keeping their original order. + json_edges = model["edges"] + by_pair: Dict[Tuple[str, str], List[int]] = {} + for i, e in enumerate(json_edges): + by_pair.setdefault((e["source"], e["target"]), []).append(i) + + # Greedily match each SVG edge to the next un-consumed JSON edge of the + # same endpoints. The remap dict translates "old (JSON-order) id" → "new + # (SVG-order) id". + remap: Dict[str, str] = {} + for svg_id, s, t in svg_edges_in_order: + candidates = by_pair.get((s, t)) or by_pair.get((t, s)) + if not candidates: + continue + json_idx = candidates.pop(0) + old_id = json_edges[json_idx]["id"] + if old_id == svg_id: + continue + remap[old_id] = svg_id + + if not remap: + return model + + # Apply the remap. ``remap`` may contain swaps (a→b and b→a). To avoid + # collisions we materialise the new IDs through a fresh dict in two + # passes: first relabel each JSON edge to its SVG-aligned id, then walk + # adjacency / layers to substitute the references. + edge_id_lookup = {e["id"]: e for e in json_edges} + for old_id, new_id in remap.items(): + edge = edge_id_lookup[old_id] + edge["id"] = new_id + + # Adjacency entries reference edge ids by string — apply the same + # substitution there. + for entries in model.get("adjacency", {}).values(): + for entry in entries: + if entry.get("edge") in remap: + entry["edge"] = remap[entry["edge"]] + + # Layer membership lists use the same string ids. + for layer in model.get("layers", []): + layer["edges"] = [remap.get(eid, eid) for eid in layer.get("edges", [])] + + return model diff --git a/alphaDeesp/core/interactive_html/template.py b/alphaDeesp/core/interactive_html/template.py new file mode 100644 index 00000000..c098f279 --- /dev/null +++ b/alphaDeesp/core/interactive_html/template.py @@ -0,0 +1,26 @@ +# Copyright (c) 2019-2020, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of ExpertOp4Grid, an expert system approach to solve flow congestions in power grids + +"""Load the externalised viewer assets (CSS / JS / HTML skeleton) and +reconstitute the self-contained HTML template.""" + +from functools import lru_cache +from pathlib import Path + +_ASSETS = Path(__file__).parent / "assets" + + +@lru_cache(maxsize=1) +def html_template() -> str: + """Self-contained HTML template with ``__TITLE__`` / ``__SVG__`` / + ``__MODEL_JSON__`` placeholders, reconstituted from the externalised + ``viewer.css`` and ``viewer.js`` assets.""" + template = (_ASSETS / "template.html").read_text(encoding="utf-8") + css = (_ASSETS / "viewer.css").read_text(encoding="utf-8") + js = (_ASSETS / "viewer.js").read_text(encoding="utf-8") + return template.replace("__CSS__", css).replace("__JS__", js) diff --git a/alphaDeesp/core/simulation.py b/alphaDeesp/core/simulation.py index 3586e847..9c0ae31a 100755 --- a/alphaDeesp/core/simulation.py +++ b/alphaDeesp/core/simulation.py @@ -8,7 +8,6 @@ import logging from abc import ABC, abstractmethod -from math import fabs from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import numpy as np @@ -211,7 +210,15 @@ def create_end_result_empty_dataframe() -> pd.DataFrame: return end_result_data_frame def create_df(self, d: Dict[str, Any], line_to_cut: List[int]) -> pd.DataFrame: - """arg: d represents a topology""" + """Build the topology + flow DataFrame for a cut line. + + ``d`` represents a topology (one row per grid line, in line-id order). + The row-per-line ordering is a load-bearing invariant: the overloaded + line's delta flow is read positionally by ``line_to_cut`` id, and + ``OverFlowGraph`` matches ``lines_to_cut`` against row positions too. + The former ``iterrows`` passes are vectorised with numpy masks; the + numerical results are identical. + """ # HERE WE CREATE DATAFRAME df = pd.DataFrame(d["edges"]) pd.set_option("display.float_format", lambda x: "%.3f" % x) @@ -219,78 +226,51 @@ def create_df(self, d: Dict[str, Any], line_to_cut: List[int]) -> pd.DataFrame: # takes a dataframe and swaps branches init_flows < 0 self.branch_direction_swaps(df) - new_flows = self.cut_lines_and_recomputes_flows(line_to_cut) - # print("new simulated flows = ", new_flows) + new_flows = np.asarray(self.cut_lines_and_recomputes_flows(line_to_cut), dtype=float) + swapped = df["swapped"].to_numpy() # here we multiply by (-1) new flows that are reversed - n_flows = [] - for f, swapped in zip(new_flows, df["swapped"]): - if swapped: - n_flows.append(f * -1) - else: - n_flows.append(f) + new_flows_signed = np.where(swapped, -new_flows, new_flows) + df["new_flows"] = new_flows_signed - df["new_flows"] = n_flows + init_flows = df["init_flows"].to_numpy(dtype=float) + abs_new = np.abs(new_flows_signed) + abs_init = np.abs(init_flows) # if new_flows < 0, and abs(new) > abs(init) then True (we invert edge direction) else False - new_flows_swapped = [] - - for i, row in df.iterrows(): - # if newf < 0. and fabs(new_flows) > fabs(initf): - if row["new_flows"] < 0 and fabs(row["new_flows"]) > fabs(row["init_flows"]): - new_flows_swapped.append(True) - else: - new_flows_swapped.append(False) - + new_flows_swapped = (new_flows_signed < 0) & (abs_new > abs_init) df["new_flows_swapped"] = new_flows_swapped - delta_flo = [] - # now we add delta flows - # report=abs(new_flows) - abs(init_flows) si le flux n'a pas change de direction - # Si le flux a changé de direction, il y a 2 cas: - # soit le nouveau flux est plus faible et dans ce cas, le report est négatif (on a déchargé la ligne) et le - # report = -(abs(new_flows) + abs(init_flows)) - # sinon le report est positif et le report = - # report = abs(new_flows) + abs(init_flows) - for i, row in df.iterrows(): - if row["new_flows_swapped"]: - delta_flo.append(fabs(row["new_flows"]) + fabs(row["init_flows"])) - # here we swap origin and ext - idx_or = row["idx_or"] - df.at[i, "idx_or"] = row["idx_ex"] - df.at[i, "idx_ex"] = idx_or - df.at[i, "init_flows"] = fabs(row["init_flows"]) - # print(f"row #{i}, swapped idxor and idxer") - elif (np.sign(row["new_flows"])!=np.sign(row["init_flows"])) and (row["new_flows"]!=0) and (row["init_flows"]!=0):#sign of 0 value is 0... - delta_flo.append(-(fabs(row["new_flows"]) + fabs(row["init_flows"])))#negative flow dispacth in that case - else: - delta_flo.append(fabs(row["new_flows"]) - fabs(row["init_flows"])) - - # delta_flows = self.df["new_flows"].abs() - self.df["init_flows"].abs() + # report = abs(new) - abs(init) if the flow did not change direction. + # If it did there are two cases: + # * new_flows_swapped -> the edge is inverted: report = abs(new) + abs(init) + # * opposite signs (both non-zero) -> discharged: report = -(abs(new) + abs(init)) + opposite_sign = ( + (np.sign(new_flows_signed) != np.sign(init_flows)) + & (new_flows_signed != 0) + & (init_flows != 0) + ) + # ``elif`` semantics: opposite_sign only applies where new_flows_swapped is False. + discharged = opposite_sign & ~new_flows_swapped + delta_flo = np.where( + new_flows_swapped, abs_new + abs_init, + np.where(discharged, -(abs_new + abs_init), abs_new - abs_init), + ) df["delta_flows"] = delta_flo - # small modification test to have multiple components - # self.df.set_value(0, "delta_flows", -32.) - # self.df.set_value(4, "delta_flows", -42.) - # self.df.set_value(5, "delta_flows", -22.) + # swap origin/extremity (and abs the init flow) on inverted edges + idx_or = df["idx_or"].to_numpy() + idx_ex = df["idx_ex"].to_numpy() + df["idx_or"] = np.where(new_flows_swapped, idx_ex, idx_or) + df["idx_ex"] = np.where(new_flows_swapped, idx_or, idx_ex) + df["init_flows"] = np.where(new_flows_swapped, abs_init, init_flows) - # DO NOT USE SET_VALUE ANYMORE, USE DF.AT INSTEAD - # df.at[5, "delta_flows"] = -22. - - # now we identify gray edges - gray_edges = [] - ltc_report = df["delta_flows"].abs()[line_to_cut[0]]#pd.DataFrame.max(df["delta_flows"].abs()) - # print("max = ", max_report) + # now we identify gray edges (below-significance redispatch). + # ``.iloc`` makes the positional (row == line id) access explicit. + ltc_report = float(df["delta_flows"].abs().iloc[line_to_cut[0]]) max_overload = ltc_report * float(self.param_options["ThresholdReportOfLine"]) - # print("max overload = ", max_overload) - for edge_value in df["delta_flows"]: - if fabs(edge_value) < max_overload: - gray_edges.append(True) - else: - gray_edges.append(False) - # print("gray edges = ", gray_edges) - df["gray_edges"] = gray_edges + df["gray_edges"] = df["delta_flows"].abs().to_numpy() < max_overload if getattr(self, "debug", False): logger.debug("==== After gray_edges added IN FUNCTION CREATE DF ====") @@ -300,27 +280,21 @@ def create_df(self, d: Dict[str, Any], line_to_cut: List[int]) -> pd.DataFrame: @staticmethod def branch_direction_swaps(df: pd.DataFrame) -> None: - """we parse self.df and invert branches init_flows < 0""" - swapped = [] - for i, row in df.iterrows(): - # print("i {} row {}".format(i, row)) - # a = row["delta_flows"] - # b = row["final_delta_flows"] - # if np.sign(a) != np.sign(b): - - a = row["init_flows"] - if a < 0 and a != 0.: - # here we swap origin and ext - idx_or = row["idx_or"] - df.at[i, "idx_or"] = row["idx_ex"] - df.at[i, "idx_ex"] = idx_or - df.at[i, "init_flows"] = fabs(row["init_flows"]) - # print(f"row #{i}, swapped idxor and idxer") - swapped.append(True) - else: - swapped.append(False) - - df["swapped"] = swapped + """Invert branches whose ``init_flows`` is negative (draw them forward). + + Vectorised replacement for the former ``iterrows`` loop; identical + results. Rows with ``init_flows < 0`` get their origin/extremity swapped, + their ``init_flows`` made positive, and ``swapped=True``. + """ + init = df["init_flows"].to_numpy(dtype=float) + swap_mask = (init < 0) & (init != 0.0) + + idx_or = df["idx_or"].to_numpy() + idx_ex = df["idx_ex"].to_numpy() + df["idx_or"] = np.where(swap_mask, idx_ex, idx_or) + df["idx_ex"] = np.where(swap_mask, idx_or, idx_ex) + df["init_flows"] = np.where(swap_mask, np.abs(init), init) + df["swapped"] = swap_mask @staticmethod def invert_dict_keys_values(d: Dict[Any, Any]) -> Dict[Any, Any]: diff --git a/alphaDeesp/tests/test_alphadeesp_combinations.py b/alphaDeesp/tests/test_alphadeesp_combinations.py new file mode 100644 index 00000000..06a6e6b8 --- /dev/null +++ b/alphaDeesp/tests/test_alphadeesp_combinations.py @@ -0,0 +1,82 @@ +"""Unit tests for the pure combinatorial / result-shaping helpers of +:class:`alphaDeesp.core.alphadeesp.AlphaDeesp` — busbar-combination +enumeration, legality filtering, best-topology cleanup and constrained-path +flattening. These do not need the full ranking pipeline (no grid2op).""" + +import pandas as pd +import pytest + +from alphaDeesp.core.alphadeesp import AlphaDeesp +from alphaDeesp.core.elements import OriginLine + + +class _Host: + compute_all_combinations = AlphaDeesp.compute_all_combinations + legal_comb = AlphaDeesp.legal_comb + clean_and_sort_best_topologies = AlphaDeesp.clean_and_sort_best_topologies + filter_constrained_path = AlphaDeesp.filter_constrained_path + + def __init__(self, elements=None): + self.simulator_data = {"substations_elements": {5: elements or []}} + + +def _lines(n): + return [OriginLine(busbar_id=0, end_substation_id=i + 1, flow_value=[1.0]) for i in range(n)] + + +class TestComputeAllCombinations: + def test_two_elements_returns_the_two_single_bus_configs(self): + host = _Host(_lines(2)) + assert host.compute_all_combinations(5) == [(1, 1), (0, 0)] + + def test_one_element_raises(self): + host = _Host(_lines(1)) + with pytest.raises(ValueError): + host.compute_all_combinations(5) + + def test_five_elements_are_all_legal_and_start_at_bus_zero(self): + host = _Host(_lines(5)) + combos = host.compute_all_combinations(5) + assert combos, "expected some legal combinations" + for c in combos: + assert c[0] == 0 # canonical: first element on bus 0 + assert not (all(x == 0 for x in c)) # not the trivial single-bus configs + assert not (all(x == 1 for x in c)) + assert sum(c) not in (1, len(c) - 1) # no isolated single element + + +class TestLegalComb: + def test_rejects_comb_not_starting_at_zero(self): + assert _Host().legal_comb([1, 0, 0], 0, 3, [0, 0, 0], [1, 1, 1]) is False + + def test_rejects_the_reference_and_symmetric_configs(self): + assert _Host().legal_comb([0, 0, 1], 0, 3, [0, 0, 1], [1, 1, 0]) is False # == config + + def test_rejects_single_element_split(self): + # sum == 1 (or n-1) means one element alone on a busbar + assert _Host().legal_comb([0, 1, 0, 0, 0], 0, 5, [0, 0, 0, 0, 0], [1, 1, 1, 1, 1]) is False + + def test_accepts_a_balanced_split(self): + assert _Host().legal_comb([0, 0, 1, 1, 0], 0, 5, [0, 0, 0, 0, 0], [1, 1, 1, 1, 1]) is True + + +class TestCleanAndSortBestTopologies: + def test_drops_sentinel_and_sorts_by_score_desc(self): + host = _Host() + df = pd.DataFrame({ + "score": ["XX", 1, 3, 2], + "topology": [["X"], [0, 1], [1, 0], [1, 1]], + "node": ["X", 5, 5, 5], + }) + result = host.clean_and_sort_best_topologies(df) + assert list(result.index) == [3, 2, 1] # sentinel "XX" dropped, sorted desc + + +class TestFilterConstrainedPath: + def test_flattens_edge_pairs_to_unique_ordered_nodes(self): + host = _Host() + assert host.filter_constrained_path([("A", "B"), ("B", "C")]) == ["A", "B", "C"] + + def test_flattens_nested_tuple_edges(self): + host = _Host() + assert host.filter_constrained_path([(("A", "B"), ("C", "D"))]) == ["A", "B", "C", "D"] diff --git a/alphaDeesp/tests/test_alphadeesp_unit.py b/alphaDeesp/tests/test_alphadeesp_unit.py index 685c8c20..e8d8ff9e 100644 --- a/alphaDeesp/tests/test_alphadeesp_unit.py +++ b/alphaDeesp/tests/test_alphadeesp_unit.py @@ -453,6 +453,112 @@ def test_no_matching_row_returns_zero(self): assert AlphaDeesp._initial_inflow_between(df, source=99, target=100) == 0.0 +class TestBuildInflowLookup: + """``_build_inflow_lookup`` precomputes the same values the linear + ``_initial_inflow_between`` scan returns, for every orientation.""" + + def test_lookup_matches_linear_scan_for_all_orientations(self): + import pandas as pd + df = pd.DataFrame({ + "idx_or": [3, 5, 4], + "idx_ex": [5, 3, 4], + "init_flows": [12.0, -7.0, 0.0], + }) + lookup = AlphaDeesp._build_inflow_lookup(df) + nodes = [3, 4, 5, 99] + for source in nodes: + for target in nodes: + assert lookup.get((source, target), 0.0) == \ + AlphaDeesp._initial_inflow_between(df, source, target) + + def test_first_matching_row_wins(self): + import pandas as pd + # two rows both mapping (3 -> 5): the first (12.0) must win, exactly + # like the first-match semantics of the linear scan. + df = pd.DataFrame({ + "idx_or": [3, 3], + "idx_ex": [5, 5], + "init_flows": [12.0, 4.0], + }) + lookup = AlphaDeesp._build_inflow_lookup(df) + assert lookup[(3, 5)] == 12.0 + assert lookup[(3, 5)] == AlphaDeesp._initial_inflow_between(df, 3, 5) + + +class TestAutoRunSeparation: + """``auto_run=False`` builds the object without running the ranking + pipeline; ``run()`` populates the results. A trivial graph (no black + constrained edge) would make the pipeline raise, which lets us prove the + constructor did NOT run it.""" + + @staticmethod + def _trivial(): + import pandas as pd + g = nx.MultiDiGraph() + g.add_edge(0, 1, color="gray", capacity=0.0, name="l0") + df = pd.DataFrame({"idx_or": [0], "idx_ex": [1], "init_flows": [0.0]}) + return g, df + + def test_no_pipeline_when_auto_run_false(self): + g, df = self._trivial() + # Would raise inside the pipeline (no constrained edge); must NOT raise. + ad = AlphaDeesp(g, df, simulator_data={"substations_elements": {}}, + auto_run=False) + assert ad.g_distribution_graph is None + assert ad.get_ranked_combinations() == [] + assert ad.rankedLoopBuses == {} + # state that the constructor still sets + assert ad.initial_graph.number_of_edges() == 1 + + def test_run_is_invoked_by_default(self): + # A real pipeline needs a valid overflow graph; here we only assert that + # auto_run defaults to True by observing the pipeline is attempted + # (raises on the trivial graph) — i.e. the default path calls run(). + g, df = self._trivial() + import pytest + with pytest.raises(Exception): + AlphaDeesp(g, df, simulator_data={"substations_elements": {}}) + + +class _SortHubsHost: + """Expose the vectorised ``sort_hubs`` on a bare object.""" + sort_hubs = AlphaDeesp.sort_hubs + + def __init__(self, df): + self.df = df + + +class TestSortHubs: + """``sort_hubs`` ranks nodes by the larger of their total absolute + incident delta-flow entering (``idx_ex``) or leaving (``idx_or``).""" + + def test_ranks_by_max_incident_abs_delta(self): + import pandas as pd + df = pd.DataFrame({ + "idx_or": [0, 1, 2], + "idx_ex": [1, 2, 0], + "delta_flows": [10.0, -30.0, 5.0], + }) + host = _SortHubsHost(df) + res = host.sort_hubs([0, 1, 2]) + strength = dict(zip(res["hubs"], res["max_flows"])) + # node 0: out={|10|}=10, in={|5|}=5 -> 10 + # node 1: out={|-30|}=30, in={|10|}=10 -> 30 + # node 2: out={|5|}=5, in={|-30|}=30 -> 30 + assert strength[0] == 10.0 + assert strength[1] == 30.0 + assert strength[2] == 30.0 + # sorted descending: node 0 (weakest) is last. + assert list(res["hubs"])[-1] == 0 + + def test_none_when_no_hubs(self): + import pandas as pd + host = _SortHubsHost(pd.DataFrame( + {"idx_or": [], "idx_ex": [], "delta_flows": []})) + assert host.sort_hubs([]) is None + assert host.sort_hubs(None) is None + + class TestBusLoopStrength: """``_bus_loop_strength`` = ``(non_red_inflow + local_production) * red_delta_inflow``.""" @@ -494,6 +600,26 @@ def test_zero_when_no_red_inflow(self): assert host._bus_loop_strength( 5, df_init, color_attrs, label_attrs) == 0.0 + def test_inflow_lookup_path_matches_linear_fallback(self): + import pandas as pd + g = nx.MultiDiGraph() + g.add_edge(10, 5, label="3", color="coral") + g.add_edge(4, 5, label="0", color="gray") + sim_data = {"substations_elements": { + 5: [Production(busbar_id=0, value=2.0)] + }} + host = _LoopBusHost(sim_data, g) + df_init = pd.DataFrame({ + "idx_or": [4], "idx_ex": [5], "init_flows": [6.0], + }) + color_attrs = nx.get_edge_attributes(g, "color") + label_attrs = nx.get_edge_attributes(g, "label") + lookup = AlphaDeesp._build_inflow_lookup(df_init) + # fast path (precomputed lookup) must equal the linear fallback. + assert host._bus_loop_strength( + 5, df_init, color_attrs, label_attrs, inflow_lookup=lookup) == \ + host._bus_loop_strength(5, df_init, color_attrs, label_attrs) + # ────────────────────────────────────────────────────────────────────── # Tests for to_DiGraph (MultiDiGraph -> weighted DiGraph conversion) @@ -518,11 +644,20 @@ def test_sums_parallel_capacities(self): assert isinstance(result, nx.DiGraph) assert result["A"]["B"]["capacity"] == 5.0 - def test_defaults_missing_capacity_to_one(self): + def test_defaults_missing_capacity_to_zero(self): + # A genuinely missing capacity must contribute 0 to the min-cut used by + # rank_red_loops, not a spurious unit weight. g = nx.MultiDiGraph() g.add_edge("A", "B") # no capacity result = self._call(g) - assert result["A"]["B"]["capacity"] == 1.0 + assert result["A"]["B"]["capacity"] == 0.0 + + def test_missing_and_present_capacity_sum(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", capacity=4.0) + g.add_edge("A", "B") # missing -> contributes 0 + result = self._call(g) + assert result["A"]["B"]["capacity"] == 4.0 def test_preserves_distinct_edges(self): g = nx.MultiDiGraph() diff --git a/alphaDeesp/tests/test_edge_roles.py b/alphaDeesp/tests/test_edge_roles.py new file mode 100644 index 00000000..2947fe83 --- /dev/null +++ b/alphaDeesp/tests/test_edge_roles.py @@ -0,0 +1,53 @@ +"""Unit tests for :mod:`alphaDeesp.core.graphs.edge_roles`. + +The role accessor is the single authority that maps an edge's base colour to a +semantic role, so consumers never parse Graphviz colour strings. +""" + +from alphaDeesp.core.graphs.edge_roles import ( + EDGE_ROLE_INSIGNIFICANT, + EDGE_ROLE_NEGATIVE, + EDGE_ROLE_NULL_NON_RECONNECTABLE, + EDGE_ROLE_OVERLOAD, + EDGE_ROLE_POSITIVE, + EDGE_ROLE_UNKNOWN, + base_color_of, + edge_role_of, +) +# also reachable via both public surfaces +from alphaDeesp.core.graphs import edge_role_of as _from_pkg +from alphaDeesp.core.graphsAndPaths import edge_role_of as _from_shim + + +class TestPublicSurface: + def test_exported_from_package_and_shim(self): + assert _from_pkg is edge_role_of + assert _from_shim is edge_role_of + + +class TestEdgeRoleOf: + def test_clean_base_colours_map_to_roles(self): + assert edge_role_of({"color": "black"}) == EDGE_ROLE_OVERLOAD + assert edge_role_of({"color": "blue"}) == EDGE_ROLE_NEGATIVE + assert edge_role_of({"color": "coral"}) == EDGE_ROLE_POSITIVE + assert edge_role_of({"color": "gray"}) == EDGE_ROLE_INSIGNIFICANT + assert edge_role_of({"color": "dimgray"}) == EDGE_ROLE_NULL_NON_RECONNECTABLE + + def test_unknown_colour_is_unknown(self): + assert edge_role_of({"color": "chartreuse"}) == EDGE_ROLE_UNKNOWN + assert edge_role_of({}) == EDGE_ROLE_UNKNOWN + + def test_compound_colour_is_stripped(self): + # After highlight the rendered colour is a compound string. + assert edge_role_of({"color": '"coral:yellow:coral"'}) == EDGE_ROLE_POSITIVE + assert edge_role_of({"color": '"black:yellow:black"'}) == EDGE_ROLE_OVERLOAD + + def test_base_color_attribute_is_authoritative(self): + # base_color wins over a (compound) rendered color. + data = {"color": '"coral:yellow:coral"', "base_color": "blue"} + assert base_color_of(data) == "blue" + assert edge_role_of(data) == EDGE_ROLE_NEGATIVE + + def test_non_string_colour_is_safe(self): + assert base_color_of({"color": 123}) == "" + assert edge_role_of({"color": None}) == EDGE_ROLE_UNKNOWN diff --git a/alphaDeesp/tests/test_elements.py b/alphaDeesp/tests/test_elements.py new file mode 100644 index 00000000..5570a38a --- /dev/null +++ b/alphaDeesp/tests/test_elements.py @@ -0,0 +1,70 @@ +"""Unit tests for the substation element model +(:mod:`alphaDeesp.core.elements`): Production, Consumption, OriginLine, +ExtremityLine — the value objects the ``Simulation`` backends emit and that +``AlphaDeesp`` consumes to enumerate busbar configurations.""" + +from alphaDeesp.core.elements import ( + Consumption, + ExtremityLine, + OriginLine, + Production, +) + + +class TestProduction: + def test_fields_and_busbar_property(self): + p = Production(busbar_id=0, value=3.5) + assert p.busbar_id == 0 + assert p.busbar == 0 + assert p.value == 3.5 + + def test_busbar_setter(self): + p = Production(busbar_id=0) + p.busbar = 1 + assert p.busbar_id == 1 + + def test_ids_increment_monotonically(self): + a = Production(0) + b = Production(0) + assert b.ID == a.ID + 1 + + def test_repr_mentions_type_and_value(self): + r = repr(Production(0, 3.5)) + assert "PRODUCTION" in r and "3.5" in r + + +class TestConsumption: + def test_fields_and_busbar_setter(self): + c = Consumption(busbar_id=1, value=4.0) + assert c.busbar == 1 and c.value == 4.0 + c.busbar = 0 + assert c.busbar_id == 0 + + def test_repr_mentions_type(self): + assert "CONSUMPTION" in repr(Consumption(0, 4.0)) + + +class TestOriginLine: + def test_fields(self): + line = OriginLine(busbar_id=0, end_substation_id=5, flow_value=[42.0]) + assert line.busbar == 0 + assert line.end_substation_id == 5 + assert line.flow_value == [42.0] + + def test_busbar_setter_and_repr(self): + line = OriginLine(0, end_substation_id=5, flow_value=[1.0]) + line.busbar = 1 + assert line.busbar_id == 1 + assert "ORIGINLINE" in repr(line) + + +class TestExtremityLine: + def test_fields(self): + line = ExtremityLine(busbar_id=1, start_substation_id=3, flow_value=[-7.0]) + assert line.busbar == 1 + assert line.start_substation_id == 3 + assert line.flow_value == [-7.0] + + def test_repr_mentions_type(self): + assert "EXTREMITYLINE" in repr( + ExtremityLine(0, start_substation_id=3, flow_value=[1.0])) diff --git a/alphaDeesp/tests/test_graph_consolidation.py b/alphaDeesp/tests/test_graph_consolidation.py new file mode 100644 index 00000000..f5582031 --- /dev/null +++ b/alphaDeesp/tests/test_graph_consolidation.py @@ -0,0 +1,179 @@ +"""Unit tests for :class:`GraphConsolidationMixin`. + +These consolidation helpers (edge reversal, loop-path recolouring, ambiguity +classification) were previously exercised only by the grid2op integration +suite. Here they run on small hand-built coloured graphs, no grid2op needed. +""" + +import networkx as nx + +from alphaDeesp.core.graphs.graph_consolidation import GraphConsolidationMixin +from alphaDeesp.core.graphs.constrained_path import ConstrainedPath + + +class _ConsolidationHost(GraphConsolidationMixin): + """Minimal host exposing the mixin with the attributes it assumes.""" + + def __init__(self, g, float_precision="%.2f"): + self.g = g + self.float_precision = float_precision + + +def _edge(g, name): + for u, v, k, d in g.edges(keys=True, data=True): + if d.get("name") == name: + return (u, v, k), d + raise AssertionError(f"edge {name!r} not found") + + +# ────────────────────────────────────────────────────────────────────── +# _is_ambiguous_component +# ────────────────────────────────────────────────────────────────────── + +class TestIsAmbiguousComponent: + def _comp(self, *edges): + g = nx.MultiDiGraph() + for u, v, color in edges: + g.add_edge(u, v, color=color) + return g, set(g.nodes) + + def test_blue_and_coral_two_colours_is_ambiguous(self): + g, comp = self._comp(("A", "B", "blue"), ("B", "A", "coral")) + assert GraphConsolidationMixin._is_ambiguous_component(g, comp) is True + + def test_single_node_is_not_ambiguous(self): + g = nx.MultiDiGraph() + g.add_node("A") + assert GraphConsolidationMixin._is_ambiguous_component(g, {"A"}) is False + + def test_three_colours_is_not_ambiguous(self): + g, comp = self._comp(("A", "B", "blue"), ("B", "C", "coral"), ("C", "A", "black")) + assert GraphConsolidationMixin._is_ambiguous_component(g, comp) is False + + def test_single_colour_is_not_ambiguous(self): + g, comp = self._comp(("A", "B", "blue"), ("B", "C", "blue")) + assert GraphConsolidationMixin._is_ambiguous_component(g, comp) is False + + +# ────────────────────────────────────────────────────────────────────── +# reverse_edges +# ────────────────────────────────────────────────────────────────────── + +class TestReverseEdges: + def test_reverses_direction_flips_capacity_and_recolours(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", name="L1", color="blue", capacity=-5.0, label="-5.00") + host = _ConsolidationHost(g) + host.reverse_edges(["L1"], target_color="coral") + + # original A->B is gone; reversed B->A exists + assert not host.g.has_edge("A", "B") + (key, data) = _edge(host.g, "L1") + assert key[:2] == ("B", "A") + assert data["color"] == "coral" + assert data["capacity"] == 5.0 + assert data["label"] == "5.00" + + def test_edge_already_target_colour_is_not_flipped(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", name="L1", color="coral", capacity=3.0, label="3.00") + host = _ConsolidationHost(g) + host.reverse_edges(["L1"], target_color="coral") + # same colour -> stays A->B, capacity unchanged (no reversal) + assert host.g.has_edge("A", "B") + _, data = _edge(host.g, "L1") + assert data["capacity"] == 3.0 + + +# ────────────────────────────────────────────────────────────────────── +# consolidate_loop_path +# ────────────────────────────────────────────────────────────────────── + +class TestConsolidateLoopPath: + def test_gray_edges_on_hub_path_become_coral(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", name="ab", color="gray", capacity=5.0) + g.add_edge("B", "C", name="bc", color="gray", capacity=5.0) + host = _ConsolidationHost(g) + host.consolidate_loop_path(["A"], ["C"]) + assert _edge(host.g, "ab")[1]["color"] == "coral" + assert _edge(host.g, "bc")[1]["color"] == "coral" + + def test_null_capacity_edges_are_ignored(self): + g = nx.MultiDiGraph() + # zero-capacity gray edges are dropped from the search graph + g.add_edge("A", "B", name="ab", color="gray", capacity=0.0) + g.add_edge("B", "C", name="bc", color="gray", capacity=0.0) + host = _ConsolidationHost(g) + host.consolidate_loop_path(["A"], ["C"]) + assert _edge(host.g, "ab")[1]["color"] == "gray" + assert _edge(host.g, "bc")[1]["color"] == "gray" + + def test_non_gray_edges_on_path_are_left_alone(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", name="ab", color="black", capacity=5.0) + g.add_edge("B", "C", name="bc", color="gray", capacity=5.0) + host = _ConsolidationHost(g) + host.consolidate_loop_path(["A"], ["C"]) + assert _edge(host.g, "ab")[1]["color"] == "black" # unchanged + assert _edge(host.g, "bc")[1]["color"] == "coral" # gray -> coral + + +# ────────────────────────────────────────────────────────────────────── +# reverse_blue_edges_in_looppaths +# ────────────────────────────────────────────────────────────────────── + +class TestReverseBlueEdgesInLoopPaths: + def test_blue_edge_off_constrained_path_is_reversed_to_coral(self): + g = nx.MultiDiGraph() + g.add_edge("X", "Y", name="xy", color="blue", capacity=-5.0, label="-5.00") + host = _ConsolidationHost(g) + host.reverse_blue_edges_in_looppaths(constrained_path=[]) + assert not host.g.has_edge("X", "Y") + (key, data) = _edge(host.g, "xy") + assert key[:2] == ("Y", "X") + assert data["color"] == "coral" + assert data["capacity"] == 5.0 + + def test_edge_on_constrained_path_is_untouched(self): + g = nx.MultiDiGraph() + g.add_edge("X", "Y", name="xy", color="blue", capacity=-5.0, label="-5.00") + host = _ConsolidationHost(g) + # X on the constrained path -> its incident blue edge is excluded + host.reverse_blue_edges_in_looppaths(constrained_path=["X"]) + assert host.g.has_edge("X", "Y") + _, data = _edge(host.g, "xy") + assert data["color"] == "blue" + + +# ────────────────────────────────────────────────────────────────────── +# desambiguation_type_path +# ────────────────────────────────────────────────────────────────────── + +class _FakeStructured: + def __init__(self, constrained_path): + self.constrained_path = constrained_path + + +class TestDesambiguationTypePath: + def _structured(self): + cp = ConstrainedPath( + amont_edges=[("A", "B", 0)], + constrained_edge=("B", "C", 0), + aval_edges=[("C", "D", 0)], + ) + return _FakeStructured(cp) + + def test_fewer_than_two_cpath_nodes_is_loop_path(self): + host = _ConsolidationHost(nx.MultiDiGraph()) + assert host.desambiguation_type_path(["A"], self._structured()) == "loop_path" + + def test_amont_only_is_constrained_path(self): + host = _ConsolidationHost(nx.MultiDiGraph()) + # A and B are both amont -> connects amont but not aval + assert host.desambiguation_type_path(["A", "B"], self._structured()) == "constrained_path" + + def test_bridging_amont_and_aval_is_loop_path(self): + host = _ConsolidationHost(nx.MultiDiGraph()) + # B (amont) and C (aval) -> bridges both sides + assert host.desambiguation_type_path(["B", "C"], self._structured()) == "loop_path" diff --git a/alphaDeesp/tests/test_graphs_package.py b/alphaDeesp/tests/test_graphs_package.py index 0e6ad0ff..a318a87f 100644 --- a/alphaDeesp/tests/test_graphs_package.py +++ b/alphaDeesp/tests/test_graphs_package.py @@ -35,14 +35,23 @@ # Structural invariants of the refactor # --------------------------------------------------------------------------- -# The 16 public names the refactor commits to exporting, in the same order +# The public names the refactor commits to exporting, in the same order # as ``alphaDeesp/core/graphs/__init__.py::__all__``. EXPECTED_PUBLIC_NAMES = frozenset({ "default_voltage_colors", "PowerFlowGraph", "OverFlowGraph", + "OverflowGraphRenderer", "ConstrainedPath", "Structured_Overload_Distribution_Graph", + "edge_role_of", + "base_color_of", + "EDGE_ROLE_OVERLOAD", + "EDGE_ROLE_NEGATIVE", + "EDGE_ROLE_POSITIVE", + "EDGE_ROLE_INSIGNIFICANT", + "EDGE_ROLE_NULL_NON_RECONNECTABLE", + "EDGE_ROLE_UNKNOWN", "from_edges_get_nodes", "delete_color_edges", "nodepath_to_edgepath", @@ -62,9 +71,18 @@ "default_voltage_colors": "alphaDeesp.core.graphs.constants", "PowerFlowGraph": "alphaDeesp.core.graphs.power_flow_graph", "OverFlowGraph": "alphaDeesp.core.graphs.overflow_graph", + "OverflowGraphRenderer": "alphaDeesp.core.graphs.overflow_renderer", "ConstrainedPath": "alphaDeesp.core.graphs.constrained_path", "Structured_Overload_Distribution_Graph": "alphaDeesp.core.graphs.structured_overload_graph", + "edge_role_of": "alphaDeesp.core.graphs.edge_roles", + "base_color_of": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_OVERLOAD": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_NEGATIVE": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_POSITIVE": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_INSIGNIFICANT": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_NULL_NON_RECONNECTABLE": "alphaDeesp.core.graphs.edge_roles", + "EDGE_ROLE_UNKNOWN": "alphaDeesp.core.graphs.edge_roles", "from_edges_get_nodes": "alphaDeesp.core.graphs.graph_utils", "delete_color_edges": "alphaDeesp.core.graphs.graph_utils", "nodepath_to_edgepath": "alphaDeesp.core.graphs.graph_utils", @@ -80,6 +98,7 @@ EXPECTED_SUBMODULES = frozenset({ "constants", + "edge_roles", "graph_utils", "null_flow", "shortest_paths", @@ -87,6 +106,7 @@ "constrained_path", "structured_overload_graph", "overflow_graph", + "overflow_renderer", }) @@ -322,6 +342,75 @@ def test_get_dispatch_edges_nodes_returns_loop_path_members(self): assert set(lines) == {"line_AX", "line_XD"} assert set(nodes) == {"A", "X", "D"} + def test_derived_colour_views_are_cached(self): + """Lazy cached properties return the same object on repeated access.""" + sg = Structured_Overload_Distribution_Graph(_make_structured_overload_input()) + assert sg.g_only_red_components is sg.g_only_red_components + assert sg.g_only_blue_components is sg.g_only_blue_components + assert sg.g_without_constrained_edge is sg.g_without_constrained_edge + + def test_lazy_results_are_order_independent(self): + """Accessing hubs before loops (or vice-versa) yields identical results — + the ``red_loops`` cache is keyed on the seed hubs, not the detected ones.""" + sg_a = Structured_Overload_Distribution_Graph(_make_structured_overload_input()) + loops_first = [list(p) for p in sg_a.red_loops["Path"].tolist()] + hubs_after = set(sg_a.get_hubs()) + + sg_b = Structured_Overload_Distribution_Graph(_make_structured_overload_input()) + hubs_first = set(sg_b.get_hubs()) + loops_after = [list(p) for p in sg_b.red_loops["Path"].tolist()] + + assert loops_first == loops_after + assert hubs_after == hubs_first == {"A", "D"} + + def test_get_loops_returns_the_cached_seed_red_loops(self): + sg = Structured_Overload_Distribution_Graph(_make_structured_overload_input()) + assert sg.get_loops() is sg.red_loops + + def test_lazy_views_are_frozen_to_construction_snapshot(self): + """The lazy views must reflect the graph *as it was at construction*, + even if the caller mutates the shared graph before the views are first + accessed (consolidate_graph removes ignored lines from the same graph + this object was built on). Matches the OLD eager construction-time copy. + """ + g = _make_structured_overload_input() + sg = Structured_Overload_Distribution_Graph(g) + # Mutate the caller's graph AFTER construction, BEFORE any view access. + g.remove_edge("A", "X") # drop a coral loop edge (line_AX) + # Views/loops must still see the construction-time coral loop A->X->D. + assert "X" in set(sg.g_only_red_components.nodes) + loops = [list(p) for p in sg.get_loops()["Path"].tolist()] + assert ["A", "X", "D"] in loops + + +def _make_loopless_overload_input(): + """Constrained path with NO coral loop path -> ``red_loops`` is empty. + + A --blue--> B --black--> C --blue--> D (no coral bypass) + """ + g = nx.MultiDiGraph() + g.add_edge("A", "B", color="blue", capacity=-5.0, name="line_AB") + g.add_edge("B", "C", color="black", capacity=-10.0, name="line_BC", + constrained=True) + g.add_edge("C", "D", color="blue", capacity=-5.0, name="line_CD") + return g + + +class TestStructuredOverloadDistributionGraphNoLoops: + """Regression: an overflow graph with no loop path must not crash + ``get_dispatch_edges_nodes`` (empty ``Path`` column's ``.sum()`` used to + return the scalar 0, breaking ``set(0)``).""" + + def test_loops_dataframe_is_empty(self): + sg = Structured_Overload_Distribution_Graph(_make_loopless_overload_input()) + assert sg.get_loops().empty + + def test_get_dispatch_edges_nodes_returns_empty_without_crashing(self): + sg = Structured_Overload_Distribution_Graph(_make_loopless_overload_input()) + lines, nodes = sg.get_dispatch_edges_nodes(only_loop_paths=True) + assert lines == [] + assert nodes == [] + # --------------------------------------------------------------------------- # Behavioural coverage — shortest_path_mandatory_and_promoted diff --git a/alphaDeesp/tests/test_interactive_html.py b/alphaDeesp/tests/test_interactive_html.py index 5c8c768e..48673b7a 100644 --- a/alphaDeesp/tests/test_interactive_html.py +++ b/alphaDeesp/tests/test_interactive_html.py @@ -495,3 +495,28 @@ def test_html_embeds_section_field_and_inserts_section_headers(): assert "Structural Paths" in sections assert "Individual entities properties" in sections assert "Flow redispatch values" in sections + + +# ────────────────────────────────────────────────────────────────────── +# Externalised assets: CSS/JS/HTML skeleton now live under assets/ and are +# reassembled at runtime by template.html_template(). +# ────────────────────────────────────────────────────────────────────── + +class TestExternalisedAssets: + def test_asset_files_exist(self): + from pathlib import Path + import alphaDeesp.core.interactive_html as ih_pkg + assets = Path(ih_pkg.__file__).parent / "assets" + for name in ("viewer.css", "viewer.js", "template.html"): + assert (assets / name).is_file(), f"missing asset {name}" + + def test_template_reconstitution_inlines_css_and_js(self): + from alphaDeesp.core.interactive_html.template import html_template + tpl = html_template() + # placeholders for per-render substitution survive… + assert "__TITLE__" in tpl and "__SVG__" in tpl and "__MODEL_JSON__" in tpl + # …and the externalised CSS + JS have been inlined back in. + assert ":root {" in tpl # from viewer.css + assert "const MODEL = __MODEL_JSON__;" in tpl # from viewer.js + # the split markers must be fully consumed + assert "__CSS__" not in tpl and "__JS__" not in tpl diff --git a/alphaDeesp/tests/test_null_flow_weighting.py b/alphaDeesp/tests/test_null_flow_weighting.py new file mode 100644 index 00000000..c4c57d83 --- /dev/null +++ b/alphaDeesp/tests/test_null_flow_weighting.py @@ -0,0 +1,94 @@ +"""Tests for the null-flow path-search routing weight (issue #1). + +``_compute_sssp_paths`` now precomputes an edge-weight attribute and runs +Dijkstra with a string weight (perf), with two modes: + +* ``capacity_weighted=False`` (default) — "bless": bit-identical to the old + per-edge callable, which on a ``MultiDiGraph`` read capacity as ``0`` and never + matched the ``(u, v, key)`` promoted set, i.e. a uniform hop weight. +* ``capacity_weighted=True`` — "fix": capacity-weighted routing. +""" + +import networkx as nx + +from alphaDeesp.tests.graphs_test_helpers import DetectEdgesHelperHost + + +def _prepared_single_source(source): + """Minimal ``prepared`` dict that lets ``_compute_sssp_paths`` run ``source``.""" + return { + "source_nodes_in_gc": [source], + "bfs_cache": {source: True}, + "targets_with_bfs": frozenset(), + "node_has_incident_interest": {source: True}, + "any_target_has_interest": True, + } + + +def _issue_repro_graph(): + # From issue #1: a heavy direct A->C vs a light A->B->C. + g = nx.MultiDiGraph() + g.add_edge("A", "B", capacity=1.0, name="ab") + g.add_edge("B", "C", capacity=2.0, name="bc") + g.add_edge("A", "C", capacity=10.0, name="ac") + return g + + +def _old_callable_sssp(g, source, edges_of_interest): + """Faithful re-implementation of the pre-refactor per-edge callable weight.""" + promoted_set = set(edges_of_interest) + + def w(u, v, attr): + real_weight = attr.get("capacity", 0) # dict-of-keys on a multigraph -> 0 + return real_weight * 1_000_000_000 + (33 if (u, v) in promoted_set else 100) + + return nx.single_source_dijkstra_path(g, source, weight=w) + + +class TestNullFlowRoutingModes: + + def test_option_A_default_is_hop_only(self): + obj = DetectEdgesHelperHost() + g = _issue_repro_graph() + res = obj._compute_sssp_paths(g, _prepared_single_source("A"), set()) + # hop-only: the 1-hop direct edge wins even though it is "heavier". + assert res["A"]["C"] == ["A", "C"] + + def test_option_B_is_capacity_weighted(self): + obj = DetectEdgesHelperHost() + g = _issue_repro_graph() + res = obj._compute_sssp_paths( + g, _prepared_single_source("A"), set(), capacity_weighted=True) + # capacity 3 (A->B->C) beats 10 (A->C) despite the extra hop. + assert res["A"]["C"] == ["A", "B", "C"] + + def test_option_A_is_bit_identical_to_old_callable(self): + # Multigraph with parallel edges and a promoted (u, v, key) set — the + # exact shape the old callable mishandled. New Option A must match it. + g = nx.MultiDiGraph() + g.add_edge("S", "M", capacity=0.0, name="sm") + g.add_edge("S", "M", capacity=5.0, name="sm2") # parallel edge + g.add_edge("M", "T", capacity=0.0, name="mt") + g.add_edge("S", "X", capacity=0.0, name="sx") + g.add_edge("X", "T", capacity=0.0, name="xt") + edges_of_interest = {("M", "T", 0)} # a (u, v, key) triple + + obj = DetectEdgesHelperHost() + new_a = obj._compute_sssp_paths( + g.copy(), _prepared_single_source("S"), edges_of_interest) + old = _old_callable_sssp(g.copy(), "S", edges_of_interest) + assert new_a["S"] == old + + def test_option_B_promoted_key_tuple_is_matched(self): + # Two equal-capacity routes S->A->T and S->B->T; promote the B route by + # its exact (u, v, key). Option B must prefer it via the lower hop cost. + g = nx.MultiDiGraph() + g.add_edge("S", "A", capacity=1.0, name="sa") + g.add_edge("A", "T", capacity=1.0, name="at") + kb1 = g.add_edge("S", "B", capacity=1.0, name="sb") + kb2 = g.add_edge("B", "T", capacity=1.0, name="bt") + promoted = {("S", "B", kb1), ("B", "T", kb2)} + obj = DetectEdgesHelperHost() + res = obj._compute_sssp_paths( + g, _prepared_single_source("S"), promoted, capacity_weighted=True) + assert res["S"]["T"] == ["S", "B", "T"] diff --git a/alphaDeesp/tests/test_overflow_graph.py b/alphaDeesp/tests/test_overflow_graph.py index 66ac2f31..e985fc87 100644 --- a/alphaDeesp/tests/test_overflow_graph.py +++ b/alphaDeesp/tests/test_overflow_graph.py @@ -944,3 +944,114 @@ def test_black_and_blue_edges_with_matching_name_are_tagged(self): ofg.tag_constrained_path(lines_constrained_path=["L1", "L2"]) for _, _, _, data in ofg.g.edges(keys=True, data=True): assert data.get("on_constrained_path") is True + + +# ────────────────────────────────────────────────────────────────────── +# rename_nodes: relabels the graph AND both endpoint columns of the df. +# Contract lock: ``idx_ex`` is remapped from the ``idx_ex`` column (the +# original code did this correctly but via a misleadingly-named loop +# variable; this pins the behaviour so a future edit can't regress it). +# ────────────────────────────────────────────────────────────────────── + + +def _rename_df(): + return pd.DataFrame({ + "idx_or": [0, 1], + "idx_ex": [1, 2], + "delta_flows": [10.0, -5.0], + "gray_edges": [False, False], + "line_name": ["L1", "L2"], + }) + + +class TestRenameNodes: + + def test_rename_updates_graph_and_both_df_columns(self): + ofg = OverFlowGraph(_basic_topo(3), [], _rename_df()) + ofg.rename_nodes({0: "A", 1: "B", 2: "C"}) + + assert set(ofg.g.nodes) == {"A", "B", "C"} + # idx_or maps 0,1 -> A,B; idx_ex maps 1,2 -> B,C (NOT A,B). + assert list(ofg.df["idx_or"]) == ["A", "B"] + assert list(ofg.df["idx_ex"]) == ["B", "C"] + + +class TestDoesNotMutateCallerDataFrame: + """OverFlowGraph must operate on a copy of the caller's DataFrame.""" + + def test_line_name_column_not_added_to_caller_df(self): + df = pd.DataFrame({ + "idx_or": [0, 1], "idx_ex": [1, 2], + "delta_flows": [10.0, -5.0], "gray_edges": [False, False], + }) + cols_before = list(df.columns) + ofg = OverFlowGraph(_basic_topo(3), [], df) + assert "line_name" not in df.columns + assert list(df.columns) == cols_before + # the internal copy still carries the generated column + assert "line_name" in ofg.df.columns + + def test_rename_nodes_does_not_touch_caller_df(self): + df = _rename_df() + ofg = OverFlowGraph(_basic_topo(3), [], df) + ofg.rename_nodes({0: "A", 1: "B", 2: "C"}) + assert list(df["idx_or"]) == [0, 1] + assert list(df["idx_ex"]) == [1, 2] + + +# ────────────────────────────────────────────────────────────────────── +# edge_role accessor + base_color authority (model/colour inversion). +# ────────────────────────────────────────────────────────────────────── + +from alphaDeesp.core.graphs.edge_roles import ( # noqa: E402 + EDGE_ROLE_OVERLOAD, + EDGE_ROLE_POSITIVE, +) + + +class TestEdgeRoleAccessor: + + def test_edge_role_by_name(self): + df = pd.DataFrame({ + "idx_or": [0, 1, 2], + "idx_ex": [1, 2, 0], + "delta_flows": [1000.0, -100.0, 50.0], + "gray_edges": [False, False, False], + "line_name": ["L1", "L2", "L3"], + }) + ofg = OverFlowGraph(_basic_topo(3), [0], df) # L1 cut -> overload (black) + assert ofg.edge_role("L1") == EDGE_ROLE_OVERLOAD + assert ofg.edge_role("L2") == "negative" # delta -100 -> blue + assert ofg.edge_role("L3") == EDGE_ROLE_POSITIVE # delta +50 -> coral + assert ofg.edge_role("nope") is None + + +class TestHighlightRecordsBaseColor: + """After highlight compounds the rendered colour, the authoritative base + colour is recorded so ``edge_role`` / ``tag_constrained_path`` never parse + the ``"c:yellow:c"`` string.""" + + def test_base_color_recorded_and_role_stable_after_highlight(self): + df = _three_line_df() # L1 +overload, L2 -, L3 + + ofg = OverFlowGraph(_basic_topo(3), [0], df) + ofg.highlight_significant_line_loading({ + "L1": {"before": 110, "after": 80}, + "L3": {"before": 75, "after": 60}, + }) + (_, l1) = _edge_by_name(ofg.g, "L1") + (_, l3) = _edge_by_name(ofg.g, "L3") + # rendered colour is compound, but base_color + edge_role stay clean + assert l1["color"] == '"black:yellow:black"' + assert l1["base_color"] == "black" + assert ofg.edge_role("L1") == EDGE_ROLE_OVERLOAD + assert l3["base_color"] == "coral" + assert ofg.edge_role("L3") == EDGE_ROLE_POSITIVE + + def test_tag_constrained_path_skips_compound_coral_via_base_color(self): + df = _three_line_df() + ofg = OverFlowGraph(_basic_topo(3), [0], df) + ofg.highlight_significant_line_loading({"L3": {"before": 75, "after": 60}}) + # L3 is coral (compound after highlight); constrained-path tagging must skip it + ofg.tag_constrained_path(lines_constrained_path=["L3"]) + (_, l3) = _edge_by_name(ofg.g, "L3") + assert l3.get("on_constrained_path") is None diff --git a/alphaDeesp/tests/test_overflow_renderer.py b/alphaDeesp/tests/test_overflow_renderer.py new file mode 100644 index 00000000..e4457f7c --- /dev/null +++ b/alphaDeesp/tests/test_overflow_renderer.py @@ -0,0 +1,97 @@ +"""Unit tests for :class:`OverflowGraphRenderer`. + +The renderer holds all Graphviz *presentation* logic extracted from +``OverFlowGraph``. These tests exercise it standalone (on bare graphs) to +prove the model/renderer split leaves the rendering reusable in isolation — +the reason downstream repositories can depend on it directly. +""" + +import networkx as nx +import pandas as pd + +from alphaDeesp.core.graphs.overflow_renderer import OverflowGraphRenderer +# The renderer must also be reachable through both public surfaces. +from alphaDeesp.core.graphs import OverflowGraphRenderer as _FromPackage +from alphaDeesp.core.graphsAndPaths import OverflowGraphRenderer as _FromShim + + +class TestPublicSurface: + def test_exported_from_package_and_shim(self): + assert _FromPackage is OverflowGraphRenderer + assert _FromShim is OverflowGraphRenderer + + +class TestPenwidthScaling: + def test_scaling_factor_and_visibility_floor(self): + scaling, min_pen = OverflowGraphRenderer.penwidth_scaling( + pd.Series([1000.0, 100.0, 10.0])) + assert scaling == 15.0 / 1000.0 + # floor = max(1 MW * 0.015, 10% * 15) = max(0.015, 1.5) = 1.5 + assert abs(min_pen - 1.5) < 1e-9 + + def test_all_zero_flow_falls_back_to_unit_scale(self): + scaling, min_pen = OverflowGraphRenderer.penwidth_scaling(pd.Series([0.0])) + assert scaling == 1.0 + assert min_pen == 1.5 + + def test_edge_penwidth_is_clamped_to_floor(self): + # tiny flow (10 MW) at the 1000-MW scale -> 0.15 raw, clamped to 1.5 + pen = OverflowGraphRenderer.edge_penwidth( + 10.0, scaling_factor=0.015, min_penwidth=1.5, float_precision="%.2f") + assert pen == 1.5 + + def test_edge_penwidth_scales_when_above_floor(self): + pen = OverflowGraphRenderer.edge_penwidth( + 1000.0, scaling_factor=0.015, min_penwidth=1.5, float_precision="%.2f") + assert pen == 15.0 + + +class TestNodeShapes: + def test_set_hub_shapes_only_marks_hubs(self): + g = nx.MultiDiGraph() + g.add_node("A") + g.add_node("B") + OverflowGraphRenderer.set_hub_shapes(g, ["A"], shape_hub="diamond") + assert g.nodes["A"]["shape"] == "diamond" + assert g.nodes["B"]["shape"] == "oval" + + def test_collapse_red_loops_collapses_pure_coral_oval(self): + g = nx.MultiDiGraph() + g.add_node("N1", shape="oval") + g.add_edge("N1", "N2", color="coral") + OverflowGraphRenderer.collapse_red_loops(g) + assert g.nodes["N1"]["shape"] == "point" + + def test_collapse_red_loops_leaves_mixed_node_alone(self): + g = nx.MultiDiGraph() + g.add_node("N1", shape="oval") + g.add_edge("N1", "N2", color="coral") + g.add_edge("N1", "N3", color="blue") + OverflowGraphRenderer.collapse_red_loops(g) + assert g.nodes["N1"]["shape"] == "oval" + + +class TestSwappedFlowStyling: + def test_swapped_lines_get_tapered_style(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", name="L1", color="coral") + g.add_edge("B", "C", name="L2", color="blue") + OverflowGraphRenderer.highlight_swapped_flows(g, ["L1"]) + l1 = g.edges[("A", "B", 0)] + l2 = g.edges[("B", "C", 0)] + assert l1["style"] == "tapered" and l1["dir"] == "both" and l1["arrowtail"] == "none" + assert "style" not in l2 + + +class TestHighlightFormatting: + def test_compound_highlight_color(self): + assert OverflowGraphRenderer.highlight_color("black") == '"black:yellow:black"' + assert OverflowGraphRenderer.highlight_color("coral") == '"coral:yellow:coral"' + + def test_overload_label_emphasises_before(self): + label = OverflowGraphRenderer.overload_label("x", 110, 80) + assert "110%" in label and "80%" in label + + def test_low_margin_label_emphasises_after(self): + label = OverflowGraphRenderer.low_margin_label("x", 90, 0) + assert "90% → 0%" in label diff --git a/alphaDeesp/tests/test_shortest_paths.py b/alphaDeesp/tests/test_shortest_paths.py index 333feb09..29d8b877 100644 --- a/alphaDeesp/tests/test_shortest_paths.py +++ b/alphaDeesp/tests/test_shortest_paths.py @@ -84,3 +84,55 @@ def test_multigraph_with_key(self): g, "A", "C", mandatory_edge=("A", "B", k1), weight_attr="weight") assert path is not None assert cost == 2 # key k1 (1) + B->C (1) + + +# ────────────────────────────────────────────────────────────────────── +# MultiDiGraph handling: networkx passes the {key: attr} view of parallel +# edges to a callable weight. The old closures did attr.get(weight) and read +# 0 for every multigraph edge; the shared factory now takes the min parallel +# weight and matches promoted (u, v) OR (u, v, key). +# ────────────────────────────────────────────────────────────────────── + +from alphaDeesp.core.graphsAndPaths import shortest_path_mandatory_and_promoted # noqa: E402 + + +class TestMultiDiGraphWeighting: + + def test_min_weight_path_chosen_on_multigraph(self): + g = nx.MultiDiGraph() + # Direct A->C is heavy; the A->B->C route is light. With the old bug all + # weights read 0, so the (shorter-hop) direct edge would win wrongly. + g.add_edge("A", "C", weight=100.0) + g.add_edge("A", "B", weight=1.0) + g.add_edge("B", "C", weight=1.0) + path, total = shortest_path_with_promoted_edges( + g, "A", "C", promoted_edges=[], weight_attr="weight") + assert path == ["A", "B", "C"] + assert total == 2.0 + + def test_parallel_edges_use_min_weight(self): + g = nx.MultiDiGraph() + g.add_edge("A", "B", weight=50.0) + g.add_edge("A", "B", weight=2.0) # cheaper parallel edge + g.add_edge("A", "X", weight=3.0) + g.add_edge("X", "B", weight=3.0) + # Cheapest A->B is the 2.0 parallel edge (< 3+3 via X). + path, total = shortest_path_with_promoted_edges( + g, "A", "B", promoted_edges=[], weight_attr="weight") + assert path == ["A", "B"] + assert total == 2.0 + + def test_promoted_key_tuple_matches_on_multigraph(self): + g = nx.MultiDiGraph() + # Two equal-weight amont routes; promote the B-route by (u, v, key). + g.add_edge("S", "A", weight=1.0) + g.add_edge("A", "M1", weight=1.0) + kb1 = g.add_edge("S", "B", weight=1.0) + kb2 = g.add_edge("B", "M1", weight=1.0) + g.add_edge("M1", "M2", weight=1.0) + g.add_edge("M2", "T", weight=1.0) + path, _ = shortest_path_mandatory_and_promoted( + g, "S", "T", mandatory_edge=("M1", "M2"), + promoted_edges=[("S", "B", kb1), ("B", "M1", kb2)], + weight_attr="weight") + assert path == ["S", "B", "M1", "M2", "T"] diff --git a/alphaDeesp/tests/test_simulation_create_df.py b/alphaDeesp/tests/test_simulation_create_df.py new file mode 100644 index 00000000..48033dcb --- /dev/null +++ b/alphaDeesp/tests/test_simulation_create_df.py @@ -0,0 +1,141 @@ +"""Differential tests for the vectorised ``Simulation.create_df``. + +``create_df`` used to run several ``iterrows`` passes; it is now vectorised with +numpy masks. Because the end-to-end path needs a grid2op backend (unavailable in +the unit environment), we pin equivalence by comparing the vectorised output +against a faithful re-implementation of the *original* row-by-row logic across a +fuzz of random inputs, plus a couple of hand-checked cases. +""" + +import math + +import numpy as np +import pandas as pd + +from alphaDeesp.core.simulation import Simulation + + +# ── Minimal host exposing create_df / branch_direction_swaps without ABC ── +class _Host: + create_df = Simulation.create_df + branch_direction_swaps = staticmethod(Simulation.branch_direction_swaps) + + def __init__(self, new_flows, threshold=0.2): + self._new_flows = np.asarray(new_flows, dtype=float) + self.param_options = {"ThresholdReportOfLine": threshold} + self.debug = False + + def cut_lines_and_recomputes_flows(self, ids): + return self._new_flows + + +# ── Faithful re-implementation of the ORIGINAL iterrows logic ── +def _reference_create_df(edges, new_flows, threshold, line_to_cut): + df = pd.DataFrame(edges) + + swapped = [] + for i, row in df.iterrows(): + a = row["init_flows"] + if a < 0 and a != 0.0: + idx_or = row["idx_or"] + df.at[i, "idx_or"] = row["idx_ex"] + df.at[i, "idx_ex"] = idx_or + df.at[i, "init_flows"] = math.fabs(row["init_flows"]) + swapped.append(True) + else: + swapped.append(False) + df["swapped"] = swapped + + n_flows = [f * -1 if sw else f for f, sw in zip(new_flows, df["swapped"])] + df["new_flows"] = n_flows + + nfs = [] + for _, row in df.iterrows(): + nfs.append(row["new_flows"] < 0 and math.fabs(row["new_flows"]) > math.fabs(row["init_flows"])) + df["new_flows_swapped"] = nfs + + delta_flo = [] + for i, row in df.iterrows(): + if row["new_flows_swapped"]: + delta_flo.append(math.fabs(row["new_flows"]) + math.fabs(row["init_flows"])) + idx_or = row["idx_or"] + df.at[i, "idx_or"] = row["idx_ex"] + df.at[i, "idx_ex"] = idx_or + df.at[i, "init_flows"] = math.fabs(row["init_flows"]) + elif (np.sign(row["new_flows"]) != np.sign(row["init_flows"])) and (row["new_flows"] != 0) and (row["init_flows"] != 0): + delta_flo.append(-(math.fabs(row["new_flows"]) + math.fabs(row["init_flows"]))) + else: + delta_flo.append(math.fabs(row["new_flows"]) - math.fabs(row["init_flows"])) + df["delta_flows"] = delta_flo + + gray_edges = [] + ltc_report = df["delta_flows"].abs()[line_to_cut[0]] + max_overload = ltc_report * float(threshold) + for edge_value in df["delta_flows"]: + gray_edges.append(math.fabs(edge_value) < max_overload) + df["gray_edges"] = gray_edges + return df + + +_COLS = ["idx_or", "idx_ex", "init_flows", "new_flows", + "new_flows_swapped", "delta_flows", "swapped", "gray_edges"] + + +def _assert_equivalent(edges, new_flows, threshold, line_to_cut): + got = _Host(new_flows, threshold).create_df({"edges": edges}, line_to_cut) + ref = _reference_create_df(edges, new_flows, threshold, line_to_cut) + for col in _COLS: + g = got[col].to_numpy() + r = ref[col].to_numpy() + if g.dtype.kind == "f" or r.dtype.kind == "f": + np.testing.assert_allclose(g.astype(float), r.astype(float), atol=1e-12, + err_msg=f"column {col} differs") + else: + np.testing.assert_array_equal(g, r, err_msg=f"column {col} differs") + + +class TestCreateDfEquivalence: + + def test_hand_crafted_mixed_case(self): + edges = { + "idx_or": [0, 1, 2, 3], + "idx_ex": [1, 2, 3, 0], + "init_flows": [10.0, -5.0, 0.0, 8.0], + } + # new flows: sign flip, overload, zero, mild change + new_flows = [-20.0, 3.0, 0.0, 9.0] + _assert_equivalent(edges, new_flows, 0.2, [0]) + + def test_all_zero_flows(self): + edges = {"idx_or": [0, 1], "idx_ex": [1, 2], "init_flows": [0.0, 0.0]} + _assert_equivalent(edges, [0.0, 0.0], 0.2, [0]) + + def test_fuzz_matches_reference(self): + rng = np.random.default_rng(20260716) + for _ in range(400): + n = int(rng.integers(2, 9)) + edges = { + "idx_or": list(rng.integers(0, n, size=n)), + "idx_ex": list(rng.integers(0, n, size=n)), + # include negatives, zeros and positives + "init_flows": list(np.round(rng.uniform(-50, 50, size=n), 3)), + } + new_flows = list(np.round(rng.uniform(-80, 80, size=n), 3)) + # occasionally force exact zeros to exercise sign(0) edge cases + if rng.random() < 0.3: + edges["init_flows"][int(rng.integers(0, n))] = 0.0 + if rng.random() < 0.3: + new_flows[int(rng.integers(0, n))] = 0.0 + line_to_cut = [int(rng.integers(0, n))] + _assert_equivalent(edges, new_flows, 0.2, line_to_cut) + + def test_positional_index_is_used_for_ltc_report(self): + # A non-overloaded first row and a big overload elsewhere: the gray + # threshold must be computed from line_to_cut's row, positionally. + edges = {"idx_or": [0, 1, 2], "idx_ex": [1, 2, 0], + "init_flows": [1.0, 1.0, 1.0]} + new_flows = [1.0, 1.0, 100.0] # row 2 is the big delta + got = _Host(new_flows, 0.2).create_df({"edges": edges}, [2]) + # ltc_report = |delta[2]| = 99; max_overload = 19.8; rows 0,1 (~0) are gray + assert bool(got["gray_edges"].iloc[0]) is True + assert bool(got["gray_edges"].iloc[2]) is False diff --git a/alphaDeesp/tests/test_simulation_helpers.py b/alphaDeesp/tests/test_simulation_helpers.py new file mode 100644 index 00000000..c767464b --- /dev/null +++ b/alphaDeesp/tests/test_simulation_helpers.py @@ -0,0 +1,94 @@ +"""Unit tests for the backend-agnostic static helpers on +:class:`alphaDeesp.core.simulation.Simulation` — the MultiIndex line-model +lookups and the small dict / empty-frame utilities. No grid2op backend needed. +""" + +import pandas as pd + +from alphaDeesp.core.simulation import Simulation +from alphaDeesp.core.elements import ExtremityLine, OriginLine + + +def _indexed(rows): + """Build the (idx_or, idx_ex)-indexed DataFrame the lookups expect.""" + return pd.DataFrame(rows).set_index(["idx_or", "idx_ex"]) + + +class TestGetModelObjFromOr: + def test_direct_match_returns_origin_line(self): + df = _indexed({ + "idx_or": [1], "idx_ex": [2], "delta_flows": [5.0], + "swapped": [False], "new_flows_swapped": [False], + }) + obj = Simulation.get_model_obj_from_or(df, substation_id=1, dest=2, busbar=0) + assert isinstance(obj, OriginLine) + assert obj.end_substation_id == 2 and obj.flow_value == [5.0] and obj.busbar_id == 0 + + def test_duplicate_rows_take_first(self): + df = _indexed({ + "idx_or": [1, 1], "idx_ex": [2, 2], "delta_flows": [5.0, 7.0], + "swapped": [False, False], "new_flows_swapped": [False, False], + }) + obj = Simulation.get_model_obj_from_or(df, 1, 2, 0) + assert obj.flow_value == [5.0] + + def test_swapped_match_same_flag_is_origin_line(self): + df = _indexed({ + "idx_or": [2], "idx_ex": [1], "delta_flows": [5.0], + "swapped": [True], "new_flows_swapped": [True], + }) + obj = Simulation.get_model_obj_from_or(df, substation_id=1, dest=2, busbar=0) + assert isinstance(obj, OriginLine) + + def test_swapped_match_diff_flag_is_extremity_line(self): + df = _indexed({ + "idx_or": [2], "idx_ex": [1], "delta_flows": [5.0], + "swapped": [True], "new_flows_swapped": [False], + }) + obj = Simulation.get_model_obj_from_or(df, substation_id=1, dest=2, busbar=0) + assert isinstance(obj, ExtremityLine) + + def test_no_match_returns_none(self): + df = _indexed({ + "idx_or": [9], "idx_ex": [10], "delta_flows": [1.0], + "swapped": [False], "new_flows_swapped": [False], + }) + assert Simulation.get_model_obj_from_or(df, 1, 2, 0) is None + + +class TestGetModelObjFromExt: + def test_direct_match_returns_extremity_line(self): + # from_ext direct match is (dest, substation_id) + df = _indexed({ + "idx_or": [2], "idx_ex": [1], "delta_flows": [5.0], + "swapped": [False], "new_flows_swapped": [False], + }) + obj = Simulation.get_model_obj_from_ext(df, substation_id=1, dest=2, busbar=0) + assert isinstance(obj, ExtremityLine) + assert obj.start_substation_id == 2 and obj.flow_value == [5.0] + + def test_swapped_match_diff_flag_is_origin_line(self): + df = _indexed({ + "idx_or": [1], "idx_ex": [2], "delta_flows": [5.0], + "swapped": [True], "new_flows_swapped": [False], + }) + obj = Simulation.get_model_obj_from_ext(df, substation_id=1, dest=2, busbar=0) + assert isinstance(obj, OriginLine) + + def test_no_match_returns_none(self): + df = _indexed({ + "idx_or": [9], "idx_ex": [10], "delta_flows": [1.0], + "swapped": [False], "new_flows_swapped": [False], + }) + assert Simulation.get_model_obj_from_ext(df, 1, 2, 0) is None + + +class TestSmallHelpers: + def test_invert_dict_keys_values(self): + assert Simulation.invert_dict_keys_values({1: "a", 2: "b"}) == {"a": 1, "b": 2} + + def test_create_end_result_empty_dataframe(self): + df = Simulation.create_end_result_empty_dataframe() + assert len(df) == 0 + for col in ("overflow ID", "Flows before", "Efficacity", "Substation ID"): + assert col in df.columns diff --git a/alphaDeesp/tests/test_topo_applicator.py b/alphaDeesp/tests/test_topo_applicator.py index 3b3ebd0b..2ea485e0 100644 --- a/alphaDeesp/tests/test_topo_applicator.py +++ b/alphaDeesp/tests/test_topo_applicator.py @@ -104,3 +104,60 @@ def test_bus_not_present_returns_none(self): kind, value = TopoApplicatorMixin._classify_bus(2, {0: 5.0}, {1: 3.0}) assert kind is None assert value == 0 + + +# ────────────────────────────────────────────────────────────────────── +# apply_new_topo_to_graph — full graph mutation (busbar split) +# ────────────────────────────────────────────────────────────────────── + +import networkx as nx # noqa: E402 +import pandas as pd # noqa: E402 + +from alphaDeesp.core.twin_nodes import twin_node_id # noqa: E402 + + +class _ApplyHost(TopoApplicatorMixin): + def __init__(self, g, df, sim_data): + self.g = g + self.df = df + self.simulator_data = sim_data + self.bag_of_graphs = {} + self.debug = False + + +class TestApplyNewTopoToGraph: + def _setup(self): + g = nx.MultiDiGraph() + for n in (0, 1, 2): + g.add_node(n) + g.add_edge(0, 1, color="blue", name="l01") + g.add_edge(0, 2, color="coral", name="l02") + df = pd.DataFrame({"idx_or": [0, 0], "idx_ex": [1, 2], "swapped": [False, False]}) + elements = [ + OriginLine(busbar_id=0, end_substation_id=1, flow_value=[5.0]), + OriginLine(busbar_id=0, end_substation_id=2, flow_value=[3.0]), + ] + sim_data = {"substations_elements": {0: elements}} + return _ApplyHost(g, df, sim_data), g + + def test_split_rewires_second_element_to_twin_node(self): + host, g = self._setup() + new_graph, internal = host.apply_new_topo_to_graph(g, [0, 1], node_to_change=0) + twin = twin_node_id(0) + # element 0 stays on node 0, element 1 moves to the twin busbar node + assert new_graph.has_edge(0, 1) + assert new_graph.has_edge(twin, 2) + # original colours are carried over onto the rewired edges + assert new_graph.edges[(0, 1, 0)]["color"] == "blue" + assert new_graph.edges[(twin, 2, 0)]["color"] == "coral" + # the topology is registered in the bag under its encoded name + assert "0_01" in host.bag_of_graphs + assert internal[0][1].busbar_id == 1 # second element reassigned to bus 1 + + def test_single_bus_topology_keeps_everything_on_node(self): + host, g = self._setup() + # all-zero topology: no split, everything stays on node 0 + new_graph, _ = host.apply_new_topo_to_graph(g, [0, 0], node_to_change=0) + assert new_graph.has_edge(0, 1) + assert new_graph.has_edge(0, 2) + assert twin_node_id(0) not in new_graph.nodes diff --git a/docs/API.rst b/docs/API.rst new file mode 100644 index 00000000..77e14e0b --- /dev/null +++ b/docs/API.rst @@ -0,0 +1,87 @@ +************* +API reference +************* + +Auto-generated from the source docstrings. See :doc:`ARCHITECTURE` for how these +pieces fit together. + +The backend port +================ + +.. automodule:: alphaDeesp.core.simulation + :members: Simulation + :member-order: bysource + +.. automodule:: alphaDeesp.core.elements + :members: + +Orchestration +============= + +.. automodule:: alphaDeesp.expert_operator + :members: + +.. autoclass:: alphaDeesp.core.alphadeesp.AlphaDeesp + :members: run, get_ranked_combinations, compute_best_topologies, + compute_all_combinations, rank_topologies, identify_routing_buses + +.. autoclass:: alphaDeesp.core.alphadeesp.AlphaDeesp_warmStart + +The graphs package +================== + +Overflow model and renderer +--------------------------- + +.. autoclass:: alphaDeesp.core.graphs.overflow_graph.OverFlowGraph + :members: + +.. autoclass:: alphaDeesp.core.graphs.overflow_renderer.OverflowGraphRenderer + :members: + +Semantic edge roles +------------------- + +.. automodule:: alphaDeesp.core.graphs.edge_roles + :members: + +Structured analysis +------------------- + +.. autoclass:: alphaDeesp.core.graphs.power_flow_graph.PowerFlowGraph + :members: + +.. autoclass:: alphaDeesp.core.graphs.structured_overload_graph.Structured_Overload_Distribution_Graph + :members: + +.. autoclass:: alphaDeesp.core.graphs.constrained_path.ConstrainedPath + :members: + +Null-flow and consolidation mixins +---------------------------------- + +.. autoclass:: alphaDeesp.core.graphs.null_flow_graph.NullFlowGraphMixin + :members: add_relevant_null_flow_lines, add_relevant_null_flow_lines_all_paths, + detect_edges_to_keep + +.. autoclass:: alphaDeesp.core.graphs.graph_consolidation.GraphConsolidationMixin + :members: consolidate_graph, consolidate_constrained_path, consolidate_loop_path, + reverse_edges + +Graph helpers +------------- + +.. automodule:: alphaDeesp.core.graphs.graph_utils + :members: + +.. automodule:: alphaDeesp.core.graphs.shortest_paths + :members: shortest_path_min_weight_then_hops, shortest_path_mandatory_and_promoted, + shortest_path_with_promoted_edges + +.. automodule:: alphaDeesp.core.graphs.null_flow + :members: + +Interactive HTML viewer +======================= + +.. autofunction:: alphaDeesp.core.interactive_html.build_interactive_html diff --git a/docs/ARCHITECTURE.rst b/docs/ARCHITECTURE.rst new file mode 100644 index 00000000..23188df7 --- /dev/null +++ b/docs/ARCHITECTURE.rst @@ -0,0 +1,188 @@ +************ +Architecture +************ + +This page is a developer-oriented map of ``alphaDeesp`` (the package behind +**ExpertOp4Grid**). It explains how the pieces fit together, the contracts +between them, and where to plug in new behaviour. For the algorithm's +*conceptual* description see :doc:`DESCRIPTION`; for a call-level walkthrough +see :doc:`DETAILS`. + +Pipeline overview +================= + +Given an overloaded line, the system builds an *influence graph* around the +overload, ranks candidate substations/topologies, simulates the top-ranked +ones and returns a score (0–4) per remediation:: + + config.ini + CLI ─► alphaDeesp.main + │ builds + ▼ + Grid2opSimulation / PypownetSimulation (subclass of core.simulation.Simulation) + │ topology, dataframe, mappings + ▼ + expert_operator.expert_operator() + │ + ├─► OverFlowGraph (core/graphs/overflow_graph.py) + ├─► Structured_Overload_Distribution_Graph + ├─► AlphaDeesp.get_ranked_combinations() + └─► sim.compute_new_network_changes() ─► end-result DataFrame + +The orchestration lives in :func:`alphaDeesp.expert_operator.expert_operator`. +Everything upstream of it is a *backend adapter*; everything downstream is the +backend-agnostic expert system. + +The Simulation contract (backend port) +====================================== + +:class:`alphaDeesp.core.simulation.Simulation` is an abstract base class — the +single seam a new grid backend must implement. Concrete backends +(``Grid2opSimulation``, the deprecated ``PypownetSimulation``) subclass it and +provide the topology, the flow DataFrame and the id mappings the expert system +needs. + +Key abstract methods a backend must supply: + +``get_dataframe()`` + One row per line with at least ``idx_or``, ``idx_ex``, ``init_flows``, + ``new_flows``, ``delta_flows``, ``swapped`` and ``gray_edges``. Built by the + base ``create_df`` helper, which the backend feeds via + ``cut_lines_and_recomputes_flows``. +``get_substation_elements()`` + ``{substation_id: [element, ...]}`` where each element is a + :class:`~alphaDeesp.core.elements.Production`, + :class:`~alphaDeesp.core.elements.Consumption`, + :class:`~alphaDeesp.core.elements.OriginLine` or + :class:`~alphaDeesp.core.elements.ExtremityLine`. +``isAntenna`` / ``isDoubleLine`` / ``getLinesAtSubAndBusbar`` + Topology predicates used to prune candidate actions. +``compute_new_network_changes(ranked_combinations)`` + Simulate the recommended topologies and return the end-result DataFrame. + +**Row-per-line invariant.** The DataFrame is one row per line, in line-id +order. The overloaded line's redispatch is read *positionally* by line id, and +``OverFlowGraph`` matches ``lines_to_cut`` against row positions — keep that +ordering when producing the frame. + +The ``graphs`` package +====================== + +``core/graphsAndPaths.py`` is a **backwards-compatible shim** re-exporting the +public surface of the ``core/graphs/`` package. External code can keep importing +``from alphaDeesp.core.graphsAndPaths import OverFlowGraph, ...``; new code should +import from ``alphaDeesp.core.graphs``. The public surface is pinned by +``tests/test_graphs_package.py`` (``EXPECTED_PUBLIC_NAMES`` / +``EXPECTED_SUBMODULES``). + +Semantic model vs. renderer +--------------------------- + +``OverFlowGraph`` is split into a **semantic model** and a **renderer**: + +* :class:`~alphaDeesp.core.graphs.overflow_graph.OverFlowGraph` owns the model — + the ``MultiDiGraph`` topology, per-edge redispatch magnitude, the edge *role* + encoded as a base colour, and the boolean semantic flags consumed downstream + (``is_overload``, ``is_monitored``, ``on_constrained_path``, ``in_red_loop``, + ``is_hub``, ``is_extra_cut``). +* :class:`~alphaDeesp.core.graphs.overflow_renderer.OverflowGraphRenderer` owns + all Graphviz *presentation* — penwidth scaling, node shapes, tapered swap + styling, the compound ``"colour:yellow:colour"`` highlight strings, HTML + loading labels, and plotting. It is **stateless** (static methods over a + passed-in graph), so downstream repositories can reuse it on any compatible + ``MultiDiGraph``. + +Semantic edge roles +------------------- + +Never parse a Graphviz colour string. The base colour of an edge encodes a +stable *role*: + +=============== ========================================== ===================================== +base colour role (``edge_roles``) meaning +=============== ========================================== ===================================== +``black`` ``EDGE_ROLE_OVERLOAD`` overloaded contingency line +``blue`` ``EDGE_ROLE_NEGATIVE`` negative redispatch (into the overload) +``coral`` ``EDGE_ROLE_POSITIVE`` positive redispatch (loop / away) +``gray`` ``EDGE_ROLE_INSIGNIFICANT`` below-threshold redispatch +``dimgray`` ``EDGE_ROLE_NULL_NON_RECONNECTABLE`` null-flow, non-reconnectable line +=============== ========================================== ===================================== + +:func:`~alphaDeesp.core.graphs.edge_roles.edge_role_of` is the single authority +mapping an edge's base colour to its role. It prefers the stable ``base_color`` +attribute (recorded by the renderer when it wraps a colour into a compound +highlight) and is compound-safe. ``OverFlowGraph.edge_role(name)`` is the +convenience by-line-name accessor. + +Structured overload graph +------------------------- + +:class:`~alphaDeesp.core.graphs.structured_overload_graph.Structured_Overload_Distribution_Graph` +turns a coloured overflow graph into the path structure the ranking needs: + +* **constrained path** — the black (overload) + blue (negative) network that + funnels current into the overloads (see + :class:`~alphaDeesp.core.graphs.constrained_path.ConstrainedPath`); +* **red loops** — parallel coral paths onto which flow can be rerouted; +* **hubs** — substations where a loop path meets the constrained path. + +Its colour-filtered views, ``red_loops`` and ``hubs`` are lazy +``functools.cached_property`` computed from a **construction-time snapshot** of +the graph (so a later mutation of the caller's graph does not leak into the +views). ``red_loops`` uses the constructor *seed* hubs; the public +``find_loops()`` re-enumerates with the *detected* hubs — this split keeps the +lazy properties order-independent while matching the historical eager +behaviour. + +Null-flow and consolidation +--------------------------- + +Two mixins fold onto ``OverFlowGraph``: + +* :class:`~alphaDeesp.core.graphs.null_flow_graph.NullFlowGraphMixin` — decides + which disconnected/reconnectable "null-flow" lines lie on short paths bridging + the constrained/dispatch sides, via a per-component Dijkstra search. The + routing weight is precomputed as an edge attribute (fast string weight); + ``capacity_weighted=False`` (default) reproduces the historical hop-only + behaviour bit-identically, ``True`` enables capacity-weighted routing. +* :class:`~alphaDeesp.core.graphs.graph_consolidation.GraphConsolidationMixin` — + disambiguates the raw graph (recolouring / reversing edges) so the structured + analysis is stable. + +AlphaDeesp (ranking) +==================== + +:class:`alphaDeesp.core.alphadeesp.AlphaDeesp` scores candidate busbar splits. +Construction runs the pipeline by default:: + + AlphaDeesp(graph, df, simulator_data, substation_in_cooldown) # auto_run=True + AlphaDeesp(graph, df, simulator_data, auto_run=False).run() # staged / testable + +The scoring helpers live in :class:`~alphaDeesp.core.topology_scorer.TopologyScorerMixin` +and the graph-mutation helpers (applying a busbar split, twin-node encoding) in +:class:`~alphaDeesp.core.topo_applicator.TopoApplicatorMixin`. +``AlphaDeesp_warmStart`` is the pre-existing "skip the pipeline" path (a caller +supplies a pre-built distribution graph). + +Interactive HTML viewer +======================= + +``core/interactive_html/`` builds a self-contained interactive viewer around a +Graphviz-rendered SVG (pan/zoom, hover, click-to-highlight, search, semantic +layer toggles). It is a package: the CSS/JS/HTML skeleton are externalised under +``assets/`` and reassembled at runtime by ``template.html_template()`` — edit the +``.css`` / ``.js`` assets directly. The viewer reads the semantic flags +(``is_overload`` …) rather than reinterpreting colours, which is why those flags +are the source of truth on the model. + +Extending the system +===================== + +* **New grid backend** — subclass :class:`~alphaDeesp.core.simulation.Simulation` + and implement its abstract methods; nothing else in the pipeline needs to + change. +* **New rendering** — reuse or subclass + :class:`~alphaDeesp.core.graphs.overflow_renderer.OverflowGraphRenderer`; it is + independent of the model. +* **New semantic layer** — stamp a boolean flag on the model + (``OverFlowGraph``) and add it to the viewer's layer configuration; do not key + new behaviour off colour strings. diff --git a/docs/CODE_REVIEW.md b/docs/CODE_REVIEW.md new file mode 100644 index 00000000..a110fd62 --- /dev/null +++ b/docs/CODE_REVIEW.md @@ -0,0 +1,304 @@ +# ExpertOp4Grid — Architecture & Code Review + +A large review of the `alphaDeesp` package covering code architecture, +interface & interaction design, performance & bottlenecks, and documentation +& maintainability — with a dedicated focus on `OverFlowGraph`, the class +consumed by other repositories. + +The **[Follow-up: what was implemented](#follow-up-what-was-implemented)** +section at the end tracks which recommendations have already been actioned. + +--- + +## How this review was conducted (principles & method) + +1. **Read the seams, not just the lines.** Start from the public entry points + (`main.py`, `expert_operator.py`, the `Simulation` ABC) and the module other + repos import (`OverFlowGraph`). Architecture lives at the boundaries where a + module hands control or data to another; debt concentrates there. +2. **Score the four concerns against one another.** Architecture, interface + design, performance and maintainability trade off. A "clean" refactor that + raises coupling is not a win. +3. **Distinguish invariants from code.** An undocumented invariant (row `i` == + line id; edge `color` strings encoding semantics) is a latent defect even + when today's callers satisfy it. +4. **Severity = blast radius × likelihood × detectability.** A silent + data-corruption bug in a cross-repo module outranks a style nit repeated 40 + times. +5. **Quick win vs. deep revision.** A quick win is local, low-risk, testable in + isolation. A deep revision changes a contract or a layering decision and + needs a migration path. Keep the two buckets separate. +6. **Verify claims against the code, not the docs.** Several `CLAUDE.md` + assertions were stale; the source was trusted and the docs corrected. + +**Overall assessment:** a healthy, actively-improving codebase. The split of +the monolithic `graphsAndPaths.py` into a typed `graphs/` package, the +rustworkx acceleration, the backward-compat shim, and a strong test suite +(~7.5k test LOC vs ~8.2k source LOC) are above the norm for a research-origin +expert system. Weaknesses were concentrated in a few correctness bugs, CI +pointing at the wrong files, a metric-driven refactor that traded cohesion for +coupling, and view/semantic entanglement inside `OverFlowGraph`. + +--- + +## Strengths + +- **Clean modular decomposition** of the graph layer: pure helpers + (`graph_utils`), path algorithms (`shortest_paths`), the constrained-path + value object, the structural analyzer, and the renderable graphs are + separated and unit-tested. +- **Backward compatibility done right:** `graphsAndPaths.py` is a re-export + shim with an explicit `__all__`, pinned by a contract test. +- **Typing and intent-capturing docstrings** in the new package. +- **A real quality pipeline** (`scripts/code_quality_report.py`; CI runs + pyflakes + mypy + radon). +- **Deliberate deprecation** of the Pypownet backend rather than deletion. +- **Broad test coverage** of the graph package. + +--- + +## Findings (bugs & correctness) + +| # | Sev | Location | Issue | Status | +|---|-----|----------|-------|--------| +| 1 | ~~High~~ → readability | `graphs/overflow_graph.py` `rename_nodes` | **Correction:** the original was *functionally correct* — the `idx_ex` comprehension iterated the `idx_ex` column with a misleadingly-named `idx_or` loop variable. The initial "column corruption" grade was a misread, caught by adversarial re-verification. | **Clarified** (loop variable renamed; behaviour unchanged) | +| 2 | Med (latent) | `graphs/structured_overload_graph.py` `get_dispatch_edges_nodes` | `red_loops.Path.sum()` on an empty loop DataFrame returns the scalar `0`, so `set(0)` raises `TypeError` on any grid with no loop paths. | **Fixed** (explicit empty guard) | +| 3 | Med | `graphs/null_flow_graph.py` `_compute_sssp_paths` | bare `except Exception` silently swallowed Dijkstra failures. | **Fixed** (narrow catch + `logger.warning`) | +| 4 | Med | `core/simulation.py` `create_df` | positional label-indexing (`df[...][line_to_cut[0]]`) assumes a contiguous `RangeIndex` aligned to line ids. | Open (documented) | +| 5 | Med | `graphs/overflow_graph.py` `__init__` | mutated the caller's DataFrame (added `line_name`, `rename_nodes` rewrote columns). | **Fixed** (`df.copy()`) | +| 6 | Low | `graphs/shortest_paths.py` | dead branch + incomplete MultiDiGraph promoted-edge matching. | Open | +| 7 | Low | `core/alphadeesp.py` `to_DiGraph` | missing `capacity` defaults to `1.0`, skewing `rank_red_loops`. | Open | + +> **On `find_loops` and the path cutoff (review perf item / QW3).** The +> `find_loops` code shipped with its `rx.all_simple_paths` cutoff *commented +> out* under a comment calling it "crucial to prevent hanging on large grids". +> Enabling it by default (cutoff = 10 nodes) was **itself a regression** — +> caught by the adversarial verification pass: rustworkx `cutoff` counts +> *nodes*, real RTE zone grids have loop paths of 15–42 nodes, so a default of +> 10 silently drops legitimate loops and can empty `find_loops` (triggering +> finding #2). The cutoff is therefore now an **opt-in parameter** +> (`loop_path_cutoff`, default `None` = unbounded = the original behaviour); the +> hang risk is documented and gateable rather than fixed by a lossy default. +> The analogous consolidation-path cutoff is likewise opt-in. + +--- + +## Architecture + +- **Metric-driven mixins reduced cohesion.** `NullFlowGraphMixin`, + `GraphConsolidationMixin`, `TopologyScorerMixin`, `TopoApplicatorMixin` were + extracted "to keep per-file LOC and average cyclomatic complexity within + A-grade bounds" — an implicit interface with no enforcement (each documents + "assumes the concrete class provides `self.g` …"). Prefer composition or a + `typing.Protocol` so the required surface is enforced. *(Deep revision — not + taken this pass, by request.)* +- **`OverFlowGraph` fused construction + rendering + semantic tagging.** + Addressed by the model/renderer split (below). +- **`Structured_Overload_Distribution_Graph.__init__` does heavy work eagerly** + (multiple full-graph copies) and the consolidation loop rebuilds the whole + object each iteration. Copy count reduced (below); the eager-construction / + staged-`run()` shape remains a future revision. +- **`AlphaDeesp.__init__` does everything** (ranking runs in the constructor; + `AlphaDeesp_warmStart` exists to skip it) — a candidate for an explicit + `run()` API. + +## Interface & interaction design + +- The `Simulation` ABC is the best-designed surface — keep it as the template. +- Naming is mixed (`snake_case` / `camelCase` / `Structured_..._Graph`) and is + load-bearing on the abstract contract and external importers. Prefer PEP8 + forwarding aliases + a deprecation clock over hard renames. +- French/English domain terms (`amont`/`aval`) leak into the API; a short + glossary in the docs would remove onboarding friction. +- The config key `ThersholdMinPowerOfLoop` is misspelled and effectively + public; accept both spellings rather than renaming. + +## Performance & bottlenecks + +- **`find_loops`** — missing cutoff + `O(n²)` source/target enumeration. + **Fixed** (cutoff re-enabled/configurable). +- **Repeated full-graph copies** — `delete_color_edges` copies per call; the + structured graph chained ~7 copies. **Fixed** (multi-colour `delete_color_edges`, + single-pass derived graphs). +- **`iterrows` hot loops in ranking** — `sort_hubs` (`O(hubs × rows)`) and + `_initial_inflow_between` (`O(buses × edges × rows)`). **Fixed** (group-sum + vectorisation + a precomputed inflow lookup). +- `create_df` makes several `iterrows` passes (backend data-prep, exercised by + the grid2op suite) — left for a follow-up with grid2op available. + +## Documentation & maintainability + +- **`CLAUDE.md` had drifted** (described `graphsAndPaths.py` as the real module; + claimed the codebase was untyped and used star imports; stale Python + version). **Refreshed.** +- **CI lints/type-checks the shim, not the package.** `.circleci/config.yml` + runs pyflakes/mypy on the 49-line `graphsAndPaths.py`, never on + `graphs/overflow_graph.py` etc. — so the cross-repo module has no static gate. + **Recommended fix (not taken this pass, by request):** point CI at the + `graphs/` package. +- `stdout` printing vs logging is inconsistent in the older backends. +- `interactive_html.py` (~976 LOC) and `Grid2opSimulation.py` (~814 LOC) are + the remaining maintainability hotspots. + +--- + +## Focus: `OverFlowGraph` (the cross-repo interface) + +**Strengths:** the shim + `__all__` keep the import path stable; the move to +explicit source-of-truth semantic flags (`is_overload`, `is_monitored`, +`in_red_loop`, `on_constrained_path`, `is_hub`, `is_extra_cut`) is the right +direction — downstream consumers query semantics without reverse-engineering +colours. + +**What was undermining it, and what changed:** + +1. **Rendering and analysis were fused in one class.** Now split into a + **semantic model** (`OverFlowGraph`) and a stateless **renderer** + (`OverflowGraphRenderer`) that owns all Graphviz vocabulary (penwidth, + shapes, tapered styling, compound `"colour:yellow:colour"` strings, HTML + labels, plotting). Other repos can reuse the renderer on any semantic graph, + or depend on the model without pulling in Graphviz concerns. Public method + signatures are unchanged; the renderer is exported from both the package and + the shim. +2. **Caller-DataFrame mutation** and the **`rename_nodes` bug** — both fixed. +3. **Semantics still partly ride on Graphviz colour strings** (e.g. + `tag_constrained_path` splits `"coral:yellow:coral"`). Fully inverting this + (model authoritative, colours derived) remains a future revision. +4. **No published API contract / version discipline for behaviour.** The + package's public *names* are pinned by `test_graphs_package.py`; behavioural + flags are pinned by `test_overflow_graph.py`. Extending semver discipline to + behaviour changes is recommended. + +--- + +## Follow-up: what was implemented + +This pass implemented review recommendations 1, 3, 4, 5 (skipping #2, the CI +repoint), plus the two named performance items and the `OverFlowGraph` +model/renderer deep revision, and refreshed `CLAUDE.md`. + +| Recommendation | Change | Tests | +|---|---|---| +| QW1 — `rename_nodes` | loop variable renamed for clarity (behaviour was already correct — see finding #1) | `TestRenameNodes` | +| QW3 — cutoff in `find_loops` + other path sites | opt-in `loop_path_cutoff` / `DEFAULT_CONSOLIDATION_PATH_CUTOFF`, **default `None`** (unbounded = original behaviour); plus an empty-loops guard in `get_dispatch_edges_nodes` (finding #2) | `TestStructuredOverloadDistributionGraphNoLoops`, existing structured-graph tests | +| QW4 — stop mutating caller df | `OverFlowGraph.__init__` copies the frame | `TestDoesNotMutateCallerDataFrame` | +| QW5 — narrow the bare `except` | `(nx.NetworkXException, ValueError)` + `logger.warning` | existing null-flow tests | +| Perf — repeated full-graph copies | `delete_color_edges` accepts multiple colours (single copy); structured graph builds each derived view in one pass | `test_graph_utils`, `test_graphs_package` | +| Perf — `iterrows` in ranking | `sort_hubs` group-sum vectorisation; `_build_inflow_lookup` precompute (kept `_initial_inflow_between` as fallback) | `TestSortHubs`, `TestBuildInflowLookup`, `TestBusLoopStrength` | +| Deep — split `OverFlowGraph` | new `OverflowGraphRenderer` (Graphviz presentation); `OverFlowGraph` delegates rendering, keeps the semantic model | `test_overflow_renderer.py`, `test_overflow_graph.py` | +| Docs | `docs/CODE_REVIEW.md` (this file); `CLAUDE.md` refreshed | — | + +**Not taken this pass (by request):** recommendation #2 (repoint CI static +analysis at the `graphs/` package) and the mixins → composition/Protocol deep +revision. Both remain recommended. + +**Validation.** The graph-package + ranking + renderer + interactive-html unit +suites pass locally without grid2op (342 tests). In addition, an **adversarial +multi-agent verification pass** (5 independent lenses) was run over the +behaviour-preserving changes: + +- `delete_color_edges` single-pass union, the `_build_inflow_lookup` / + `sort_hubs` vectorisations, the `OverFlowGraph` model/renderer split, the + `except` narrowing, and the `df.copy()` were all **verified equivalent** + (including a 20k-trial differential fuzz of the ranking helpers). +- The pass **caught two real issues**: the `find_loops` default-cutoff + regression (now reverted to opt-in) and the misread severity of the + `rename_nodes` finding (now corrected above). + +The grid2op integration suites (`alphadeesp_test.py`, `test_expert_op.py`, +`test_expert_rules.py`, the `grid2op/` tests) require `grid2op` + +`lightsim2grid` and were **not** run in this environment. With the cutoffs now +defaulting to `None`, the graph-analysis behaviour on those grids is unchanged +from `master`; running them in CI remains the recommended confirmation. + +--- + +## Downstream impact — `Expert_op4grid_recommender` + +The marota fork of `Expert_op4grid_recommender` (which depends on +`expertop4grid`, pinned `==0.3.2.post3`) was audited against every change in +this pass via a 5-lens cross-repo analysis, independently spot-checked. + +**Verdict: no breaking changes; one strictly-beneficial behavioural change.** + +| Vector | Result | +|---|---| +| `df.copy()` (QW4) | **Benign.** All three `OverFlowGraph(...)` sites set `df_of_g["line_name"]` themselves before construction (so the old injection guard never fired) and relabel `g_overflow.g` via `nx.relabel_nodes` directly — they never call `rename_nodes` and never read `g_overflow.df`. The old code therefore never mutated their frame either; the copy is unobservable. | +| `rename_nodes` clarification | **None.** Not called by the recommender. | +| Model/renderer split | **Benign.** `plot` / `set_hubs_shape` / `collapse_red_loops` / `highlight_significant_line_loading` / `highlight_swapped_flows` keep identical signatures and behaviour; the recommender does not import the moved private penwidth constants nor subclass `OverFlowGraph`. | +| `delete_color_edges` multi-colour | **None.** Single-colour calls unchanged; the union form yields byte-identical derived graphs. | +| `Structured_Overload_Distribution_Graph` 3rd param + single-pass views | **None.** `possible_hubs=` stays the 2nd param (the recommender's `try/except TypeError` fallback simply stops triggering); `find_loops` default `None` = original unbounded enumeration; derived colour graphs identical. | +| `get_dispatch_edges_nodes` empty-loop guard | **Behavioural, beneficial.** `_orchestrator.py` calls `get_dispatch_edges_nodes(only_loop_paths=True)` unguarded in its non-antenna branch; on the old library that raises `TypeError` when a grid has no red loops. The guard now returns `([], [])`. The recommender already documents this exact hazard and works around it in antenna mode — it never relied on the crash. | +| `consolidate_graph` / null-flow cutoffs | **None.** Default `None` (unbounded) preserves prior behaviour. | +| `_compute_sssp_paths` `except` narrowing | **None.** `nx.NodeNotFound` ⊂ `nx.NetworkXException`; still caught. | +| `AlphaDeesp_warmStart` | **None.** Signature and constructor body unchanged; the recommender instantiates it and never calls the (vectorised) ranking methods on it. | +| Import surface | **None.** Every symbol the recommender imports (`OverFlowGraph`, `Structured_Overload_Distribution_Graph`, `AlphaDeesp_warmStart`, `Grid2opSimulation`) still resolves through the shim/package. | + +**Consumption path.** The recommender is unaffected until a new `expertop4grid` +is released *and* its pin is bumped. When that happens it works unchanged; it +may optionally drop the antenna-mode skip workaround in `_orchestrator.py` +since the underlying raise is fixed, but this is not required. + +--- + +## Round 2 — deep revisions & open findings implemented + +A second implementation pass cleared the three named deep revisions, the three +open findings (#4/#6/#7), and the interactive-viewer decomposition: + +| Item | Change | Tests | +|---|---|---| +| Deep — finish model/colour inversion | new `graphs/edge_roles.py` (`edge_role_of` / `base_color_of` / `EDGE_ROLE_*`); `tag_constrained_path` reads the role, not the colour string; `highlight_*` records an authoritative `base_color`; new `OverFlowGraph.edge_role(name)` | `test_edge_roles.py`, `test_overflow_graph.py` | +| Deep — `AlphaDeesp` explicit `run()` | pipeline moved to `run()`; `auto_run=True` default keeps backwards-compat; results initialised empty | `test_alphadeesp_unit.py::TestAutoRunSeparation` | +| Deep — structured graph laziness | colour views / `red_loops` / `hubs` are `cached_property`; `red_loops` uses seed hubs, `find_loops()` uses detected hubs (order-independent, behaviour-identical) | `test_graphs_package.py` (caching + order-independence) | +| #4 — `create_df` | vectorised the `iterrows` passes; positional `.iloc[line_to_cut[0]]` | `test_simulation_create_df.py` (400-case differential fuzz vs the original loop) | +| #6 — `shortest_paths.py` | shared `_make_incentivized_weight`; dead branch removed; **multigraph-correct** min-parallel-weight + `(u,v)`/`(u,v,key)` promoted matching | `test_shortest_paths.py::TestMultiDiGraphWeighting` | +| #7 — `to_DiGraph` | missing `capacity` defaults to `0.0` (neutral in the min-cut) not `1.0` | `test_alphadeesp_unit.py::TestToDiGraph` | +| Maint — interactive viewer | `interactive_html.py` (976 LOC) → package `interactive_html/` (8 focused modules, largest 191 LOC) + externalised `assets/{viewer.css,viewer.js,template.html}` reassembled byte-exactly at runtime; shipped via `package_data`/`MANIFEST` | `test_interactive_html.py` (+ asset tests) | + +## Pistes de travail restantes (remaining work) + +Ranked roughly by leverage. Nothing below is required for the shipped changes to +be correct. + +### Maintainability (highest leverage first) +- **Repoint CI static analysis at the `graphs/` package** (review rec. #2, skipped + by request). `.circleci/config.yml` runs pyflakes/mypy on the 49-line + `graphsAndPaths.py` shim, never on `overflow_graph.py` / `null_flow_graph.py` / + etc. — so the cross-repo module has no static gate. +- **Mixins → composition or `typing.Protocol`.** `NullFlowGraphMixin`, + `GraphConsolidationMixin`, `TopologyScorerMixin`, `TopoApplicatorMixin` assume + `self.g` / `self.float_precision` with no enforced contract. +- **`Grid2opSimulation.py` (~814 LOC)** monolith — split by concern. +- **Logging vs `print`** — route the ~40 remaining `print()` calls in the older + backends (`grid2op/`, `pypownet/`, `network.py`, `printer.py`) through `logging`. +- **`null_flow_graph._compute_sssp_paths`** — the multigraph weight subtlety + (issue #1) is now addressed: the weight is precomputed as an edge attribute + (string-weight Dijkstra, perf) with a `capacity_weighted` flag — default + `False` = "bless" (hop-only, **bit-identical** to before), `True` = the + capacity-weighted fix. Callers switch via `add_relevant_null_flow_lines[_all_paths] + (..., capacity_weighted=True)`. The capacity-weighted routing still wants + validation on reference cases (needs grid2op). Follow-up perf headroom noted in + issue #1: target-side reverse Dijkstra and cross-call memoisation. + +### Deeper revisions +- **`Structured_Overload_Distribution_Graph`** consolidation loop still rebuilds the + whole object each iteration (inherent to the algorithm; the views are now lazy so + each rebuild computes only what it touches). An incremental-update redesign is the + next step if consolidation becomes a hotspot. + +### Interface / naming (backward-compatible only) +- PEP8 forwarding aliases + a deprecation clock for the `camelCase` + (`isAntenna`, …) and `Structured_Overload_Distribution_Graph` names. +- Accept both spellings of the misspelled config key `ThersholdMinPowerOfLoop`. +- A short `amont`/`aval` (upstream/downstream) glossary in the docs. + +### Validation & release +- **Run the grid2op integration suites in CI** (`alphadeesp_test.py`, + `test_expert_op.py`, `test_expert_rules.py`, `grid2op/`) — not runnable in this + sandbox. With the cutoffs defaulting to `None` the graph behaviour is unchanged + from `master`, but this should be confirmed on real grids. +- **Release + bump.** Publishing a new `expertop4grid` and bumping the pin in + `Expert_op4grid_recommender` is what actually delivers these changes downstream; + the recommender may then optionally drop its antenna-mode `get_dispatch_edges_nodes` + workaround. diff --git a/docs/DETAILS.rst b/docs/DETAILS.rst index 7d86afdd..4f346811 100644 --- a/docs/DETAILS.rst +++ b/docs/DETAILS.rst @@ -7,20 +7,25 @@ Call Calling the alphaDeesp engine is done like so : -``alphadeesp = AlphaDeesp(g_over, df_of_g, custom_layout, printer, simulator_data,sim.substation_in_cooldown, debug = debug)`` +``alphadeesp = AlphaDeesp(g_over, df_of_g, simulator_data, sim.substation_in_cooldown, debug=debug)`` ``ranked_combinations = alphadeesp.get_ranked_combinations()`` -Alphadeesp hence gives you an oredered list of substations and topologies that should be relevant to solve your overload +By default the ranking pipeline runs in the constructor. Pass ``auto_run=False`` +and call ``alphadeesp.run()`` for staged/testable execution. + +Alphadeesp hence gives you an ordered list of substations and topologies that should be relevant to solve your overload Inputs ====== The following inputs will be required to be computed by the Simulation override. * ``g_over`` - A newtorkx graph representation of the grid with flow values + A networkx ``MultiDiGraph`` representation of the grid with flow values (an + :class:`~alphaDeesp.core.graphs.overflow_graph.OverFlowGraph`'s ``.g``). * ``df_of_g`` - A dataframe representing a detailed view of the graph + A dataframe representing a detailed view of the graph (one row per line; + see :meth:`~alphaDeesp.core.simulation.Simulation.get_dataframe`). .. image:: ../alphaDeesp/ressources/df_of_g_l9PNG.png @@ -30,12 +35,6 @@ The following inputs will be required to be computed by the Simulation override. .. image:: ../alphaDeesp/ressources/g_over_df_l9.png -* ``custom_layout`` - The layout of the graph (list of (X,Y) coordinate for edges. Used for plotting. - -* ``printer`` - A printer service for logs and graphs - * ``simulator_data`` A dict composed of : diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/docs/conf.py b/docs/conf.py index ff1c73d1..891301cc 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,12 +7,11 @@ # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +# add these directories to sys.path here. So that autodoc can import the package, +# put the repository root on the path. +import os +import sys +sys.path.insert(0, os.path.abspath('..')) # -- Project information ----------------------------------------------------- @@ -22,7 +21,7 @@ author = 'Antoine Marot, Mario Jothy, Nicolas Megel' # The full version, including alpha/beta/rc tags -release = '0.3.2.post3' +release = '0.3.3' # -- General configuration --------------------------------------------------- @@ -31,8 +30,25 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx_rtd_theme' + 'sphinx_rtd_theme', + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', # NumPy / Google style docstrings + 'sphinx.ext.viewcode', +] + +# Heavy optional backends must not be required to build the API docs — the +# documented modules (the graphs package, Simulation, elements) do not import +# them, but mocking keeps a full-tree build robust on doc-only environments. +autodoc_mock_imports = [ + 'grid2op', 'lightsim2grid', 'pandapower', 'pypownet', 'oct2py', 'pypower', ] +autodoc_default_options = { + 'members': True, + 'undoc-members': False, + 'show-inheritance': True, +} +napoleon_google_docstring = True +napoleon_numpy_docstring = True # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index 67ee937d..a4e00fbb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,3 +20,10 @@ Welcome to ExpertOp4Grid's documentation! DESCRIPTION.rst DETAILS.rst + +.. toctree:: + :maxdepth: 2 + :caption: Developer documentation + + ARCHITECTURE.rst + API.rst diff --git a/setup.py b/setup.py index c621d7be..b1fcaa3b 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ } setup(name='ExpertOp4Grid', - version='0.3.2.post4', + version='0.3.3', description='Expert analysis algorithm for solving overloads in a powergrid', long_description_content_type="text/markdown", python_requires=">=3.9", @@ -50,11 +50,12 @@ author='Antoine Marot', author_email='antoine.marot@rte-france.com', url="https://github.com/marota/ExpertOp4Grid/", - download_url = 'https://github.com/marota/ExpertOp4Grid/archive/refs/tags/v0.3.2.post4.tar.gz', + download_url = 'https://github.com/marota/ExpertOp4Grid/archive/refs/tags/v0.3.3.tar.gz', license='Mozilla Public License 2.0 (MPL 2.0)', packages=setuptools.find_packages(), extras_require=pkgs["extras"], include_package_data=True, + package_data={"alphaDeesp.core.interactive_html": ["assets/*"]}, install_requires=pkgs["required"], zip_safe=False, entry_points={'console_scripts': ['expertop4grid=alphaDeesp.main:main']},