diff --git a/README.md b/README.md index 2293aa0..b5941b9 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,11 @@ Each pathway generates a directory under `output/`: ``` output/_R-HSA-/ -├── logic_network.csv # Main output: edges of the perturbation graph -├── stid_to_uuid_mapping.csv # UUID → Reactome stable ID +├── logic_network.csv # The curated network: edges from Reactome reactions +├── boundary_edges.csv # Derived: root/terminal complexes decomposed (same columns) +├── stid_to_uuid_mapping.csv # UUID → Reactome stable ID (nodes of both files) +├── containment.csv # What each entity contains, flattened to leaves +├── containment_structure.csv # What each entity is made of, one level at a time, by relation └── cache/ ├── reaction_connections.csv ├── decomposed_uid_mapping.csv @@ -74,10 +77,10 @@ output/_R-HSA-/ | `target_id` | UUID of target node | | `pos_neg` | `pos` (activates / produces) or `neg` (negative regulator) | | `and_or` | `and` (required), `or` (alternative source), or empty (single producer) | -| `edge_type` | `input`, `output`, `catalyst`, `regulator`, `assembly`, or `dissociation` | +| `edge_type` | `input`, `output`, `catalyst`, `regulator`, `set_member`, `depletion`; in `boundary_edges.csv`, `assembly` or `dissociation` | | `stoichiometry` | Stoichiometric coefficient from Reactome | -`assembly` and `dissociation` edges only appear at boundaries: a leaf protein assembles into a root-input complex, or a terminal-output complex dissociates into its components. +`logic_network.csv` holds what curators curated. The generator also decomposes root-input and terminal-output complexes so a single protein can be perturbed or read: a leaf assembles into a root complex (`assembly`), a terminal complex dissociates into its components (`dissociation`). Those edges are **not curated**, so they are written to `boundary_edges.csv` (same columns), not to `logic_network.csv`. Load both for the decomposed network (DeltaSignal does); load `logic_network.csv` alone for the network as curated, where root complexes are roots again. `containment_structure.csv` gives each entity's structure one level at a time, with the relation (`component`, set `member`, or `candidate`) and stoichiometry, so a consumer can tell a required subunit from one alternative of a set. ## Validation diff --git a/src/logic_network_generator.py b/src/logic_network_generator.py index 921706e..ecfdd50 100755 --- a/src/logic_network_generator.py +++ b/src/logic_network_generator.py @@ -33,6 +33,11 @@ class PathwayResult(NamedTuple): catalyst_regulator_map: pd.DataFrame reaction_id_map: pd.DataFrame entity_uuid_registry: Dict[tuple, str] = {} + # One flag per logic_network row: True for an edge the generator derived + # from complex/set STRUCTURE at a root input or terminal output (assembly, + # dissociation, composition), not from a curated reaction. Written to + # boundary_edges.csv instead of logic_network.csv (deltasignal specs/044). + boundary_mask: List[bool] = [] def _get_reactome_id_from_hash(decomposed_uid_mapping: pd.DataFrame, hash_value: str) -> str: @@ -1766,6 +1771,20 @@ def _emit_composition_edges( def _emit_boundary_decomposition_edges( pathway_logic_network_data: List[Dict[str, Any]], reactome_id_to_uuid: Dict[str, str], +) -> None: + n_before = len(pathway_logic_network_data) + try: + _emit_boundary_decomposition_edges_inner(pathway_logic_network_data, reactome_id_to_uuid) + finally: + # Every edge this pass adds is derived from structure, not curated: + # tag it so the export can keep it out of logic_network.csv. + for e in pathway_logic_network_data[n_before:]: + e["_boundary"] = True + + +def _emit_boundary_decomposition_edges_inner( + pathway_logic_network_data: List[Dict[str, Any]], + reactome_id_to_uuid: Dict[str, str], ) -> None: """Expose the members of every root-input and terminal-output complex. @@ -2788,6 +2807,7 @@ def create_pathway_logic_network( # Create final DataFrame pathway_logic_network = pd.DataFrame(pathway_logic_network_data, columns=list(columns.keys())) + boundary_mask = [bool(e.get("_boundary")) for e in pathway_logic_network_data] # Coerce stoichiometry to nullable Int64 — emission sites use a mix of # int (`1`) and float (`1.0`) literals, which makes pandas infer float64 # for the column and serialize as `1.0` in the CSV. Force integer so the @@ -2821,6 +2841,7 @@ def create_pathway_logic_network( catalyst_regulator_map=catalyst_regulator_uuid_map, reaction_id_map=reaction_id_map, entity_uuid_registry=entity_uuid_registry, + boundary_mask=boundary_mask, ) def find_root_inputs(pathway_logic_network: pd.DataFrame) -> List[Any]: @@ -3310,6 +3331,65 @@ def export_containment(reactome_id_to_uuid: Dict[str, str], f"of {len(leaves_by_stid)} distinct") +_STRUCTURE_RELATION = {"hasComponent": "component", "hasMember": "member", + "hasCandidate": "candidate"} + + +def export_containment_structure(reactome_id_to_uuid: Dict[str, str], + output_file: str) -> None: + """Write containment_structure.csv: what each entity is made of, ONE level + at a time, with the kind of relation (deltasignal specs/044). + + containment.csv flattens to leaves, so it cannot say whether a protein is a + required component of a complex or one alternative of a set inside it; a + consumer reading it would make every alternative required (LNG code review + 2026-10-02, F7). This file is Reactome's own structure, unflattened and + with no generator rule applied (modifier sets are walked like any other): + + - parent_stable_id, parent_type: the container and its class + (Complex, DefinedSet, CandidateSet, OpenSet, Polymer, ...) + - child_stable_id + - relation: component (Complex hasComponent), member (set hasMember), + candidate (CandidateSet hasCandidate) + - stoichiometry: for a component; empty for set membership + - reactome_release + + Every entity in the network and everything below it is walked. + """ + from src.neo4j_connector import get_graph, get_reactome_release + release = get_reactome_release() + graph = get_graph() + rows = [] + seen: Set[str] = set() + frontier = sorted({str(v) for v in reactome_id_to_uuid.values() if v}) + while frontier: + seen.update(frontier) + found = graph.run( + """UNWIND $ids AS id + MATCH (p:PhysicalEntity {stId: id})-[r:hasComponent|hasMember|hasCandidate]->(c:PhysicalEntity) + RETURN p.stId AS parent, [l IN labels(p) WHERE l IN ['Complex','DefinedSet','CandidateSet', + 'OpenSet','EntitySet','Polymer']] AS ptype, + c.stId AS child, type(r) AS rel, r.stoichiometry AS stoich""", + ids=frontier).data() + nxt: Set[str] = set() + for r in found: + ptype = [t for t in r["ptype"] if t != "EntitySet"] or r["ptype"] or [""] + relation = _STRUCTURE_RELATION[r["rel"]] + rows.append({"parent_stable_id": r["parent"], "parent_type": ptype[0], + "child_stable_id": r["child"], "relation": relation, + "stoichiometry": r["stoich"] if relation == "component" else None, + "reactome_release": release}) + if r["child"] not in seen: + nxt.add(r["child"]) + frontier = sorted(nxt) + df = pd.DataFrame(rows, columns=["parent_stable_id", "parent_type", "child_stable_id", + "relation", "stoichiometry", "reactome_release"]) + df = df.drop_duplicates().sort_values(["parent_stable_id", "relation", "child_stable_id"]) + df["stoichiometry"] = df["stoichiometry"].astype("Int64") + df.to_csv(output_file, index=False) + logger.info(f"Containment structure: {len(df)} rows over {len(seen)} entities") + + def export_cofactors(pathway_logic_network: pd.DataFrame, reactome_id_to_uuid: Dict[str, str], output_file: str) -> None: diff --git a/src/pathway_generator.py b/src/pathway_generator.py index 0fa3b85..765b960 100755 --- a/src/pathway_generator.py +++ b/src/pathway_generator.py @@ -15,6 +15,7 @@ export_drugs, export_pools, export_containment, + export_containment_structure, export_entity_reaction_proxy_mapping, export_node_reaction_context, export_node_resolution, @@ -216,7 +217,8 @@ def _write_cache_fingerprint( BUNDLE_FILES = ( "logic_network.csv", "stid_to_uuid_mapping.csv", "entity_reaction_proxy_mapping.csv", "nodes.csv", "node_reaction_context.csv", "node_resolution.csv", "node_exclusions.csv", - "cofactors.csv", "containment.csv", "drugs.csv", + "boundary_edges.csv", "cofactors.csv", "containment.csv", "containment_structure.csv", + "drugs.csv", "pools.csv", "pool_transitions.csv", "pool_carriers.csv", ) @@ -429,11 +431,25 @@ def generate_pathway_file( ) # Save logic network (main output file users need) + # logic_network.csv holds what curators curated. The edges the + # generator DERIVES from complex/set structure at root inputs and + # terminal outputs (assembly, dissociation, composition) go to + # boundary_edges.csv, same columns, so a consumer that wants the + # curated network alone can have it, and one that wants the + # decomposition loads both (deltasignal specs/044). output_file = pathway_output_dir / "logic_network.csv" + boundary_file = pathway_output_dir / "boundary_edges.csv" try: - result.logic_network.to_csv(output_file, index=False) + net = result.logic_network + flags = list(result.boundary_mask) or [False] * len(net) + if len(flags) != len(net): + raise ValueError(f"boundary mask has {len(flags)} flags for {len(net)} edges") + mask = pd.Series(flags, index=net.index, dtype=bool) + net[~mask].to_csv(output_file, index=False) + net[mask].to_csv(boundary_file, index=False) logger.info(f"Successfully generated logic network: {output_file}") - logger.info(f"Network contains {len(result.logic_network)} edges") + logger.info(f"Network contains {int((~mask).sum())} curated edges and " + f"{int(mask.sum())} boundary edges") except IOError as e: logger.error(f"Failed to write output file {output_file}: {e}") raise @@ -511,6 +527,10 @@ def generate_pathway_file( result.uuid_mapping, str(pathway_output_dir / "containment.csv"), ) + export_containment_structure( + result.uuid_mapping, + str(pathway_output_dir / "containment_structure.csv"), + ) except Exception as e: # These are not supplementary: containment.csv, cofactors.csv and # node_resolution.csv feed the consumer's default model. Swallowing diff --git a/tests/test_boundary_edges_file.py b/tests/test_boundary_edges_file.py new file mode 100644 index 0000000..38e634b --- /dev/null +++ b/tests/test_boundary_edges_file.py @@ -0,0 +1,53 @@ +"""Edges derived from complex/set structure at root inputs and terminal outputs +go to boundary_edges.csv; logic_network.csv holds only what was curated +(deltasignal specs/044).""" +import pandas as pd + +import src.logic_network_generator as lng +import src.pathway_generator as pg +from src.logic_network_generator import PathwayResult +from tests.test_export_failfast import mocked # noqa: F401 (fixture) + + +def test_the_boundary_pass_tags_exactly_what_it_adds(monkeypatch): + data = [{"source_id": "r", "target_id": "x", "edge_type": "input"}] + + def inner(d, _m): + d.append({"source_id": "a", "target_id": "X", "edge_type": "assembly"}) + d.append({"source_id": "Y", "target_id": "s", "edge_type": "dissociation"}) + monkeypatch.setattr(lng, "_emit_boundary_decomposition_edges_inner", inner) + lng._emit_boundary_decomposition_edges(data, {}) + assert [bool(e.get("_boundary")) for e in data] == [False, True, True] + + +def test_the_export_splits_by_the_mask(tmp_path, mocked): # noqa: F811 + net = pd.DataFrame([ + {"source_id": "x", "target_id": "y", "pos_neg": "pos", "and_or": "and", + "edge_type": "input", "stoichiometry": 1, "edge_reaction_id": "R-HSA-1"}, + {"source_id": "a", "target_id": "x", "pos_neg": "pos", "and_or": "and", + "edge_type": "assembly", "stoichiometry": 1, "edge_reaction_id": None}, + ]) + mocked.setattr(pg, "create_pathway_logic_network", lambda *a, **k: PathwayResult( + net, {"x": "R-HSA-2"}, pd.DataFrame(), pd.DataFrame({"uid": [], "reactome_id": []}), {}, + boundary_mask=[False, True])) + pg.generate_pathway_file("R-HSA-1", "x", str(tmp_path)) + d = tmp_path / "R-HSA-1" + curated = pd.read_csv(d / "logic_network.csv") + boundary = pd.read_csv(d / "boundary_edges.csv") + assert list(curated["edge_type"]) == ["input"] + assert list(boundary["edge_type"]) == ["assembly"] + assert list(curated.columns) == list(boundary.columns) + + +def test_a_mask_of_the_wrong_length_fails(tmp_path, mocked): # noqa: F811 + net = pd.DataFrame([{"source_id": "x", "target_id": "y", "pos_neg": "pos", "and_or": "and", + "edge_type": "input", "stoichiometry": 1}]) + mocked.setattr(pg, "create_pathway_logic_network", lambda *a, **k: PathwayResult( + net, {"x": "R-HSA-2"}, pd.DataFrame(), pd.DataFrame({"uid": [], "reactome_id": []}), {}, + boundary_mask=[False, True])) + try: + pg.generate_pathway_file("R-HSA-1", "x", str(tmp_path)) + except Exception as e: + assert "boundary mask" in str(e) + else: + raise AssertionError("a misaligned mask must fail the pathway") diff --git a/tests/test_export_failfast.py b/tests/test_export_failfast.py index 13fd603..4121e30 100644 --- a/tests/test_export_failfast.py +++ b/tests/test_export_failfast.py @@ -23,7 +23,8 @@ EXPORTS = ["export_uuid_to_reactome_mapping", "export_entity_reaction_proxy_mapping", "export_nodes", "export_node_reaction_context", "export_node_resolution", - "export_cofactors", "export_containment", "export_drugs", "export_pools"] + "export_cofactors", "export_containment", "export_containment_structure", + "export_drugs", "export_pools"] @pytest.fixture