feat: steer lineages by shared subject, not plausibility alone

src/api/app.py

@@ -126,7 +126,9 @@ def lineage(
     ) -> Lineage:
         graph = loaded.graph
         _present(graph, work_id)
-        learned = learned_flow(graph, work_id, loaded.model, embedding=loaded.embedding)
+        learned = learned_flow(
+            graph, work_id, loaded.model, embedding=loaded.embedding, content=loaded.content
+        )
         ranked = rank(learned, top)
         shown = [work_id, *(node for node, _ in ranked)]
         return Lineage(
@@ -155,7 +157,9 @@ def narrative(
     ) -> Account:
         graph = loaded.graph
         _present(graph, work_id)
-        scores = learned_flow(graph, work_id, loaded.model, embedding=loaded.embedding)
+        scores = learned_flow(
+            graph, work_id, loaded.model, embedding=loaded.embedding, content=loaded.content
+        )
         if not scores:
             raise HTTPException(404, f"{work_id} has no ancestors in this field's snapshot")
         try:

src/api/state.py

@@ -20,6 +20,7 @@
 import networkx as nx
 from torch import Tensor
 
+from gnn.content import Content
 from gnn.model import LinkPredictor
 from gnn.originators import embed
 from gnn.train import load_model
@@ -38,6 +39,7 @@ class Field:
     graph: nx.DiGraph
     model: LinkPredictor
     embedding: tuple[dict[str, int], Tensor]
+    content: Content
 
 
 class Library:
@@ -77,4 +79,6 @@ def _build(self, name: str) -> Field:
         logger.info("loading field %s from %s", name, snapshot)
         graph = load_snapshot(snapshot)
         model = load_model(snapshot)
-        return Field(load_field(name, self.fields_dir), graph, model, embed(graph, model))
+        return Field(
+            load_field(name, self.fields_dir), graph, model, embed(graph, model), Content(graph)
+        )

src/gnn/PLAN.md

@@ -53,21 +53,22 @@ One PR each, named for what it does.
   - Hashed title words: -0.02, overfits.
   - `topic_id`: +0.06, but it leaks. OpenAlex assigns topics from citations as of the crawl, the reason `cited_by_count` is excluded.
   - Held-out share, learning rate, depth, width, dropout, weight decay, and MRR instead of AP for checkpoint selection: nothing beyond noise.
-- [ ] **Originator ranking evaluation.** `gnn.lineages` is built: Recall@10 and nDCG@10 over reachable gold originators for the learned ranking, the same walk with equal weights, and every baseline; coverage apart; top-10 overlap across 3 training seeds and with 10% of citations dropped. Waiting on the gold set, `fields/gold/gnn.yaml` (not `fields/*.yaml`, which the API lists as fields): 18 targets drafted, being verified. On the draft, served snapshot, before and after recovering citations from Semantic Scholar:
+- [x] **Originator ranking evaluation.** `gnn.lineages` against `fields/gold/gnn.yaml`: 23 papers and 63 originators, each checked against how the paper cites the work, split into `dev` (may steer changes) and `test` (report only) before any tuning. The walk now also follows shared subject (`gnn.content`, TF-IDF over titles and abstracts), chosen on dev only. Served snapshot:
 
-  | Scorer | Recall@10 | nDCG@10 | Overlap, 10% dropped | Gold found |
+  | Scorer | dev found | dev Recall@10 / nDCG@10 | test found | test Recall@10 / nDCG@10 |
   |---|---|---|---|---|
-  | learned | 0.484 / 0.465 | 0.466 / 0.439 | 0.844 / 0.894 | 45 / 44 |
-  | equal-weight walk | 0.410 / 0.397 | 0.366 / 0.333 | 0.844 / 0.867 | 38 / 38 |
-  | time_decayed_pagerank | 0.432 / 0.401 | 0.380 / 0.352 | 0.833 / 0.872 | 40 / 38 |
-  | in_degree | 0.127 / 0.097 | 0.076 / 0.060 | 0.833 / 0.861 | |
-  | gateway | 0.063 / 0.093 | 0.081 / 0.117 | 0.811 / 0.778 | |
-  | path_weight | 0.000 / 0.000 | 0.000 / 0.000 | 0.900 / 0.883 | |
-
-  Coverage 90/94, then 92/94; learned top-10 overlap across seeds 0.94 to 0.95. The learned ranking leads, and the model's weights are worth +0.07 recall over the same walk unweighted. Recovered citations make more gold originators reachable and every ranking steadier when citations go missing, but find no more of them: recall is over reachable originators, so it drops as the denominator grows. `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent.
+  | learned (model and content, served) | 23/32 | 0.903 / 0.717 | 17/31 | 0.626 / 0.556 |
+  | model only | 22/32 | 0.806 / 0.650 | 16/31 | 0.523 / 0.340 |
+  | equal-weight walk | 22/32 | 0.806 / 0.571 | 14/31 | 0.474 / 0.344 |
+  | time_decayed_pagerank | 21/32 | 0.778 / 0.567 | 14/31 | 0.474 / 0.303 |
+  | gateway | 5/32 | 0.106 / 0.090 | 4/31 | 0.167 / 0.080 |
+  | in_degree | 2/32 | 0.097 / 0.092 | 2/31 | 0.076 / 0.038 |
+  | path_weight | 0/32 | 0 / 0 | 0/31 | 0 / 0 |
+
+  Learned top-10 overlap across seeds 0.94 dev, 0.97 test; with 10% of citations dropped 0.92 and 0.93. Tried on dev and left out: walk damping, age half-life and sharpening the model's weights (a hit either way, noise at 12 papers), and Semantic Scholar's influence rate per work (Adam's is higher than ChebNet's). `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent.
 - [x] **Recovered citations.** `graph.semanticscholar`, an optional stage between crawl and build: 6811 MAG-era works' Semantic Scholar references matched back to crawled works by MAG id, else by title and year. 22582 citations matched, 6808 of them lost from OpenAlex; GCN regains Bruna, Henaff and Weston. Served.
-- [ ] **Citation influence labels.** Semantic Scholar `isInfluential` per citation; per-paper AUC of the decoder on influential vs incidental references.
-- [ ] **Influence-aligned objective.** Fine-tune the decoder listwise on influential references, link prediction as pretraining. Validate on the gold set.
+- [x] **Citation influence labels.** 5202 Semantic Scholar `isInfluential` labels from 141 non-gold papers. The decoder's plausibility alone predicts them at AUC 0.70, hand features at 0.71, and the model's embeddings add nothing. The flag mostly counts how often a paper mentions a work, so tools earn it too.
+- [x] **Influence-aligned objective.** Not adopted. Walk weights from a model of those labels tie content weighting on dev (24 against 23 of 32 found) and add nothing on top of it, for an extra pipeline stage and a rate-limited API at ranking time.
 - [x] **Crawl expansion.** Uncapped backward crawl, plus `forward_depth: 1`: the in-topic works citing the seeds. 2789 works and 15k citations became 9249 and 104k. GAT is seeded by its primary record. Cleaning re-dates 262 works OpenAlex dates years after their citers, and a year heavier than its split's share no longer empties the split after it. Gold originators reachable from their targets: 88/94 to 90/94. The other four are references OpenAlex holds under dead ids (about 12% of all references), among them GCN's to Bruna, Henaff and Planetoid.
 
 Optional: blinded LLM pairwise judge, trusted only after it agrees with the gold set.

src/gnn/content.py

@@ -0,0 +1,67 @@
+"""How much two works share in what they are about, from their titles and abstracts.
+
+A paper is built on work about the same things. An optimiser, a dataset or a
+regularisation trick is cited by nearly every paper and is about none of them,
+and in the citation graph alone it looks just like a real precursor: both are
+hubs. TF-IDF over titles and abstracts tells them apart, with no model to train
+and no dependency to add.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+
+import networkx as nx
+
+from graph.build import is_missing
+
+# Common English only. A word common to a whole field is discounted by its own
+# document frequency, so no field's vocabulary has to be written down here.
+# fmt: off
+STOPWORDS = frozenset((
+    "the", "and", "for", "with", "that", "this", "from", "are", "our", "can", "which", "have",
+    "has", "been", "using", "used", "use", "new", "two", "its", "their", "these", "such",
+    "into", "than", "also", "based", "via", "over", "more", "not", "all", "one", "only",
+    "between", "both", "each", "other",
+))
+# fmt: on
+
+_WORD = re.compile(r"[a-z][a-z0-9-]{2,}")
+
+
+class Content:
+    """A unit TF-IDF vector for every work in a graph."""
+
+    def __init__(self, graph: nx.DiGraph) -> None:
+        counts = {node: Counter(_words(graph.nodes[node])) for node in graph}
+        frequency = Counter(word for words in counts.values() for word in words)
+        self._vectors = {
+            node: _unit(
+                {
+                    word: (1 + math.log(count)) * math.log(len(counts) / frequency[word])
+                    for word, count in words.items()
+                }
+            )
+            for node, words in counts.items()
+        }
+
+    def similarity(self, a: str, b: str) -> float:
+        """Cosine similarity: 1 for the same words, 0 for none shared or none on record."""
+        left, right = self._vectors.get(a, {}), self._vectors.get(b, {})
+        if len(left) > len(right):
+            left, right = right, left
+        return sum(weight * right.get(word, 0.0) for word, weight in left.items())
+
+
+def _words(attributes: dict) -> list[str]:
+    text = " ".join(
+        str(value) for key in ("title", "abstract") if not is_missing(value := attributes.get(key))
+    )
+    return [word for word in _WORD.findall(text.lower()) if word not in STOPWORDS]
+
+
+def _unit(vector: dict[str, float]) -> dict[str, float]:
+    norm = math.sqrt(sum(value * value for value in vector.values())) or 1.0
+    return {word: value / norm for word, value in vector.items()}

src/gnn/lineages.py

@@ -32,6 +32,7 @@
 from pydantic import BaseModel
 from torch import Tensor
 
+from gnn.content import Content
 from gnn.dataset import build_dataset
 from gnn.model import LinkPredictor
 from gnn.originators import embed, flow, learned_flow
@@ -69,11 +70,13 @@ def rankings(
     target: str,
     model: LinkPredictor,
     embedding: tuple[dict[str, int], Tensor],
+    content: Content,
     top: int,
 ) -> dict[str, list[str]]:
-    """Every scorer's top works for one paper: the model, its walk unweighted, the baselines."""
+    """Every scorer's top works for one paper: as served, without content, unweighted, baselines."""
     scores = {
-        "learned": learned_flow(graph, target, model, embedding=embedding),
+        "learned": learned_flow(graph, target, model, embedding=embedding, content=content),
+        "model_only": learned_flow(graph, target, model, embedding=embedding),
         "uniform_flow": flow(acyclic(ancestry(graph, target)), target),
         **{name: scorer(graph, target) for name, scorer in SCORERS.items()},
     }
@@ -129,6 +132,7 @@ def assess(
             logger.warning("gold target %s is not in the graph", lineage.target)
 
     embedding = embed(graph, model)
+    content = Content(graph)
     thinned = _thinned(graph, dropout, seed=0)
     thinned_embedding = embed(thinned, model)
     others = [
@@ -141,10 +145,15 @@ def assess(
     for lineage in lineages:
         reachable = set(ancestry(graph, lineage.target))
         findable = [ids for ids in lineage.originators if reachable.intersection(ids)]
-        ranked = rankings(graph, lineage.target, model, embedding, top)
-        stressed = rankings(thinned, lineage.target, model, thinned_embedding, top)
+        ranked = rankings(graph, lineage.target, model, embedding, content, top)
+        stressed = rankings(thinned, lineage.target, model, thinned_embedding, content, top)
         learned = [ranked["learned"]] + [
-            [node for node, _ in rank(learned_flow(graph, lineage.target, other, embedding=e), top)]
+            [
+                node
+                for node, _ in rank(
+                    learned_flow(graph, lineage.target, other, embedding=e, content=content), top
+                )
+            ]
             for other, e in zip(others, other_embeddings, strict=True)
         ]
         title = graph.nodes[lineage.target].get("title")

src/gnn/originators.py

@@ -6,6 +6,10 @@
 decoder: at each step backwards the walk prefers the citations the model finds
 most plausible, which is the one thing structure alone cannot tell it.
 
+A plausible citation can still be an incidental one: every paper cites its
+optimiser. So each step also follows how much the two works share in subject,
+from their titles and abstracts, which is what tells a precursor from a tool.
+
 The walk is then decayed by age. Backwards through a citation graph there is
 always another ancestor, and without decay the ranking ends at the founding of
 the discipline rather than at the work that made this paper possible.
@@ -22,6 +26,7 @@
 from torch import Tensor
 from torch_geometric.utils import to_undirected
 
+from gnn.content import Content
 from gnn.dataset import observed_view
 from gnn.model import LinkPredictor
 from gnn.train import load_model
@@ -31,6 +36,11 @@
 
 logger = logging.getLogger(__name__)
 
+# Square-rooted, so shared subject tempers the model rather than overruling it,
+# and floored, so a work with no abstract on record still gets a share.
+CONTENT_WEIGHT = 0.5
+CONTENT_FLOOR = 0.02
+
 
 def embed(graph: nx.DiGraph, model: LinkPredictor) -> tuple[dict[str, int], Tensor]:
     """One encode pass over everything on record. Where each node sits, and as what."""
@@ -63,16 +73,20 @@ def learned_flow(
     damping: float = 0.85,
     half_life: float = 10.0,
     embedding: tuple[dict[str, int], Tensor] | None = None,
+    content: Content | None = None,
 ) -> Scores:
-    """Weight of a damped walk backwards from `target`, steered by the model."""
+    """Weight of a damped walk backwards from `target`, steered by the model and by content."""
     lineage = acyclic(ancestry(graph, target))
     position, z = embedding or embed(graph, model)
 
     edges = list(lineage.edges())
-    plausibility = dict(
-        zip(edges, citation_probability(model, z, position, edges).tolist(), strict=True)
-    )
-    return flow(lineage, target, plausibility, damping=damping, half_life=half_life)
+    weight = dict(zip(edges, citation_probability(model, z, position, edges).tolist(), strict=True))
+    if content is not None:
+        weight = {
+            edge: share * (CONTENT_FLOOR + content.similarity(*edge)) ** CONTENT_WEIGHT
+            for edge, share in weight.items()
+        }
+    return flow(lineage, target, weight, damping=damping, half_life=half_life)
 
 
 def flow(
@@ -118,7 +132,7 @@ def score_target(
     graph = load_snapshot(snapshot)
     model = load_model(snapshot)
     return (
-        learned_flow(graph, target, model, half_life=half_life),
+        learned_flow(graph, target, model, half_life=half_life, content=Content(graph)),
         {name: scorer(graph, target) for name, scorer in SCORERS.items()},
     )
 

tests/test_api.py

@@ -7,6 +7,7 @@
 
 from api.app import create_app
 from api.state import Field, Library
+from gnn.content import Content
 from graph.config import load_field
 from llm.base import ProviderError
 from llm.narrative import Claim, Judgement, Narrative, Verdict
@@ -73,6 +74,7 @@ def _build(self, name):
             self.graph,
             Indifferent(),
             (position, torch.zeros(len(position), 2)),
+            Content(self.graph),
         )
 
 

tests/test_content.py

@@ -0,0 +1,43 @@
+import networkx as nx
+import pytest
+
+from gnn.content import Content
+
+
+def works(**texts):
+    graph = nx.DiGraph()
+    for node, (title, abstract) in texts.items():
+        graph.add_node(node, title=title, abstract=abstract)
+    return graph
+
+
+def test_the_same_words_are_the_same_subject_and_none_shared_is_none():
+    content = Content(
+        works(
+            A=("Graph attention networks", "Attention over neighbours."),
+            B=("Graph attention networks", "Attention over neighbours."),
+            C=("Stochastic optimisation", "Adaptive moment estimates."),
+        )
+    )
+
+    assert content.similarity("A", "B") == pytest.approx(1.0)
+    assert content.similarity("A", "C") == 0.0
+
+
+def test_case_and_common_english_carry_no_subject():
+    content = Content(
+        works(
+            A=("THE Attention", "With the attention."),
+            B=("attention for the", None),
+            C=("Kernels", "Unrelated."),
+        )
+    )
+
+    assert content.similarity("A", "B") == pytest.approx(1.0)
+
+
+def test_a_work_with_nothing_on_record_shares_nothing():
+    content = Content(works(A=("Attention", "Attention."), B=(None, float("nan"))))
+
+    assert content.similarity("A", "B") == 0.0
+    assert content.similarity("A", "NOT-IN-GRAPH") == 0.0

tests/test_lineages.py

@@ -78,7 +78,7 @@ def test_an_assessment_judges_covers_and_stresses_every_scorer():
 
     assert result["targets_in_graph"] == "2/3"
     assert (dev["coverage"], test["coverage"]) == ("1/2", "1/1")
-    assert set(dev["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"}
+    assert set(dev["scorers"]) >= {"learned", "model_only", "uniform_flow", "in_degree"}
     assert dev["scorers"]["learned"]["recall"] == 1.0
     assert dev["scorers"]["learned"]["found"] == "1/2"
     assert "U" not in result["targets"][0]["scorers"]["learned"]["top"]

tests/test_originators.py

@@ -5,6 +5,7 @@
 import pytest
 import torch
 
+from gnn.content import Content
 from gnn.dataset import build_dataset
 from gnn.originators import embed, learned_flow, score_target
 from gnn.train import TrainConfig, save_model, train
@@ -74,6 +75,21 @@ def test_structural_twins_are_separated_by_the_model_alone():
     assert scores["A"] > scores["B"]
 
 
+def test_structural_twins_are_separated_by_what_they_are_about():
+    """A precursor and a tool the model finds equally plausible: content tells them apart."""
+    graph = lineage(DIAMOND)
+    abstracts = {"T": "graph attention", "A": "attention alignment", "B": "stochastic optimiser"}
+    for node, abstract in abstracts.items():
+        graph.nodes[node]["abstract"] = abstract
+    indifferent = FixedOpinion(graph, graph.edges)
+
+    plain = learned_flow(graph, "T", indifferent)
+    informed = learned_flow(graph, "T", indifferent, content=Content(graph))
+
+    assert plain["A"] == plain["B"]
+    assert informed["A"] > informed["B"]
+
+
 def test_the_distant_past_is_discounted():
     graph = lineage([("T", "RECENT"), ("T", "ANCIENT")], years={"T": 2020, "RECENT": 2018})
     graph.nodes["ANCIENT"]["publication_year"] = 1970