From cb81520c25890ef7abf1768544e8a9fb99a8ad4f Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:34:26 +0300 Subject: [PATCH 1/4] feat: measure how much two works share in subject, from titles and abstracts --- src/gnn/content.py | 67 +++++++++++++++++++++++++++++++++++++++++++ tests/test_content.py | 43 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 src/gnn/content.py create mode 100644 tests/test_content.py diff --git a/src/gnn/content.py b/src/gnn/content.py new file mode 100644 index 0000000..2ea8c03 --- /dev/null +++ b/src/gnn/content.py @@ -0,0 +1,67 @@ +"""How much two works share in what they are about, from their titles and abstracts. + +A paper is built on work about the same things. An optimiser, a dataset or a +regularisation trick is cited by nearly every paper and is about none of them, +and in the citation graph alone it looks just like a real precursor: both are +hubs. TF-IDF over titles and abstracts tells them apart, with no model to train +and no dependency to add. +""" + +from __future__ import annotations + +import math +import re +from collections import Counter + +import networkx as nx + +from graph.build import is_missing + +# Common English only. A word common to a whole field is discounted by its own +# document frequency, so no field's vocabulary has to be written down here. +# fmt: off +STOPWORDS = frozenset(( + "the", "and", "for", "with", "that", "this", "from", "are", "our", "can", "which", "have", + "has", "been", "using", "used", "use", "new", "two", "its", "their", "these", "such", + "into", "than", "also", "based", "via", "over", "more", "not", "all", "one", "only", + "between", "both", "each", "other", +)) +# fmt: on + +_WORD = re.compile(r"[a-z][a-z0-9-]{2,}") + + +class Content: + """A unit TF-IDF vector for every work in a graph.""" + + def __init__(self, graph: nx.DiGraph) -> None: + counts = {node: Counter(_words(graph.nodes[node])) for node in graph} + frequency = Counter(word for words in counts.values() for word in words) + self._vectors = { + node: _unit( + { + word: (1 + math.log(count)) * math.log(len(counts) / frequency[word]) + for word, count in words.items() + } + ) + for node, words in counts.items() + } + + def similarity(self, a: str, b: str) -> float: + """Cosine similarity: 1 for the same words, 0 for none shared or none on record.""" + left, right = self._vectors.get(a, {}), self._vectors.get(b, {}) + if len(left) > len(right): + left, right = right, left + return sum(weight * right.get(word, 0.0) for word, weight in left.items()) + + +def _words(attributes: dict) -> list[str]: + text = " ".join( + str(value) for key in ("title", "abstract") if not is_missing(value := attributes.get(key)) + ) + return [word for word in _WORD.findall(text.lower()) if word not in STOPWORDS] + + +def _unit(vector: dict[str, float]) -> dict[str, float]: + norm = math.sqrt(sum(value * value for value in vector.values())) or 1.0 + return {word: value / norm for word, value in vector.items()} diff --git a/tests/test_content.py b/tests/test_content.py new file mode 100644 index 0000000..20bc55d --- /dev/null +++ b/tests/test_content.py @@ -0,0 +1,43 @@ +import networkx as nx +import pytest + +from gnn.content import Content + + +def works(**texts): + graph = nx.DiGraph() + for node, (title, abstract) in texts.items(): + graph.add_node(node, title=title, abstract=abstract) + return graph + + +def test_the_same_words_are_the_same_subject_and_none_shared_is_none(): + content = Content( + works( + A=("Graph attention networks", "Attention over neighbours."), + B=("Graph attention networks", "Attention over neighbours."), + C=("Stochastic optimisation", "Adaptive moment estimates."), + ) + ) + + assert content.similarity("A", "B") == pytest.approx(1.0) + assert content.similarity("A", "C") == 0.0 + + +def test_case_and_common_english_carry_no_subject(): + content = Content( + works( + A=("THE Attention", "With the attention."), + B=("attention for the", None), + C=("Kernels", "Unrelated."), + ) + ) + + assert content.similarity("A", "B") == pytest.approx(1.0) + + +def test_a_work_with_nothing_on_record_shares_nothing(): + content = Content(works(A=("Attention", "Attention."), B=(None, float("nan")))) + + assert content.similarity("A", "B") == 0.0 + assert content.similarity("A", "NOT-IN-GRAPH") == 0.0 From bd31156b10c533bfa71c3739cc32c265482ac11c Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:34:26 +0300 Subject: [PATCH 2/4] feat: steer the lineage walk by shared subject as well as plausibility --- src/api/app.py | 8 ++++++-- src/api/state.py | 6 +++++- src/gnn/originators.py | 26 ++++++++++++++++++++------ tests/test_api.py | 2 ++ tests/test_originators.py | 16 ++++++++++++++++ 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/api/app.py b/src/api/app.py index 3f5939b..550382f 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -126,7 +126,9 @@ def lineage( ) -> Lineage: graph = loaded.graph _present(graph, work_id) - learned = learned_flow(graph, work_id, loaded.model, embedding=loaded.embedding) + learned = learned_flow( + graph, work_id, loaded.model, embedding=loaded.embedding, content=loaded.content + ) ranked = rank(learned, top) shown = [work_id, *(node for node, _ in ranked)] return Lineage( @@ -155,7 +157,9 @@ def narrative( ) -> Account: graph = loaded.graph _present(graph, work_id) - scores = learned_flow(graph, work_id, loaded.model, embedding=loaded.embedding) + scores = learned_flow( + graph, work_id, loaded.model, embedding=loaded.embedding, content=loaded.content + ) if not scores: raise HTTPException(404, f"{work_id} has no ancestors in this field's snapshot") try: diff --git a/src/api/state.py b/src/api/state.py index 4fc5fea..a31c098 100644 --- a/src/api/state.py +++ b/src/api/state.py @@ -20,6 +20,7 @@ import networkx as nx from torch import Tensor +from gnn.content import Content from gnn.model import LinkPredictor from gnn.originators import embed from gnn.train import load_model @@ -38,6 +39,7 @@ class Field: graph: nx.DiGraph model: LinkPredictor embedding: tuple[dict[str, int], Tensor] + content: Content class Library: @@ -77,4 +79,6 @@ def _build(self, name: str) -> Field: logger.info("loading field %s from %s", name, snapshot) graph = load_snapshot(snapshot) model = load_model(snapshot) - return Field(load_field(name, self.fields_dir), graph, model, embed(graph, model)) + return Field( + load_field(name, self.fields_dir), graph, model, embed(graph, model), Content(graph) + ) diff --git a/src/gnn/originators.py b/src/gnn/originators.py index 56a29f3..6942a81 100644 --- a/src/gnn/originators.py +++ b/src/gnn/originators.py @@ -6,6 +6,10 @@ decoder: at each step backwards the walk prefers the citations the model finds most plausible, which is the one thing structure alone cannot tell it. +A plausible citation can still be an incidental one: every paper cites its +optimiser. So each step also follows how much the two works share in subject, +from their titles and abstracts, which is what tells a precursor from a tool. + The walk is then decayed by age. Backwards through a citation graph there is always another ancestor, and without decay the ranking ends at the founding of the discipline rather than at the work that made this paper possible. @@ -22,6 +26,7 @@ from torch import Tensor from torch_geometric.utils import to_undirected +from gnn.content import Content from gnn.dataset import observed_view from gnn.model import LinkPredictor from gnn.train import load_model @@ -31,6 +36,11 @@ logger = logging.getLogger(__name__) +# Square-rooted, so shared subject tempers the model rather than overruling it, +# and floored, so a work with no abstract on record still gets a share. +CONTENT_WEIGHT = 0.5 +CONTENT_FLOOR = 0.02 + def embed(graph: nx.DiGraph, model: LinkPredictor) -> tuple[dict[str, int], Tensor]: """One encode pass over everything on record. Where each node sits, and as what.""" @@ -63,16 +73,20 @@ def learned_flow( damping: float = 0.85, half_life: float = 10.0, embedding: tuple[dict[str, int], Tensor] | None = None, + content: Content | None = None, ) -> Scores: - """Weight of a damped walk backwards from `target`, steered by the model.""" + """Weight of a damped walk backwards from `target`, steered by the model and by content.""" lineage = acyclic(ancestry(graph, target)) position, z = embedding or embed(graph, model) edges = list(lineage.edges()) - plausibility = dict( - zip(edges, citation_probability(model, z, position, edges).tolist(), strict=True) - ) - return flow(lineage, target, plausibility, damping=damping, half_life=half_life) + weight = dict(zip(edges, citation_probability(model, z, position, edges).tolist(), strict=True)) + if content is not None: + weight = { + edge: share * (CONTENT_FLOOR + content.similarity(*edge)) ** CONTENT_WEIGHT + for edge, share in weight.items() + } + return flow(lineage, target, weight, damping=damping, half_life=half_life) def flow( @@ -118,7 +132,7 @@ def score_target( graph = load_snapshot(snapshot) model = load_model(snapshot) return ( - learned_flow(graph, target, model, half_life=half_life), + learned_flow(graph, target, model, half_life=half_life, content=Content(graph)), {name: scorer(graph, target) for name, scorer in SCORERS.items()}, ) diff --git a/tests/test_api.py b/tests/test_api.py index 989def2..ac9ecff 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -7,6 +7,7 @@ from api.app import create_app from api.state import Field, Library +from gnn.content import Content from graph.config import load_field from llm.base import ProviderError from llm.narrative import Claim, Judgement, Narrative, Verdict @@ -73,6 +74,7 @@ def _build(self, name): self.graph, Indifferent(), (position, torch.zeros(len(position), 2)), + Content(self.graph), ) diff --git a/tests/test_originators.py b/tests/test_originators.py index ebde788..a64a2a8 100644 --- a/tests/test_originators.py +++ b/tests/test_originators.py @@ -5,6 +5,7 @@ import pytest import torch +from gnn.content import Content from gnn.dataset import build_dataset from gnn.originators import embed, learned_flow, score_target from gnn.train import TrainConfig, save_model, train @@ -74,6 +75,21 @@ def test_structural_twins_are_separated_by_the_model_alone(): assert scores["A"] > scores["B"] +def test_structural_twins_are_separated_by_what_they_are_about(): + """A precursor and a tool the model finds equally plausible: content tells them apart.""" + graph = lineage(DIAMOND) + abstracts = {"T": "graph attention", "A": "attention alignment", "B": "stochastic optimiser"} + for node, abstract in abstracts.items(): + graph.nodes[node]["abstract"] = abstract + indifferent = FixedOpinion(graph, graph.edges) + + plain = learned_flow(graph, "T", indifferent) + informed = learned_flow(graph, "T", indifferent, content=Content(graph)) + + assert plain["A"] == plain["B"] + assert informed["A"] > informed["B"] + + def test_the_distant_past_is_discounted(): graph = lineage([("T", "RECENT"), ("T", "ANCIENT")], years={"T": 2020, "RECENT": 2018}) graph.nodes["ANCIENT"]["publication_year"] = 1970 From 0f3d74634be2f72247131e59073446cc4ba4dfb4 Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:34:26 +0300 Subject: [PATCH 3/4] feat: judge the served ranking beside its model-only ablation --- src/gnn/lineages.py | 19 ++++++++++++++----- tests/test_lineages.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/gnn/lineages.py b/src/gnn/lineages.py index c6a48da..092e2f3 100644 --- a/src/gnn/lineages.py +++ b/src/gnn/lineages.py @@ -32,6 +32,7 @@ from pydantic import BaseModel from torch import Tensor +from gnn.content import Content from gnn.dataset import build_dataset from gnn.model import LinkPredictor from gnn.originators import embed, flow, learned_flow @@ -69,11 +70,13 @@ def rankings( target: str, model: LinkPredictor, embedding: tuple[dict[str, int], Tensor], + content: Content, top: int, ) -> dict[str, list[str]]: - """Every scorer's top works for one paper: the model, its walk unweighted, the baselines.""" + """Every scorer's top works for one paper: as served, without content, unweighted, baselines.""" scores = { - "learned": learned_flow(graph, target, model, embedding=embedding), + "learned": learned_flow(graph, target, model, embedding=embedding, content=content), + "model_only": learned_flow(graph, target, model, embedding=embedding), "uniform_flow": flow(acyclic(ancestry(graph, target)), target), **{name: scorer(graph, target) for name, scorer in SCORERS.items()}, } @@ -129,6 +132,7 @@ def assess( logger.warning("gold target %s is not in the graph", lineage.target) embedding = embed(graph, model) + content = Content(graph) thinned = _thinned(graph, dropout, seed=0) thinned_embedding = embed(thinned, model) others = [ @@ -141,10 +145,15 @@ def assess( for lineage in lineages: reachable = set(ancestry(graph, lineage.target)) findable = [ids for ids in lineage.originators if reachable.intersection(ids)] - ranked = rankings(graph, lineage.target, model, embedding, top) - stressed = rankings(thinned, lineage.target, model, thinned_embedding, top) + ranked = rankings(graph, lineage.target, model, embedding, content, top) + stressed = rankings(thinned, lineage.target, model, thinned_embedding, content, top) learned = [ranked["learned"]] + [ - [node for node, _ in rank(learned_flow(graph, lineage.target, other, embedding=e), top)] + [ + node + for node, _ in rank( + learned_flow(graph, lineage.target, other, embedding=e, content=content), top + ) + ] for other, e in zip(others, other_embeddings, strict=True) ] title = graph.nodes[lineage.target].get("title") diff --git a/tests/test_lineages.py b/tests/test_lineages.py index 22064b7..dd63149 100644 --- a/tests/test_lineages.py +++ b/tests/test_lineages.py @@ -78,7 +78,7 @@ def test_an_assessment_judges_covers_and_stresses_every_scorer(): assert result["targets_in_graph"] == "2/3" assert (dev["coverage"], test["coverage"]) == ("1/2", "1/1") - assert set(dev["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"} + assert set(dev["scorers"]) >= {"learned", "model_only", "uniform_flow", "in_degree"} assert dev["scorers"]["learned"]["recall"] == 1.0 assert dev["scorers"]["learned"]["found"] == "1/2" assert "U" not in result["targets"][0]["scorers"]["learned"]["top"] From a69327a7e948653f47e27b9a638986868c5edeac Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:36:42 +0300 Subject: [PATCH 4/4] docs: record the ranking results on held-out papers, and what was left out --- src/gnn/PLAN.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/gnn/PLAN.md b/src/gnn/PLAN.md index 34cc25d..7e13a1d 100644 --- a/src/gnn/PLAN.md +++ b/src/gnn/PLAN.md @@ -53,21 +53,22 @@ One PR each, named for what it does. - Hashed title words: -0.02, overfits. - `topic_id`: +0.06, but it leaks. OpenAlex assigns topics from citations as of the crawl, the reason `cited_by_count` is excluded. - Held-out share, learning rate, depth, width, dropout, weight decay, and MRR instead of AP for checkpoint selection: nothing beyond noise. -- [ ] **Originator ranking evaluation.** `gnn.lineages` is built: Recall@10 and nDCG@10 over reachable gold originators for the learned ranking, the same walk with equal weights, and every baseline; coverage apart; top-10 overlap across 3 training seeds and with 10% of citations dropped. Waiting on the gold set, `fields/gold/gnn.yaml` (not `fields/*.yaml`, which the API lists as fields): 18 targets drafted, being verified. On the draft, served snapshot, before and after recovering citations from Semantic Scholar: +- [x] **Originator ranking evaluation.** `gnn.lineages` against `fields/gold/gnn.yaml`: 23 papers and 63 originators, each checked against how the paper cites the work, split into `dev` (may steer changes) and `test` (report only) before any tuning. The walk now also follows shared subject (`gnn.content`, TF-IDF over titles and abstracts), chosen on dev only. Served snapshot: - | Scorer | Recall@10 | nDCG@10 | Overlap, 10% dropped | Gold found | + | Scorer | dev found | dev Recall@10 / nDCG@10 | test found | test Recall@10 / nDCG@10 | |---|---|---|---|---| - | learned | 0.484 / 0.465 | 0.466 / 0.439 | 0.844 / 0.894 | 45 / 44 | - | equal-weight walk | 0.410 / 0.397 | 0.366 / 0.333 | 0.844 / 0.867 | 38 / 38 | - | time_decayed_pagerank | 0.432 / 0.401 | 0.380 / 0.352 | 0.833 / 0.872 | 40 / 38 | - | in_degree | 0.127 / 0.097 | 0.076 / 0.060 | 0.833 / 0.861 | | - | gateway | 0.063 / 0.093 | 0.081 / 0.117 | 0.811 / 0.778 | | - | path_weight | 0.000 / 0.000 | 0.000 / 0.000 | 0.900 / 0.883 | | - - Coverage 90/94, then 92/94; learned top-10 overlap across seeds 0.94 to 0.95. The learned ranking leads, and the model's weights are worth +0.07 recall over the same walk unweighted. Recovered citations make more gold originators reachable and every ranking steadier when citations go missing, but find no more of them: recall is over reachable originators, so it drops as the denominator grows. `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent. + | learned (model and content, served) | 23/32 | 0.903 / 0.717 | 17/31 | 0.626 / 0.556 | + | model only | 22/32 | 0.806 / 0.650 | 16/31 | 0.523 / 0.340 | + | equal-weight walk | 22/32 | 0.806 / 0.571 | 14/31 | 0.474 / 0.344 | + | time_decayed_pagerank | 21/32 | 0.778 / 0.567 | 14/31 | 0.474 / 0.303 | + | gateway | 5/32 | 0.106 / 0.090 | 4/31 | 0.167 / 0.080 | + | in_degree | 2/32 | 0.097 / 0.092 | 2/31 | 0.076 / 0.038 | + | path_weight | 0/32 | 0 / 0 | 0/31 | 0 / 0 | + + Learned top-10 overlap across seeds 0.94 dev, 0.97 test; with 10% of citations dropped 0.92 and 0.93. Tried on dev and left out: walk damping, age half-life and sharpening the model's weights (a hit either way, noise at 12 papers), and Semantic Scholar's influence rate per work (Adam's is higher than ChebNet's). `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent. - [x] **Recovered citations.** `graph.semanticscholar`, an optional stage between crawl and build: 6811 MAG-era works' Semantic Scholar references matched back to crawled works by MAG id, else by title and year. 22582 citations matched, 6808 of them lost from OpenAlex; GCN regains Bruna, Henaff and Weston. Served. -- [ ] **Citation influence labels.** Semantic Scholar `isInfluential` per citation; per-paper AUC of the decoder on influential vs incidental references. -- [ ] **Influence-aligned objective.** Fine-tune the decoder listwise on influential references, link prediction as pretraining. Validate on the gold set. +- [x] **Citation influence labels.** 5202 Semantic Scholar `isInfluential` labels from 141 non-gold papers. The decoder's plausibility alone predicts them at AUC 0.70, hand features at 0.71, and the model's embeddings add nothing. The flag mostly counts how often a paper mentions a work, so tools earn it too. +- [x] **Influence-aligned objective.** Not adopted. Walk weights from a model of those labels tie content weighting on dev (24 against 23 of 32 found) and add nothing on top of it, for an extra pipeline stage and a rate-limited API at ranking time. - [x] **Crawl expansion.** Uncapped backward crawl, plus `forward_depth: 1`: the in-topic works citing the seeds. 2789 works and 15k citations became 9249 and 104k. GAT is seeded by its primary record. Cleaning re-dates 262 works OpenAlex dates years after their citers, and a year heavier than its split's share no longer empties the split after it. Gold originators reachable from their targets: 88/94 to 90/94. The other four are references OpenAlex holds under dead ids (about 12% of all references), among them GCN's to Bruna, Henaff and Planetoid. Optional: blinded LLM pairwise judge, trusted only after it agrees with the gold set.