From e93d1f477f05396f6ab9d09af443458814f462d9 Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 14:13:18 +0300 Subject: [PATCH 1/3] refactor: split the damped walk out of learned_flow --- src/gnn/originators.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/gnn/originators.py b/src/gnn/originators.py index 4fef9c1..56a29f3 100644 --- a/src/gnn/originators.py +++ b/src/gnn/originators.py @@ -64,11 +64,7 @@ def learned_flow( half_life: float = 10.0, embedding: tuple[dict[str, int], Tensor] | None = None, ) -> Scores: - """Weight of a damped walk backwards from `target`, steered by the model. - - Exact rather than iterated: the lineage is acyclic, so one pass in - topological order settles every node's share. - """ + """Weight of a damped walk backwards from `target`, steered by the model.""" lineage = acyclic(ancestry(graph, target)) position, z = embedding or embed(graph, model) @@ -76,21 +72,36 @@ def learned_flow( plausibility = dict( zip(edges, citation_probability(model, z, position, edges).tolist(), strict=True) ) + return flow(lineage, target, plausibility, damping=damping, half_life=half_life) + + +def flow( + lineage: nx.DiGraph, + target: str, + weight: dict[tuple[str, str], float] | None = None, + *, + damping: float = 0.85, + half_life: float = 10.0, +) -> Scores: + """The damped walk itself, splitting each step by `weight`, or evenly without one. - flow = dict.fromkeys(lineage, 0.0) - flow[target] = 1.0 + Exact rather than iterated: the lineage is acyclic, so one pass in + topological order settles every node's share. + """ + reached = dict.fromkeys(lineage, 0.0) + reached[target] = 1.0 for node in nx.topological_sort(lineage): cited = list(lineage.successors(node)) - shares = [plausibility[(node, work)] for work in cited] + shares = [1.0 if weight is None else weight[(node, work)] for work in cited] total = sum(shares) if not total: continue for work, share in zip(cited, shares, strict=True): - flow[work] += flow[node] * damping * share / total + reached[work] += reached[node] * damping * share / total return { - node: weight * 0.5 ** (year_gap(lineage, target, node) / half_life) - for node, weight in flow.items() + node: share * 0.5 ** (year_gap(lineage, target, node) / half_life) + for node, share in reached.items() if node != target } From 08223d7d51d22f6a5b9d1c4db19321af4791ecc7 Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 14:13:18 +0300 Subject: [PATCH 2/3] feat: judge originator rankings against gold lineages --- src/gnn/lineages.py | 247 +++++++++++++++++++++++++++++++++++++++++ tests/test_lineages.py | 80 +++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 src/gnn/lineages.py create mode 100644 tests/test_lineages.py diff --git a/src/gnn/lineages.py b/src/gnn/lineages.py new file mode 100644 index 0000000..f265cb3 --- /dev/null +++ b/src/gnn/lineages.py @@ -0,0 +1,247 @@ +"""Judge originator rankings against lineages a person has checked. + +Link prediction says whether the model understands citations. It does not say +whether the ranking a reader sees names the works that made a paper possible. +That takes an answer key, `fields/gold/.yaml`: papers, and for each the +earlier works it grew from. Every scorer ranks each paper's ancestry and is +judged on how many of those works reach its top ten, and how high. + +A gold originator the crawl never reached, or reached by no path from its paper, +cannot be ranked by anything. It counts against coverage, reported on its own, +rather than against a scorer. + +Two further checks ask whether a ranking can be trusted at all. The learned one +should not change with the seed its model was trained from, and no ranking +should change much when a tenth of the citations go missing, since OpenAlex +already loses about that many. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import math +import random +from itertools import combinations +from pathlib import Path + +import networkx as nx +import yaml +from pydantic import BaseModel +from torch import Tensor + +from gnn.dataset import build_dataset +from gnn.model import LinkPredictor +from gnn.originators import embed, flow, learned_flow +from gnn.train import TrainConfig, load_model, train +from graph.build import is_missing, latest_snapshot, load_snapshot +from graph.config import FIELDS_DIR +from graph.crawler import DATA_DIR +from graph.score import SCORERS, acyclic, ancestry, rank + +logger = logging.getLogger(__name__) + +GOLD_DIR = FIELDS_DIR / "gold" + + +class Lineage(BaseModel): + target: str + # One entry per work, as every OpenAlex id it goes by: any of them counts. + originators: list[list[str]] + + +class Gold(BaseModel): + lineages: list[Lineage] + + +def load_gold(path: Path) -> list[Lineage]: + if not path.is_file(): + raise FileNotFoundError(f"no gold lineages at {path}") + return Gold.model_validate(yaml.safe_load(path.read_text())).lineages + + +def rankings( + graph: nx.DiGraph, + target: str, + model: LinkPredictor, + embedding: tuple[dict[str, int], Tensor], + top: int, +) -> dict[str, list[str]]: + """Every scorer's top works for one paper: the model, its walk unweighted, the baselines.""" + scores = { + "learned": learned_flow(graph, target, model, embedding=embedding), + "uniform_flow": flow(acyclic(ancestry(graph, target)), target), + **{name: scorer(graph, target) for name, scorer in SCORERS.items()}, + } + return {name: [node for node, _ in rank(ranked, top)] for name, ranked in scores.items()} + + +def judge(ranking: list[str], findable: list[list[str]], top: int) -> tuple[float, float]: + """Recall and nDCG of a top-`top` ranking, over the gold originators it could have found.""" + if not findable: + return math.nan, math.nan + work = {work_id: order for order, ids in enumerate(findable) for work_id in ids} + found: set[int] = set() + gain = 0.0 + for position, node in enumerate(ranking, start=1): + if (order := work.get(node)) is not None and order not in found: + found.add(order) + gain += 1 / math.log2(position + 1) + ideal = sum(1 / math.log2(position + 1) for position in range(1, min(len(findable), top) + 1)) + return len(found) / len(findable), gain / ideal + + +def _overlap(left: list[str], right: list[str]) -> float: + return len(set(left) & set(right)) / len(left) if left else math.nan + + +def _thinned(graph: nx.DiGraph, share: float, seed: int) -> nx.DiGraph: + """The graph with a random `share` of its citations gone.""" + chance = random.Random(seed) + thinned = nx.DiGraph() + thinned.add_nodes_from(graph.nodes(data=True)) + thinned.add_edges_from(edge for edge in graph.edges if chance.random() >= share) + return thinned + + +def _mean(values) -> float: + kept = [value for value in values if not math.isnan(value)] + return sum(kept) / len(kept) if kept else math.nan + + +def assess( + graph: nx.DiGraph, + gold: list[Lineage], + model: LinkPredictor, + *, + top: int = 10, + seeds: int = 3, + dropout: float = 0.1, +) -> dict: + """Score, cover and stress every scorer on every gold paper the graph holds.""" + lineages = [lineage for lineage in gold if lineage.target in graph] + for lineage in gold: + if lineage.target not in graph: + logger.warning("gold target %s is not in the graph", lineage.target) + + embedding = embed(graph, model) + thinned = _thinned(graph, dropout, seed=0) + thinned_embedding = embed(thinned, model) + others = [ + train(build_dataset(graph, seed=seed), config=TrainConfig(seed=seed))[0] + for seed in range(1, seeds) + ] + other_embeddings = [embed(graph, other) for other in others] + + targets = [] + for lineage in lineages: + reachable = set(ancestry(graph, lineage.target)) + findable = [ids for ids in lineage.originators if reachable.intersection(ids)] + ranked = rankings(graph, lineage.target, model, embedding, top) + stressed = rankings(thinned, lineage.target, model, thinned_embedding, top) + learned = [ranked["learned"]] + [ + [node for node, _ in rank(learned_flow(graph, lineage.target, other, embedding=e), top)] + for other, e in zip(others, other_embeddings, strict=True) + ] + title = graph.nodes[lineage.target].get("title") + targets.append( + { + "target": lineage.target, + "title": None if is_missing(title) else str(title), + "originators": len(lineage.originators), + "reachable": len(findable), + "seed_overlap": _mean(_overlap(a, b) for a, b in combinations(learned, 2)), + "scorers": { + name: dict( + zip(("recall", "ndcg"), judge(ranking, findable, top), strict=True), + dropout_overlap=_overlap(ranking, stressed[name]), + top=ranking, + ) + for name, ranking in ranked.items() + }, + } + ) + + names = targets[0]["scorers"] if targets else {} + reachable, originators = (sum(t[key] for t in targets) for key in ("reachable", "originators")) + return { + "top": top, + "targets_in_graph": f"{len(lineages)}/{len(gold)}", + "coverage": f"{reachable}/{originators}", + "seed_overlap": _mean(t["seed_overlap"] for t in targets), + "scorers": { + name: { + metric: _mean(t["scorers"][name][metric] for t in targets) + for metric in ("recall", "ndcg", "dropout_overlap") + } + for name in names + }, + "targets": targets, + } + + +def log_assessment(result: dict) -> None: + logger.info("%-22s %10s %10s %16s", "scorer", "recall", "ndcg", "dropout overlap") + for name, metrics in result["scorers"].items(): + logger.info( + "%-22s %10.3f %10.3f %16.3f", + name, + metrics["recall"], + metrics["ndcg"], + metrics["dropout_overlap"], + ) + logger.info("gold targets in graph %s", result["targets_in_graph"]) + logger.info("gold originators reachable %s", result["coverage"]) + logger.info("learned top-%d overlap across seeds %.3f", result["top"], result["seed_overlap"]) + + +def assess_field( + field_name: str, + *, + data_dir: Path = DATA_DIR, + gold_path: Path | None = None, + top: int = 10, + seeds: int = 3, + dropout: float = 0.1, +) -> tuple[Path, dict]: + """Judge a field's served snapshot and model, and write the verdict beside them.""" + snapshot = latest_snapshot(data_dir, field_name) + result = assess( + load_snapshot(snapshot), + load_gold(gold_path or GOLD_DIR / f"{field_name}.yaml"), + load_model(snapshot), + top=top, + seeds=seeds, + dropout=dropout, + ) + path = snapshot / "lineages.json" + path.write_text(json.dumps(result, indent=2)) + return path, result + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--field", default="gnn", help="field config name under fields/") + parser.add_argument("--data-dir", type=Path, default=DATA_DIR) + parser.add_argument("--gold", type=Path, default=None, help="defaults to fields/gold/") + parser.add_argument("--top", type=int, default=10) + parser.add_argument("--seeds", type=int, default=3, help="models trained to test stability") + parser.add_argument("--dropout", type=float, default=0.1, help="share of citations removed") + args = parser.parse_args(argv) + + logging.basicConfig(level=logging.INFO, format="%(message)s") + path, result = assess_field( + args.field, + data_dir=args.data_dir, + gold_path=args.gold, + top=args.top, + seeds=args.seeds, + dropout=args.dropout, + ) + log_assessment(result) + logger.info("verdict written to %s", path) + + +if __name__ == "__main__": + main() diff --git a/tests/test_lineages.py b/tests/test_lineages.py new file mode 100644 index 0000000..04b1577 --- /dev/null +++ b/tests/test_lineages.py @@ -0,0 +1,80 @@ +import math + +import pytest +from pydantic import ValidationError + +from gnn.lineages import Lineage, _thinned, assess, judge, load_gold +from gnn.originators import flow, learned_flow +from test_originators import DIAMOND, FixedOpinion, lineage + + +def test_a_ranking_with_every_originator_on_top_scores_one(): + assert judge(["A", "B", "C"], [["A"], ["B"]], top=3) == (1.0, 1.0) + + +def test_a_missed_originator_costs_recall_and_a_late_one_costs_ndcg(): + recall, ndcg = judge(["X", "A", "Y"], [["A"], ["B"]], top=3) + + assert recall == 0.5 + assert ndcg == pytest.approx((1 / math.log2(3)) / (1 + 1 / math.log2(3))) + + +def test_any_id_of_a_work_finds_it_but_only_once(): + assert judge(["A2", "A1"], [["A1", "A2"]], top=2) == (1.0, 1.0) + + +def test_more_originators_than_places_do_not_lower_the_ideal_ranking(): + _, ndcg = judge(["A", "B"], [["A"], ["B"], ["C"]], top=2) + + assert ndcg == 1.0 + + +def test_a_paper_with_nothing_findable_has_nothing_to_score(): + assert all(math.isnan(value) for value in judge(["A"], [], top=1)) + + +def test_gold_files_are_checked_on_load(tmp_path): + path = tmp_path / "gold.yaml" + path.write_text("lineages:\n - target: W1\n originators: [[W2, W3], [W4]]\n") + assert load_gold(path) == [Lineage(target="W1", originators=[["W2", "W3"], ["W4"]])] + + path.write_text("lineages:\n - target: W1\n originators: W2\n") + with pytest.raises(ValidationError): + load_gold(path) + + with pytest.raises(FileNotFoundError): + load_gold(tmp_path / "absent.yaml") + + +def test_thinning_drops_citations_but_keeps_every_work(): + graph = lineage([(f"P{n}", f"P{n + 1}") for n in range(200)]) + + thinned = _thinned(graph, 0.5, seed=0) + + assert set(thinned) == set(graph) + assert 60 < thinned.number_of_edges() < 140 + assert thinned.nodes["P0"] == graph.nodes["P0"] + + +def test_the_unweighted_walk_is_the_learned_walk_of_an_indifferent_model(): + graph = lineage(DIAMOND) + + indifferent = learned_flow(graph, "T", FixedOpinion(graph, graph.edges)) + + assert flow(graph, "T") == pytest.approx(indifferent) + + +def test_an_assessment_judges_covers_and_stresses_every_scorer(): + graph = lineage([("T", "A"), ("T", "B"), ("A", "G"), ("B", "X"), ("U", "T")]) + gold = [ + Lineage(target="T", originators=[["G"], ["NEVER-CRAWLED"]]), + Lineage(target="ABSENT", originators=[["G"]]), + ] + + result = assess(graph, gold, FixedOpinion(graph, [("T", "A"), ("A", "G")]), seeds=1) + + assert result["targets_in_graph"] == "1/2" + assert result["coverage"] == "1/2" + assert set(result["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"} + assert result["scorers"]["learned"]["recall"] == 1.0 + assert "U" not in result["targets"][0]["scorers"]["learned"]["top"] From 2664a902736d1721e57d6330fa261ec680b6366d Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 14:13:18 +0300 Subject: [PATCH 3/3] docs: record the first originator ranking results --- README.md | 1 + src/gnn/PLAN.md | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 383fbe3..f0d5c64 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ uv run python -m graph.build --field gnn # clean it into a dated snapshot uv run python -m gnn.dataset --field gnn # cut the temporal link-prediction split uv run python -m gnn.train --field gnn # fit, and score on citations it never saw uv run python -m gnn.evaluate --field gnn # score it beside the heuristics it has to beat +uv run python -m gnn.lineages --field gnn # judge the rankings against fields/gold/gnn.yaml uv run python -m llm.narrative --field gnn --target W2519887557 ``` diff --git a/src/gnn/PLAN.md b/src/gnn/PLAN.md index e05a03c..2348253 100644 --- a/src/gnn/PLAN.md +++ b/src/gnn/PLAN.md @@ -53,7 +53,18 @@ One PR each, named for what it does. - Hashed title words: -0.02, overfits. - `topic_id`: +0.06, but it leaks. OpenAlex assigns topics from citations as of the crawl, the reason `cited_by_count` is excluded. - Held-out share, learning rate, depth, width, dropout, weight decay, and MRR instead of AP for checkpoint selection: nothing beyond noise. -- [ ] **Originator ranking evaluation.** Gold lineages for 20-30 targets in `fields/gold/gnn.yaml` (not `fields/*.yaml`, which the API lists as fields), from survey history sections. nDCG@10 and Recall@10 vs all baselines. Ablation: `learned_flow` with uniform edge weights. Stability: top-10 overlap across seeds and 10% edge dropout. +- [ ] **Originator ranking evaluation.** `gnn.lineages` is built: Recall@10 and nDCG@10 over reachable gold originators for the learned ranking, the same walk with equal weights, and every baseline; coverage apart; top-10 overlap across 3 training seeds and with 10% of citations dropped. Waiting on the gold set, `fields/gold/gnn.yaml` (not `fields/*.yaml`, which the API lists as fields): 18 targets drafted, being verified. On the draft, served snapshot: + + | Scorer | Recall@10 | nDCG@10 | Overlap, 10% dropped | + |---|---|---|---| + | learned | 0.495 | 0.484 | 0.844 | + | equal-weight walk | 0.444 | 0.395 | 0.856 | + | time_decayed_pagerank | 0.432 | 0.380 | 0.833 | + | in_degree | 0.127 | 0.076 | 0.833 | + | gateway | 0.074 | 0.097 | 0.822 | + | path_weight | 0.000 | 0.000 | 0.939 | + + Coverage 90/94; learned top-10 overlap across seeds 0.961. `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent. - [ ] **Citation influence labels.** Semantic Scholar `isInfluential` per citation; per-paper AUC of the decoder on influential vs incidental references. - [ ] **Influence-aligned objective.** Fine-tune the decoder listwise on influential references, link prediction as pretraining. Validate on the gold set. - [x] **Crawl expansion.** Uncapped backward crawl, plus `forward_depth: 1`: the in-topic works citing the seeds. 2789 works and 15k citations became 9249 and 104k. GAT is seeded by its primary record. Cleaning re-dates 262 works OpenAlex dates years after their citers, and a year heavier than its split's share no longer empties the split after it. Gold originators reachable from their targets: 88/94 to 90/94. The other four are references OpenAlex holds under dead ids (about 12% of all references), among them GCN's to Bruna, Henaff and Planetoid. @@ -63,5 +74,5 @@ Optional: blinded LLM pairwise judge, trusted only after it agrees with the gold ## Open decisions - Semantic Scholar as a second data source: citation influence labels, and the references OpenAlex holds under dead ids. -- Who verifies the gold set (originator ranking evaluation). +- Gold set verification, then commit `fields/gold/gnn.yaml` and rerun `gnn.lineages`. - Abstract embeddings need a new dependency; deferred until originator ranking evaluation shows text is the bottleneck.