From 87cc551b73047d6cfd1751a8e6f50f01536d938d Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:12:24 +0300 Subject: [PATCH 1/2] feat: report ranking quality on dev and test papers apart, with gold found --- src/gnn/lineages.py | 55 ++++++++++++++++++++++++++++++------------ tests/test_lineages.py | 22 ++++++++++------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/gnn/lineages.py b/src/gnn/lineages.py index f265cb3..c6a48da 100644 --- a/src/gnn/lineages.py +++ b/src/gnn/lineages.py @@ -25,6 +25,7 @@ import random from itertools import combinations from pathlib import Path +from typing import Literal import networkx as nx import yaml @@ -47,6 +48,8 @@ class Lineage(BaseModel): target: str + # `dev` papers may steer changes to the ranking; `test` papers only report it. + split: Literal["dev", "test"] # One entry per work, as every OpenAlex id it goes by: any of them counts. originators: list[list[str]] @@ -148,6 +151,7 @@ def assess( targets.append( { "target": lineage.target, + "split": lineage.split, "title": None if is_missing(title) else str(title), "originators": len(lineage.originators), "reachable": len(findable), @@ -156,6 +160,7 @@ def assess( name: dict( zip(("recall", "ndcg"), judge(ranking, findable, top), strict=True), dropout_overlap=_overlap(ranking, stressed[name]), + found=sum(bool(set(ranking) & set(ids)) for ids in lineage.originators), top=ranking, ) for name, ranking in ranked.items() @@ -163,37 +168,57 @@ def assess( } ) - names = targets[0]["scorers"] if targets else {} - reachable, originators = (sum(t[key] for t in targets) for key in ("reachable", "originators")) return { "top": top, "targets_in_graph": f"{len(lineages)}/{len(gold)}", + "splits": { + split: _summarise([t for t in targets if t["split"] == split]) + for split in ("dev", "test") + }, + "targets": targets, + } + + +def _summarise(targets: list[dict]) -> dict: + """Means over papers, except gold found, which is counted over every originator.""" + names = targets[0]["scorers"] if targets else {} + reachable, originators = (sum(t[key] for t in targets) for key in ("reachable", "originators")) + return { "coverage": f"{reachable}/{originators}", "seed_overlap": _mean(t["seed_overlap"] for t in targets), "scorers": { name: { - metric: _mean(t["scorers"][name][metric] for t in targets) - for metric in ("recall", "ndcg", "dropout_overlap") + **{ + metric: _mean(t["scorers"][name][metric] for t in targets) + for metric in ("recall", "ndcg", "dropout_overlap") + }, + "found": f"{sum(t['scorers'][name]['found'] for t in targets)}/{originators}", } for name in names }, - "targets": targets, } def log_assessment(result: dict) -> None: - logger.info("%-22s %10s %10s %16s", "scorer", "recall", "ndcg", "dropout overlap") - for name, metrics in result["scorers"].items(): + logger.info("gold targets in graph %s", result["targets_in_graph"]) + for split, summary in result["splits"].items(): logger.info( - "%-22s %10.3f %10.3f %16.3f", - name, - metrics["recall"], - metrics["ndcg"], - metrics["dropout_overlap"], + "\n%s: originators reachable %s, learned top-%d overlap across seeds %.3f", + split, + summary["coverage"], + result["top"], + summary["seed_overlap"], ) - logger.info("gold targets in graph %s", result["targets_in_graph"]) - logger.info("gold originators reachable %s", result["coverage"]) - logger.info("learned top-%d overlap across seeds %.3f", result["top"], result["seed_overlap"]) + logger.info("%-22s %8s %8s %8s %10s", "scorer", "found", "recall", "ndcg", "dropout") + for name, metrics in summary["scorers"].items(): + logger.info( + "%-22s %8s %8.3f %8.3f %10.3f", + name, + metrics["found"], + metrics["recall"], + metrics["ndcg"], + metrics["dropout_overlap"], + ) def assess_field( diff --git a/tests/test_lineages.py b/tests/test_lineages.py index 04b1577..22064b7 100644 --- a/tests/test_lineages.py +++ b/tests/test_lineages.py @@ -35,10 +35,11 @@ def test_a_paper_with_nothing_findable_has_nothing_to_score(): def test_gold_files_are_checked_on_load(tmp_path): path = tmp_path / "gold.yaml" - path.write_text("lineages:\n - target: W1\n originators: [[W2, W3], [W4]]\n") - assert load_gold(path) == [Lineage(target="W1", originators=[["W2", "W3"], ["W4"]])] + path.write_text("lineages:\n- target: W1\n split: dev\n originators: [[W2, W3], [W4]]\n") + expected = Lineage(target="W1", split="dev", originators=[["W2", "W3"], ["W4"]]) + assert load_gold(path) == [expected] - path.write_text("lineages:\n - target: W1\n originators: W2\n") + path.write_text("lineages:\n - target: W1\n split: train\n originators: [[W2]]\n") with pytest.raises(ValidationError): load_gold(path) @@ -67,14 +68,17 @@ def test_the_unweighted_walk_is_the_learned_walk_of_an_indifferent_model(): def test_an_assessment_judges_covers_and_stresses_every_scorer(): graph = lineage([("T", "A"), ("T", "B"), ("A", "G"), ("B", "X"), ("U", "T")]) gold = [ - Lineage(target="T", originators=[["G"], ["NEVER-CRAWLED"]]), - Lineage(target="ABSENT", originators=[["G"]]), + Lineage(target="T", split="dev", originators=[["G"], ["NEVER-CRAWLED"]]), + Lineage(target="B", split="test", originators=[["X"]]), + Lineage(target="ABSENT", split="test", originators=[["G"]]), ] result = assess(graph, gold, FixedOpinion(graph, [("T", "A"), ("A", "G")]), seeds=1) + dev, test = result["splits"]["dev"], result["splits"]["test"] - assert result["targets_in_graph"] == "1/2" - assert result["coverage"] == "1/2" - assert set(result["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"} - assert result["scorers"]["learned"]["recall"] == 1.0 + assert result["targets_in_graph"] == "2/3" + assert (dev["coverage"], test["coverage"]) == ("1/2", "1/1") + assert set(dev["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"} + assert dev["scorers"]["learned"]["recall"] == 1.0 + assert dev["scorers"]["learned"]["found"] == "1/2" assert "U" not in result["targets"][0]["scorers"]["learned"]["top"] From 2bdb676e5e94e7474beb5b53a94e9dade7740306 Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 29 Sep 2026 15:12:24 +0300 Subject: [PATCH 2/2] feat: add verified gold lineages for the gnn field --- fields/gold/gnn.yaml | 168 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 fields/gold/gnn.yaml diff --git a/fields/gold/gnn.yaml b/fields/gold/gnn.yaml new file mode 100644 index 0000000..4c62ec9 --- /dev/null +++ b/fields/gold/gnn.yaml @@ -0,0 +1,168 @@ +# Gold lineages for `gnn.lineages`: papers, and the earlier works each was built on. +# +# An originator is a work the paper's method is built on, extends, or makes +# trainable: not a tool, dataset, benchmark split, baseline, related work, or an +# approach the paper replaces. Every entry was checked against the sentences in +# which the paper cites the work (Semantic Scholar citation contexts), or where +# those say too little, against the paper's abstract. Papers whose citations +# come with no context were left out. Each originator is every OpenAlex id the +# work goes by; any of them counts. +# +# `dev` papers may be used to tune the ranking; `test` papers only to report it. +# The split alternates in publication order and was fixed before any tuning. + +lineages: + - target: W2116341502 # The Graph Neural Network Model (2008) + split: dev + originators: + - [W1501856433] # Gori et al. 2005: the model's first form, by the same authors + - [W2166681504] # Sperduti & Starita 1997: the recursive networks it extends, per its abstract + - [W2127827747] # Frasconi et al. 1998: likewise + - [W1881179843] # Almeida: its learning algorithm is based on Almeida-Pineda + - [W2007431958] # Pineda 1987: likewise + - [W2066636486] # Brin & Page 1998: the random-walk model it extends, per its abstract + + - target: W1662382123 # Spectral Networks and Locally Connected Networks on Graphs (2013) + split: test + originators: + - [W2112796928, W2310919327] # LeCun et al. 1998: the CNN it generalizes to graphs, per its abstract + - [W2156718197] # Belkin & Niyogi 2002: the graph Laplacian as harmonic analysis on graphs + - [W2106869737] # Coates & Ng 2011: feature similarity to build locally connected networks + + - target: W2154851992 # DeepWalk (2014) + split: dev + originators: + - [W1614298861, W2950577311] # Mikolov et al. 2013: the skip-gram relaxation it applies to walks + - [W2153579005] # Mikolov et al. 2013: likewise + + - target: W2405933695 # Learning Deep Representations for Graph Clustering (2014) + split: test + originators: + - [W2997574889, W2145094598] # Vincent et al. 2010: stacked autoencoders, its building block + - [W2121947440] # Shi & Malik 2000: the spectral clustering it recasts as reconstruction + - [W2132914434, W2949364901] # von Luxburg 2007: likewise + + - target: W1888005072 # LINE (2015) + split: dev + originators: + - [W2153579005] # Mikolov et al. 2013: adopts its negative sampling + + - target: W2173027866 # Convolutional Networks on Graphs for Learning Molecular Fingerprints (2015) + split: test + originators: + - [W1988037271] # Rogers & Hahn 2010: the circular fingerprint it makes differentiable + + - target: W2244807774 # Gated Graph Sequence Neural Networks (2015) + split: dev + originators: + - [W2116341502] # Scarselli et al.: the GNN it modifies + - [W1501856433] # Gori et al. 2005: likewise + - [W2950635152, W2157331557] # Cho et al. 2014: the GRU it propagates with + + - target: W2299115575 # Discriminative Embeddings of Latent Variable Models for Structured Dat (2016) + split: test + originators: + - [W1946137962] # Smola et al. 2007: embeds graphical models into feature spaces + - [W2146641075] # Song et al. 2009: likewise + - [W2159080219] # Pearl 1988: belief propagation, one inference it embeds + - [W2120340025] # Wainwright & Jordan 2008: mean field, the other + + - target: W2519887557 # Semi-Supervised Classification with Graph Convolutional Networks (2016) + split: dev + originators: + - [W2468907370, W2964321699] # Defferrard et al. 2016: GCN is its first-order case + - [W1662382123] # Bruna et al.: its method is based on their spectral graph CNNs + - [W2158787690] # Hammond et al. 2011: the Chebyshev approximation it truncates + + - target: W2554952599 # Variational Graph Auto-Encoders (2016) + split: test + originators: + - [W1959608418] # Kingma & Welling 2013: based on the variational autoencoder + - [W2519887557] # GCN: its encoder + + - target: W2962756421 # node2vec (2016) + split: dev + originators: + - [W2154851992] # DeepWalk: the uniform walk it generalizes + - [W1614298861, W2950577311] # Mikolov et al. 2013: extends skip-gram to networks + - [W2153579005] # Mikolov et al. 2013: approximates with negative sampling + + - target: W2604942799 # Community Preserving Network Embedding (2017) + split: test + originators: + - [W2135029798] # Lee & Seung: its factorization and update rules + - [W2151936673] # Newman 2006: adopts modularity maximization + - [W2015953751] # Newman 2006: likewise + - [W1888005072] # LINE: the first- and second-order proximities it preserves + + - target: W2606780347 # Neural Message Passing for Quantum Chemistry (2017) + split: dev + originators: + - [W2244807774] # GG-NN: its GRU update, and one of the models it unifies + - [W2173027866] # Duvenaud et al.: one of the models it unifies + - [W2552391307] # Battaglia et al. 2016, Interaction Networks: likewise + - [W2290847742] # Kearnes et al. 2016, Molecular Graph Convolutions: likewise + - [W1662382123] # Bruna et al.: likewise, among the Laplacian-based methods + - [W2468907370, W2964321699] # Defferrard et al. 2016: likewise + - [W2519887557] # GCN: likewise, treated as a special case + + - target: W2624431344 # Inductive Representation Learning on Large Graphs (2017) + split: test + originators: + - [W2519887557] # GCN: its convolutional variant extends GCN inductively + - [W2142498761] # Shervashidze et al. 2011: the WL relabelling its algorithm mirrors + + - target: W2766453196 # Graph Attention Networks (2017) + split: dev + originators: + - [W2133564696] # Bahdanau et al. 2015: its attentional setup closely follows theirs + - [W2626778328] # Vaswani et al. 2017: the self-attention that motivated it + + - target: W2604314403 # Modeling Relational Data with Graph Convolutional Networks (2018) + split: test + originators: + - [W2519887557] # GCN: motivated as an adaptation of previous work on GCNs + - [W1662382123] # Bruna et al.: likewise + - [W2173027866] # Duvenaud et al.: likewise + - [W2468907370, W2964321699] # Defferrard et al.: likewise + - [W2606780347] # Gilmer et al. 2017: a simplification of their message passing network + - [W1533230146, W2951077644] # Yang et al. 2014, DistMult: its decoder + + - target: W2786915849 # FastGCN: Fast Learning with Graph Convolutional Networks via Importanc (2018) + split: dev + originators: + - [W2519887557] # GCN: the model it trains by importance sampling, per its abstract + + - target: W2804057010 # Representation Learning on Graphs with Jumping Knowledge Networks (2018) + split: test + originators: + - [W2519887557] # GCN: a neighbourhood aggregation model it adds layer aggregation to + - [W2624431344] # GraphSAGE: likewise + + - target: W2893944917 # Deep Graph Infomax. (2018) + split: dev + originators: + - [W2951873722] # Hjelm et al. 2018, Deep InfoMax: follows its intuitions + - [W2519887557] # GCN: its encoder + + - target: W2951659295 # Hierarchical Graph Representation Learning with Differentiable Pooling (2018) + split: test + originators: + - [W2163605009] # Krizhevsky et al. 2012: a graph analogue of CNN spatial pooling + - [W2624431344] # GraphSAGE: its GNN module + + - target: W2916106175 # Simplifying Graph Convolutional Networks (2019) + split: dev + originators: + - [W2519887557] # GCN: the model it simplifies + + - target: W2945827377 # Cluster-GCN (2019) + split: test + originators: + - [W2070232376] # Karypis & Kumar 1998, METIS: the partitioning its batches come from + - [W2519887557] # GCN: the model it makes trainable on large graphs + + - target: W2961295589 # GraphSAINT: Graph Sampling Based Inductive Learning Method (2019) + split: dev + originators: + - [W2786915849] # FastGCN: its sampler is inspired by their layer sampler