feat: verified gold lineages, held out into dev and test

fields/gold/gnn.yaml

@@ -0,0 +1,168 @@
+# Gold lineages for `gnn.lineages`: papers, and the earlier works each was built on.
+#
+# An originator is a work the paper's method is built on, extends, or makes
+# trainable: not a tool, dataset, benchmark split, baseline, related work, or an
+# approach the paper replaces. Every entry was checked against the sentences in
+# which the paper cites the work (Semantic Scholar citation contexts), or where
+# those say too little, against the paper's abstract. Papers whose citations
+# come with no context were left out. Each originator is every OpenAlex id the
+# work goes by; any of them counts.
+#
+# `dev` papers may be used to tune the ranking; `test` papers only to report it.
+# The split alternates in publication order and was fixed before any tuning.
+
+lineages:
+  - target: W2116341502  # The Graph Neural Network Model (2008)
+    split: dev
+    originators:
+      - [W1501856433]  # Gori et al. 2005: the model's first form, by the same authors
+      - [W2166681504]  # Sperduti & Starita 1997: the recursive networks it extends, per its abstract
+      - [W2127827747]  # Frasconi et al. 1998: likewise
+      - [W1881179843]  # Almeida: its learning algorithm is based on Almeida-Pineda
+      - [W2007431958]  # Pineda 1987: likewise
+      - [W2066636486]  # Brin & Page 1998: the random-walk model it extends, per its abstract
+
+  - target: W1662382123  # Spectral Networks and Locally Connected Networks on Graphs (2013)
+    split: test
+    originators:
+      - [W2112796928, W2310919327]  # LeCun et al. 1998: the CNN it generalizes to graphs, per its abstract
+      - [W2156718197]  # Belkin & Niyogi 2002: the graph Laplacian as harmonic analysis on graphs
+      - [W2106869737]  # Coates & Ng 2011: feature similarity to build locally connected networks
+
+  - target: W2154851992  # DeepWalk (2014)
+    split: dev
+    originators:
+      - [W1614298861, W2950577311]  # Mikolov et al. 2013: the skip-gram relaxation it applies to walks
+      - [W2153579005]  # Mikolov et al. 2013: likewise
+
+  - target: W2405933695  # Learning Deep Representations for Graph Clustering (2014)
+    split: test
+    originators:
+      - [W2997574889, W2145094598]  # Vincent et al. 2010: stacked autoencoders, its building block
+      - [W2121947440]  # Shi & Malik 2000: the spectral clustering it recasts as reconstruction
+      - [W2132914434, W2949364901]  # von Luxburg 2007: likewise
+
+  - target: W1888005072  # LINE (2015)
+    split: dev
+    originators:
+      - [W2153579005]  # Mikolov et al. 2013: adopts its negative sampling
+
+  - target: W2173027866  # Convolutional Networks on Graphs for Learning Molecular Fingerprints (2015)
+    split: test
+    originators:
+      - [W1988037271]  # Rogers & Hahn 2010: the circular fingerprint it makes differentiable
+
+  - target: W2244807774  # Gated Graph Sequence Neural Networks (2015)
+    split: dev
+    originators:
+      - [W2116341502]  # Scarselli et al.: the GNN it modifies
+      - [W1501856433]  # Gori et al. 2005: likewise
+      - [W2950635152, W2157331557]  # Cho et al. 2014: the GRU it propagates with
+
+  - target: W2299115575  # Discriminative Embeddings of Latent Variable Models for Structured Dat (2016)
+    split: test
+    originators:
+      - [W1946137962]  # Smola et al. 2007: embeds graphical models into feature spaces
+      - [W2146641075]  # Song et al. 2009: likewise
+      - [W2159080219]  # Pearl 1988: belief propagation, one inference it embeds
+      - [W2120340025]  # Wainwright & Jordan 2008: mean field, the other
+
+  - target: W2519887557  # Semi-Supervised Classification with Graph Convolutional Networks (2016)
+    split: dev
+    originators:
+      - [W2468907370, W2964321699]  # Defferrard et al. 2016: GCN is its first-order case
+      - [W1662382123]  # Bruna et al.: its method is based on their spectral graph CNNs
+      - [W2158787690]  # Hammond et al. 2011: the Chebyshev approximation it truncates
+
+  - target: W2554952599  # Variational Graph Auto-Encoders (2016)
+    split: test
+    originators:
+      - [W1959608418]  # Kingma & Welling 2013: based on the variational autoencoder
+      - [W2519887557]  # GCN: its encoder
+
+  - target: W2962756421  # node2vec (2016)
+    split: dev
+    originators:
+      - [W2154851992]  # DeepWalk: the uniform walk it generalizes
+      - [W1614298861, W2950577311]  # Mikolov et al. 2013: extends skip-gram to networks
+      - [W2153579005]  # Mikolov et al. 2013: approximates with negative sampling
+
+  - target: W2604942799  # Community Preserving Network Embedding (2017)
+    split: test
+    originators:
+      - [W2135029798]  # Lee & Seung: its factorization and update rules
+      - [W2151936673]  # Newman 2006: adopts modularity maximization
+      - [W2015953751]  # Newman 2006: likewise
+      - [W1888005072]  # LINE: the first- and second-order proximities it preserves
+
+  - target: W2606780347  # Neural Message Passing for Quantum Chemistry (2017)
+    split: dev
+    originators:
+      - [W2244807774]  # GG-NN: its GRU update, and one of the models it unifies
+      - [W2173027866]  # Duvenaud et al.: one of the models it unifies
+      - [W2552391307]  # Battaglia et al. 2016, Interaction Networks: likewise
+      - [W2290847742]  # Kearnes et al. 2016, Molecular Graph Convolutions: likewise
+      - [W1662382123]  # Bruna et al.: likewise, among the Laplacian-based methods
+      - [W2468907370, W2964321699]  # Defferrard et al. 2016: likewise
+      - [W2519887557]  # GCN: likewise, treated as a special case
+
+  - target: W2624431344  # Inductive Representation Learning on Large Graphs (2017)
+    split: test
+    originators:
+      - [W2519887557]  # GCN: its convolutional variant extends GCN inductively
+      - [W2142498761]  # Shervashidze et al. 2011: the WL relabelling its algorithm mirrors
+
+  - target: W2766453196  # Graph Attention Networks (2017)
+    split: dev
+    originators:
+      - [W2133564696]  # Bahdanau et al. 2015: its attentional setup closely follows theirs
+      - [W2626778328]  # Vaswani et al. 2017: the self-attention that motivated it
+
+  - target: W2604314403  # Modeling Relational Data with Graph Convolutional Networks (2018)
+    split: test
+    originators:
+      - [W2519887557]  # GCN: motivated as an adaptation of previous work on GCNs
+      - [W1662382123]  # Bruna et al.: likewise
+      - [W2173027866]  # Duvenaud et al.: likewise
+      - [W2468907370, W2964321699]  # Defferrard et al.: likewise
+      - [W2606780347]  # Gilmer et al. 2017: a simplification of their message passing network
+      - [W1533230146, W2951077644]  # Yang et al. 2014, DistMult: its decoder
+
+  - target: W2786915849  # FastGCN: Fast Learning with Graph Convolutional Networks via Importanc (2018)
+    split: dev
+    originators:
+      - [W2519887557]  # GCN: the model it trains by importance sampling, per its abstract
+
+  - target: W2804057010  # Representation Learning on Graphs with Jumping Knowledge Networks (2018)
+    split: test
+    originators:
+      - [W2519887557]  # GCN: a neighbourhood aggregation model it adds layer aggregation to
+      - [W2624431344]  # GraphSAGE: likewise
+
+  - target: W2893944917  # Deep Graph Infomax. (2018)
+    split: dev
+    originators:
+      - [W2951873722]  # Hjelm et al. 2018, Deep InfoMax: follows its intuitions
+      - [W2519887557]  # GCN: its encoder
+
+  - target: W2951659295  # Hierarchical Graph Representation Learning with Differentiable Pooling (2018)
+    split: test
+    originators:
+      - [W2163605009]  # Krizhevsky et al. 2012: a graph analogue of CNN spatial pooling
+      - [W2624431344]  # GraphSAGE: its GNN module
+
+  - target: W2916106175  # Simplifying Graph Convolutional Networks (2019)
+    split: dev
+    originators:
+      - [W2519887557]  # GCN: the model it simplifies
+
+  - target: W2945827377  # Cluster-GCN (2019)
+    split: test
+    originators:
+      - [W2070232376]  # Karypis & Kumar 1998, METIS: the partitioning its batches come from
+      - [W2519887557]  # GCN: the model it makes trainable on large graphs
+
+  - target: W2961295589  # GraphSAINT: Graph Sampling Based Inductive Learning Method (2019)
+    split: dev
+    originators:
+      - [W2786915849]  # FastGCN: its sampler is inspired by their layer sampler

src/gnn/lineages.py

@@ -25,6 +25,7 @@
 import random
 from itertools import combinations
 from pathlib import Path
+from typing import Literal
 
 import networkx as nx
 import yaml
@@ -47,6 +48,8 @@
 
 class Lineage(BaseModel):
     target: str
+    # `dev` papers may steer changes to the ranking; `test` papers only report it.
+    split: Literal["dev", "test"]
     # One entry per work, as every OpenAlex id it goes by: any of them counts.
     originators: list[list[str]]
 
@@ -148,6 +151,7 @@ def assess(
         targets.append(
             {
                 "target": lineage.target,
+                "split": lineage.split,
                 "title": None if is_missing(title) else str(title),
                 "originators": len(lineage.originators),
                 "reachable": len(findable),
@@ -156,44 +160,65 @@ def assess(
                     name: dict(
                         zip(("recall", "ndcg"), judge(ranking, findable, top), strict=True),
                         dropout_overlap=_overlap(ranking, stressed[name]),
+                        found=sum(bool(set(ranking) & set(ids)) for ids in lineage.originators),
                         top=ranking,
                     )
                     for name, ranking in ranked.items()
                 },
             }
         )
 
-    names = targets[0]["scorers"] if targets else {}
-    reachable, originators = (sum(t[key] for t in targets) for key in ("reachable", "originators"))
     return {
         "top": top,
         "targets_in_graph": f"{len(lineages)}/{len(gold)}",
+        "splits": {
+            split: _summarise([t for t in targets if t["split"] == split])
+            for split in ("dev", "test")
+        },
+        "targets": targets,
+    }
+
+
+def _summarise(targets: list[dict]) -> dict:
+    """Means over papers, except gold found, which is counted over every originator."""
+    names = targets[0]["scorers"] if targets else {}
+    reachable, originators = (sum(t[key] for t in targets) for key in ("reachable", "originators"))
+    return {
         "coverage": f"{reachable}/{originators}",
         "seed_overlap": _mean(t["seed_overlap"] for t in targets),
         "scorers": {
             name: {
-                metric: _mean(t["scorers"][name][metric] for t in targets)
-                for metric in ("recall", "ndcg", "dropout_overlap")
+                **{
+                    metric: _mean(t["scorers"][name][metric] for t in targets)
+                    for metric in ("recall", "ndcg", "dropout_overlap")
+                },
+                "found": f"{sum(t['scorers'][name]['found'] for t in targets)}/{originators}",
             }
             for name in names
         },
-        "targets": targets,
     }
 
 
 def log_assessment(result: dict) -> None:
-    logger.info("%-22s %10s %10s %16s", "scorer", "recall", "ndcg", "dropout overlap")
-    for name, metrics in result["scorers"].items():
+    logger.info("gold targets in graph %s", result["targets_in_graph"])
+    for split, summary in result["splits"].items():
         logger.info(
-            "%-22s %10.3f %10.3f %16.3f",
-            name,
-            metrics["recall"],
-            metrics["ndcg"],
-            metrics["dropout_overlap"],
+            "\n%s: originators reachable %s, learned top-%d overlap across seeds %.3f",
+            split,
+            summary["coverage"],
+            result["top"],
+            summary["seed_overlap"],
         )
-    logger.info("gold targets in graph      %s", result["targets_in_graph"])
-    logger.info("gold originators reachable %s", result["coverage"])
-    logger.info("learned top-%d overlap across seeds %.3f", result["top"], result["seed_overlap"])
+        logger.info("%-22s %8s %8s %8s %10s", "scorer", "found", "recall", "ndcg", "dropout")
+        for name, metrics in summary["scorers"].items():
+            logger.info(
+                "%-22s %8s %8.3f %8.3f %10.3f",
+                name,
+                metrics["found"],
+                metrics["recall"],
+                metrics["ndcg"],
+                metrics["dropout_overlap"],
+            )
 
 
 def assess_field(

tests/test_lineages.py

@@ -35,10 +35,11 @@ def test_a_paper_with_nothing_findable_has_nothing_to_score():
 
 def test_gold_files_are_checked_on_load(tmp_path):
     path = tmp_path / "gold.yaml"
-    path.write_text("lineages:\n  - target: W1\n    originators: [[W2, W3], [W4]]\n")
-    assert load_gold(path) == [Lineage(target="W1", originators=[["W2", "W3"], ["W4"]])]
+    path.write_text("lineages:\n- target: W1\n  split: dev\n  originators: [[W2, W3], [W4]]\n")
+    expected = Lineage(target="W1", split="dev", originators=[["W2", "W3"], ["W4"]])
+    assert load_gold(path) == [expected]
 
-    path.write_text("lineages:\n  - target: W1\n    originators: W2\n")
+    path.write_text("lineages:\n  - target: W1\n    split: train\n    originators: [[W2]]\n")
     with pytest.raises(ValidationError):
         load_gold(path)
 
@@ -67,14 +68,17 @@ def test_the_unweighted_walk_is_the_learned_walk_of_an_indifferent_model():
 def test_an_assessment_judges_covers_and_stresses_every_scorer():
     graph = lineage([("T", "A"), ("T", "B"), ("A", "G"), ("B", "X"), ("U", "T")])
     gold = [
-        Lineage(target="T", originators=[["G"], ["NEVER-CRAWLED"]]),
-        Lineage(target="ABSENT", originators=[["G"]]),
+        Lineage(target="T", split="dev", originators=[["G"], ["NEVER-CRAWLED"]]),
+        Lineage(target="B", split="test", originators=[["X"]]),
+        Lineage(target="ABSENT", split="test", originators=[["G"]]),
     ]
 
     result = assess(graph, gold, FixedOpinion(graph, [("T", "A"), ("A", "G")]), seeds=1)
+    dev, test = result["splits"]["dev"], result["splits"]["test"]
 
-    assert result["targets_in_graph"] == "1/2"
-    assert result["coverage"] == "1/2"
-    assert set(result["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"}
-    assert result["scorers"]["learned"]["recall"] == 1.0
+    assert result["targets_in_graph"] == "2/3"
+    assert (dev["coverage"], test["coverage"]) == ("1/2", "1/1")
+    assert set(dev["scorers"]) >= {"learned", "uniform_flow", "in_degree", "gateway"}
+    assert dev["scorers"]["learned"]["recall"] == 1.0
+    assert dev["scorers"]["learned"]["found"] == "1/2"
     assert "U" not in result["targets"][0]["scorers"]["learned"]["top"]