diff --git a/README.md b/README.md index 447c85e..d317d78 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ topic_id: null # OpenAlex topic delimiting the field seed_papers: [] # OpenAlex work IDs the crawl starts from exclude_works: [] # records whose metadata OpenAlex got wrong corrected_titles: {} # records filed under another work's title +duplicate_works: {} # records of a work the graph holds under another id crawl: hop_depth: 2 # references, backwards from the seeds forward_depth: 1 # works citing the seeds, within topic_id diff --git a/fields/gnn.yaml b/fields/gnn.yaml index 9e679c9..f22ca5c 100644 --- a/fields/gnn.yaml +++ b/fields/gnn.yaml @@ -35,13 +35,34 @@ seed_papers: # the pipeline cannot detect, so they would surface in rankings under a name # that is not theirs. exclude_works: - - W2964015378 # cited 2347 times inside the graph, titled as a 2025 vulnerability-detection paper + - W4292779060 # titled as a 2023 essay on AI consciousness; too few citers to tell what it is # OpenAlex records filed under another work's title. Cleaning restores the # title and drops the abstract, which came from the same wrong record. corrected_titles: # ChebNet (Defferrard et al., 2016), filed as a GSP'18 workshop lecture. W2964321699: Convolutional Neural Networks on Graphs with Fast Localized Spectral Filtering + # Shchur et al. 2018, filed as a paper on multimodal temporal GNNs. + W2900470550: Pitfalls of Graph Neural Network Evaluation + # Bello et al. 2016, filed as a paper on system architectures. The authors are still wrong. + W2952332632: Neural Combinatorial Optimization with Reinforcement Learning + +# OpenAlex records of a work the graph holds under another id, each folded into +# that one with its citations. Records that share a title, a first author and +# roughly a year are folded without being named here; these are the ones filed +# under another work's metadata, named by what their citers cite on Semantic +# Scholar. +duplicate_works: + W2964015378: W2519887557 # GCN, filed as a 2025 paper on vulnerability detection + W4294558607: W2624431344 # GraphSAGE, filed as a 2024 paper on maximum independent sets + W2962711740: W2894175828 # GIN, filed as a 2025 paper on Kolmogorov-Arnold networks + W4322614756: W2554952599 # VGAE, filed as a 2025 paper on road networks + W4385245566: W2626778328 # Attention Is All You Need, filed as a 2023 paper on tutoring systems + W2964121744: W1522301498 # Adam, filed as a 2026 paper on automated research + W4293651439: W2755092149 # Hamilton et al. 2017, filed as a 2026 database paper + W2896457183: W2963341956 # BERT, filed as a 2018 paper on health support + W4285723986: W2766453196 # GAT: a 2020 book chapter of the same name, cited for it from 2018 on + W4289389616: W2900763475 # APPNP, under its first author's later name crawl: # How many citation hops to expand outward from the seeds. diff --git a/fields/gold/gnn.yaml b/fields/gold/gnn.yaml index 4c62ec9..cd808d2 100644 --- a/fields/gold/gnn.yaml +++ b/fields/gold/gnn.yaml @@ -139,7 +139,7 @@ lineages: - [W2519887557] # GCN: a neighbourhood aggregation model it adds layer aggregation to - [W2624431344] # GraphSAGE: likewise - - target: W2893944917 # Deep Graph Infomax. (2018) + - target: W2963782635 # Deep Graph Infomax (2018), which W2893944917 is folded into split: dev originators: - [W2951873722] # Hjelm et al. 2018, Deep InfoMax: follows its intuitions diff --git a/src/gnn/PLAN.md b/src/gnn/PLAN.md index 25bdcfb..fefff91 100644 --- a/src/gnn/PLAN.md +++ b/src/gnn/PLAN.md @@ -57,24 +57,24 @@ One PR each, named for what it does. | Scorer | dev found | dev Recall@10 / nDCG@10 | test found | test Recall@10 / nDCG@10 | |---|---|---|---|---| - | learned (model and content, served) | 23/32 | 0.837 / 0.695 | 17/31 | 0.614 / 0.542 | - | model only | 24/32 | 0.806 / 0.594 | 17/31 | 0.553 / 0.344 | - | equal-weight walk | 21/32 | 0.754 / 0.514 | 12/31 | 0.402 / 0.308 | - | time_decayed_pagerank | 22/32 | 0.766 / 0.536 | 13/31 | 0.447 / 0.289 | - | gateway | 4/32 | 0.079 / 0.070 | 3/31 | 0.121 / 0.096 | - | in_degree | 3/32 | 0.181 / 0.175 | 3/31 | 0.121 / 0.109 | + | learned (model and content, served) | 23/32 | 0.837 / 0.690 | 16/31 | 0.598 / 0.509 | + | model only | 24/32 | 0.806 / 0.604 | 16/31 | 0.523 / 0.345 | + | equal-weight walk | 23/32 | 0.794 / 0.528 | 12/31 | 0.402 / 0.300 | + | time_decayed_pagerank | 22/32 | 0.766 / 0.530 | 13/31 | 0.447 / 0.289 | + | gateway | 4/32 | 0.079 / 0.070 | 4/31 | 0.152 / 0.111 | + | in_degree | 3/32 | 0.181 / 0.174 | 3/31 | 0.121 / 0.109 | | path_weight | 0/32 | 0 / 0 | 0/31 | 0 / 0 | - On the filled-in graph, with ChebNet's title corrected (its OpenAlex record carried a lecture's title and abstract, which content weighting read as off-topic). Before the fill the learned ranking found 24/32 dev and 17/31 test, nDCG 0.756 and 0.561. Learned top-10 overlap across seeds 0.98 on both; with 10% of citations dropped 0.92 dev, 0.86 test. Tried on dev and left out: walk damping, age half-life and sharpening the model's weights (a hit either way, noise at 12 papers), and Semantic Scholar's influence rate per work (Adam's is higher than ChebNet's). `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent. + On the filled-in graph with duplicate records folded (`gnn-20260930`). Before the fold the learned ranking found 23/32 dev and 17/31 test, nDCG 0.695 and 0.542; before the fill, 24/32 and 17/31, nDCG 0.756 and 0.561. Learned top-10 overlap across seeds 0.975 dev, 0.961 test; with 10% of citations dropped 0.84 dev, 0.87 test. Tried on dev and left out: walk damping, age half-life and sharpening the model's weights (a hit either way, noise at 12 papers), and Semantic Scholar's influence rate per work (Adam's is higher than ChebNet's). `path_weight` ranks 1980s foundations first, the failure time decay exists to prevent. - [x] **Recovered citations.** `graph.semanticscholar`, an optional stage between crawl and build: 6811 MAG-era works' Semantic Scholar references matched back to crawled works by MAG id, else by title and year. 22582 citations matched, 6808 of them lost from OpenAlex; GCN regains Bruna, Henaff and Weston. Served. - [x] **Citation influence labels.** 5202 Semantic Scholar `isInfluential` labels from 141 non-gold papers. The decoder's plausibility alone predicts them at AUC 0.70, hand features at 0.71, and the model's embeddings add nothing. The flag mostly counts how often a paper mentions a work, so tools earn it too. - [x] **Influence-aligned objective.** Not adopted. Walk weights from a model of those labels tie content weighting on dev (24 against 23 of 32 found) and add nothing on top of it, for an extra pipeline stage and a rate-limited API at ranking time. - [x] **Fill-in of cross-field works.** `crawl.fill_cited_by: 20` fetches, without following, the uncrawled works 20 or more crawled works cite: 1282 candidates, 827 alive, 10332 works in the raw crawl. Recovered citations rise from 6808 to 10344. Gold originators reachable: dev 28/32 to 31/32, test 30/31 to 31/31: Bahdanau, Interaction Networks, Kearnes and DistMult. Found stays 17/31 on test and drops one on dev. The fill also brings in heavily cited datasets and baselines (Planetoid, QM9, TransE, RESCAL), which compete for the same places; telling "built on" from "compared against" needs how a paper cites a work, not whether it does. - [x] **Crawl expansion.** Uncapped backward crawl, plus `forward_depth: 1`: the in-topic works citing the seeds. 2789 works and 15k citations became 9249 and 104k. GAT is seeded by its primary record. Cleaning re-dates 262 works OpenAlex dates years after their citers, and a year heavier than its split's share no longer empties the split after it. Gold originators reachable from their targets: 88/94 to 90/94. The other four are references OpenAlex holds under dead ids (about 12% of all references), among them GCN's to Bruna, Henaff and Planetoid. +- [x] **Folded duplicate records.** OpenAlex holds about 450 crawled works twice, as preprint and venue records that split the work's citations (ChebNet 1446 + 435, DiffPool 415 + 414). Cleaning folds records with one title, one first author's family name and years at most two apart, after re-dating, into the most cited. Ten more carry another work's metadata and are named in `duplicate_works` by what their citers cite on Semantic Scholar: GCN (excluded until now, with 2458 citations), GraphSAGE, GIN, VGAE, Attention Is All You Need, Adam, BERT, Hamilton et al. 2017, APPNP under its first author's former name, and a 2020 book chapter credited with GAT's citations. 10051 works become 9599, and GCN's citations inside the graph rise from 2453 to 3779. The proxy gate still holds, with 15/15 paired MRR wins over logistic on both samplers (+0.023 uniform, +0.014 year-matched). Every scorer's MRR drops (model 0.515 to 0.457), since a citer's second version no longer makes its citations easy to predict. On ranking, dev is flat and test finds one fewer (table above). Adam, now whole, enters GGSNN's top 5: tools competing with originators is still the main error. Optional: blinded LLM pairwise judge, trusted only after it agrees with the gold set. ## Open decisions -- Gold set verification, then commit `fields/gold/gnn.yaml` and rerun `gnn.lineages`. - Abstract embeddings need a new dependency; deferred until originator ranking evaluation shows text is the bottleneck. diff --git a/src/graph/build.py b/src/graph/build.py index 88876d3..8f1d001 100644 --- a/src/graph/build.py +++ b/src/graph/build.py @@ -12,6 +12,7 @@ import json import logging import re +import unicodedata from collections.abc import Iterable, Mapping from dataclasses import asdict, dataclass, field from datetime import date @@ -29,6 +30,9 @@ # A preprint can be cited a year before the venue date OpenAlex gives it. PREPRINT_SLACK = 1 +# A preprint and its venue version are dated at most this many years apart. +VERSION_GAP = 2 + # Citations `graph.semanticscholar` recovered, kept apart from the crawl's own. RECOVERED_EDGES = "recovered_edges.parquet" @@ -56,6 +60,7 @@ class CleaningReport: raw_edges: int = 0 excluded_nodes: int = 0 retitled_nodes: int = 0 + merged_nodes: int = 0 renamed_authors: int = 0 duplicate_nodes: int = 0 duplicate_edges: int = 0 @@ -145,6 +150,7 @@ def clean( date_range: DateRange | None = None, exclude: Iterable[str] = (), retitle: Mapping[str, str] | None = None, + merge: Mapping[str, str] | None = None, largest_component_only: bool = True, ) -> tuple[pd.DataFrame, pd.DataFrame, CleaningReport]: """Reduce raw crawl tables to a consistent node set and edge set.""" @@ -167,6 +173,11 @@ def clean( if "authors" in nodes: nodes["authors"], report.renamed_authors = _rename_authors(nodes["authors"]) + held = len(nodes) + nodes, edges = _fold(nodes, edges, merge or {}) + nodes, edges = _fold(nodes, edges, _versions(nodes, edges)) + report.merged_nodes = held - len(nodes) + if date_range is not None: in_range = nodes["publication_year"].map(date_range.contains) report.out_of_range_nodes = int((~in_range).sum()) @@ -196,6 +207,75 @@ def clean( return nodes, edges, report +def _fold( + nodes: pd.DataFrame, edges: pd.DataFrame, duplicates: Mapping[str, str] +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Fold each duplicate record into the one it duplicates, citations and all.""" + held = set(nodes["id"]) + duplicates = {dup: kept for dup, kept in duplicates.items() if dup in held and kept in held} + if not duplicates: + return nodes, edges + + edges = edges.assign( + source=edges["source"].map(duplicates).fillna(edges["source"]), + target=edges["target"].map(duplicates).fillna(edges["target"]), + ) + folded = nodes["id"].isin(duplicates.keys()) + if "abstract" in nodes: + spare = nodes[folded & ~nodes["abstract"].map(is_missing)] + spare = spare.set_index(spare["id"].map(duplicates))["abstract"] + spare = spare[~spare.index.duplicated()] + lacking = ~folded & nodes["abstract"].map(is_missing) & nodes["id"].isin(spare.index) + nodes = nodes.copy() + nodes.loc[lacking, "abstract"] = nodes.loc[lacking, "id"].map(spare) + return nodes[~folded], edges + + +def _versions(nodes: pd.DataFrame, edges: pd.DataFrame) -> dict[str, str]: + """Preprint and venue records of one work, each mapped to the one the crawl cites most. + + They share a title, a first author's family name and roughly a year. A title + alone is not enough: `Deep learning` is both LeCun et al. 2015 and Goodfellow + et al. 2016. A given name is too much: DiffPool's first author is both Rex and + Zhitao Ying. + """ + if "authors" not in nodes or nodes.empty: + return {} + # A version OpenAlex dates by a late reprint is dated by its citers instead. + records, _ = _redate(nodes, edges) + records = records.assign( + key=records["title"].map(_title_key), + first=records["authors"].map(_first_author), + cited=records["id"].map(edges["target"].value_counts()).fillna(0), + ) + records = records[(records["key"] != "") & (records["first"] != "")] + records = records[records.duplicated(["key", "first"], keep=False)] + duplicates = {} + for _, group in records.sort_values(["cited", "id"], ascending=[False, True]).groupby( + ["key", "first"], sort=False + ): + kept, *others = group.itertuples(index=False) + for other in others: + if abs(other.publication_year - kept.publication_year) <= VERSION_GAP: + duplicates[other.id] = kept.id + return duplicates + + +def _title_key(title: object) -> str: + return "" if is_missing(title) else re.sub(r"[^a-z0-9]", "", str(title).lower()) + + +def _first_author(names: object) -> str: + """The first author's family name; a name with no Latin letters as written.""" + if not isinstance(names, list | tuple | np.ndarray) or len(names) == 0: + return "" + name = str(names[0]) + words = re.findall( + r"[a-z]+", unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode().lower() + ) + return words[-1] if words else "".join(name.split()) + + def _redate(nodes: pd.DataFrame, edges: pd.DataFrame) -> tuple[pd.DataFrame, int]: """Pull back any work dated well after the works that cite it. @@ -323,6 +403,7 @@ def build_field( date_range=config.crawl.date_range, exclude=config.exclude_works, retitle=config.corrected_titles, + merge=config.duplicate_works, largest_component_only=largest_component_only, ) surviving = set(nodes["id"]) diff --git a/src/graph/config.py b/src/graph/config.py index c4376a4..6bd7173 100644 --- a/src/graph/config.py +++ b/src/graph/config.py @@ -35,6 +35,7 @@ class FieldConfig(BaseModel): seed_papers: list[str] = Field(default_factory=list) exclude_works: list[str] = Field(default_factory=list) corrected_titles: dict[str, str] = Field(default_factory=dict) + duplicate_works: dict[str, str] = Field(default_factory=dict) crawl: CrawlConfig @field_validator("description") diff --git a/tests/test_build.py b/tests/test_build.py index 938bb2e..ec8bea9 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -18,7 +18,7 @@ from graph.config import DateRange -def node(work_id, year=2015, abstract="an abstract", title=None): +def node(work_id, year=2015, abstract="an abstract", title=None, authors=("Ada Lovelace",)): return { "id": work_id, "title": title if title is not None else f"Paper {work_id}", @@ -27,7 +27,7 @@ def node(work_id, year=2015, abstract="an abstract", title=None): "publication_date": f"{year}-01-01", "cited_by_count": 10, "topic_id": "T1", - "authors": ["Ada Lovelace"], + "authors": list(authors), } @@ -86,6 +86,83 @@ def test_a_record_filed_under_another_title_gets_its_own_and_loses_the_abstract( assert kept.loc["A", "title"] == "Paper A" +def test_a_named_duplicate_is_folded_into_its_work_with_its_citations(): + nodes, edges = frames( + [node("A"), node("B"), node("C", title="A tutoring paper"), node("D")], + [("A", "B"), ("A", "C"), ("D", "C")], + ) + + nodes, edges, report = clean(nodes, edges, merge={"C": "B"}) + + assert report.merged_nodes == 1 + assert sorted(nodes["id"]) == ["A", "B", "D"] + assert sorted(edges.itertuples(index=False, name=None)) == [("A", "B"), ("D", "B")] + + +def test_a_duplicate_of_a_work_outside_the_crawl_is_left_alone(): + nodes, edges = frames([node("A"), node("C")], [("A", "C")]) + + nodes, _, report = clean(nodes, edges, merge={"C": "B"}) + + assert report.merged_nodes == 0 + assert sorted(nodes["id"]) == ["A", "C"] + + +def test_a_preprint_folds_into_its_more_cited_venue_version(): + nodes, edges = frames( + [ + node("A"), + node("D"), + node("E"), + node("P", 2016, None, "How Powerful are GNNs?", ["K. Xu"]), + node("V", 2018, "an abstract", "How powerful are GNNs", ["Keyulu Xu", "Weihua Hu"]), + ], + [("A", "V"), ("D", "V"), ("E", "P")], + ) + + nodes, edges, report = clean(nodes, edges) + + assert report.merged_nodes == 1 + assert "P" not in set(nodes["id"]) + assert sorted(edges["target"]) == ["V", "V", "V"] + + +def test_the_kept_version_takes_an_abstract_only_its_duplicate_has(): + nodes, edges = frames( + [node("A"), node("D"), node("P", abstract="the real one"), node("V", abstract=None)], + [("A", "V"), ("D", "V"), ("A", "P")], + ) + nodes["title"] = nodes["id"].map({"P": "ChebNet", "V": "ChebNet"}).fillna(nodes["title"]) + + nodes, _, _ = clean(nodes, edges) + + assert nodes.set_index("id").loc["V", "abstract"] == "the real one" + + +@pytest.mark.parametrize( + "other", + [ + node("Q", 2016, title="Deep learning", authors=["Ian Goodfellow", "Yoshua Bengio"]), + node("Q", 2021, title="Deep learning", authors=["Yann LeCun"]), + ], +) +def test_one_title_by_another_first_author_or_years_apart_is_another_work(other): + nodes, edges = frames( + [ + node("A"), + node("B", 2022), + node("P", 2015, title="Deep learning", authors=["Yann LeCun", "Yoshua Bengio"]), + other, + ], + [("A", "P"), ("B", "Q"), ("B", "A")], + ) + + nodes, _, report = clean(nodes, edges) + + assert report.merged_nodes == 0 + assert {"P", "Q"} <= set(nodes["id"]) + + @pytest.mark.parametrize( "catalogued,shown", [