diff --git a/README.md b/README.md index 1822ff8..383fbe3 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ description: > Neural network architectures that operate directly on graph-structured data... topic_id: null # OpenAlex topic delimiting the field seed_papers: [] # OpenAlex work IDs the crawl starts from +exclude_works: [] # records whose metadata OpenAlex got wrong crawl: hop_depth: 2 # references, backwards from the seeds forward_depth: 1 # works citing the seeds, within topic_id diff --git a/fields/gnn.yaml b/fields/gnn.yaml index b995ff6..76acdba 100644 --- a/fields/gnn.yaml +++ b/fields/gnn.yaml @@ -31,6 +31,12 @@ seed_papers: - W2624431344 # Inductive Representation Learning on Large Graphs (2017) - W2766453196 # Graph Attention Networks (2017) +# Works dropped at cleaning: OpenAlex records whose metadata is wrong in a way +# the pipeline cannot detect, so they would surface in rankings under a name +# that is not theirs. +exclude_works: + - W2964015378 # cited 2347 times inside the graph, titled as a 2025 vulnerability-detection paper + crawl: # How many citation hops to expand outward from the seeds. hop_depth: 2 diff --git a/src/graph/build.py b/src/graph/build.py index 1922458..785d2cd 100644 --- a/src/graph/build.py +++ b/src/graph/build.py @@ -11,6 +11,7 @@ import argparse import json import logging +from collections.abc import Iterable from dataclasses import asdict, dataclass, field from datetime import date from pathlib import Path @@ -44,6 +45,7 @@ class CleaningReport: raw_nodes: int = 0 raw_edges: int = 0 + excluded_nodes: int = 0 duplicate_nodes: int = 0 duplicate_edges: int = 0 out_of_range_nodes: int = 0 @@ -104,6 +106,7 @@ def clean( edges: pd.DataFrame, *, date_range: DateRange | None = None, + exclude: Iterable[str] = (), largest_component_only: bool = True, ) -> tuple[pd.DataFrame, pd.DataFrame, CleaningReport]: """Reduce raw crawl tables to a consistent node set and edge set.""" @@ -112,6 +115,10 @@ def clean( nodes = nodes.drop_duplicates(subset="id") report.duplicate_nodes = report.raw_nodes - len(nodes) + excluded = nodes["id"].isin(set(exclude)) + report.excluded_nodes = int(excluded.sum()) + nodes = nodes[~excluded] + if date_range is not None: in_range = nodes["publication_year"].map(date_range.contains) report.out_of_range_nodes = int((~in_range).sum()) @@ -266,6 +273,7 @@ def build_field( nodes, edges, date_range=config.crawl.date_range, + exclude=config.exclude_works, largest_component_only=largest_component_only, ) surviving = set(nodes["id"]) diff --git a/src/graph/config.py b/src/graph/config.py index 89219fe..cbf5ab1 100644 --- a/src/graph/config.py +++ b/src/graph/config.py @@ -32,6 +32,7 @@ class FieldConfig(BaseModel): description: str topic_id: str | None = None seed_papers: list[str] = Field(default_factory=list) + exclude_works: list[str] = Field(default_factory=list) crawl: CrawlConfig @field_validator("description") diff --git a/tests/test_build.py b/tests/test_build.py index 0855e18..36d56f9 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -62,6 +62,16 @@ def test_edges_past_the_crawl_horizon_are_dropped(): assert list(edges["target"]) == ["B"] +def test_excluded_works_are_dropped_with_their_edges(): + nodes, edges = frames([node("A"), node("B"), node("C")], [("A", "B"), ("A", "C")]) + + nodes, edges, report = clean(nodes, edges, exclude=["B"]) + + assert report.excluded_nodes == 1 + assert sorted(nodes["id"]) == ["A", "C"] + assert list(edges.itertuples(index=False, name=None)) == [("A", "C")] + + def test_nodes_outside_the_date_range_are_dropped_with_their_edges(): nodes, edges = frames( [node("A", year=2015), node("B", year=1970), node("C", year=2016)],