From 48c837db2e4457d68dd333bfc182bf2d7a7f6600 Mon Sep 17 00:00:00 2001 From: Radin Hamidi Rad Date: Tue, 23 Jun 2026 21:54:22 -0400 Subject: [PATCH] leaderboard: method paper citations, dataset HuggingFace links, model icons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Method dropdown on the home page now lists a paper link per method, sourced from the curated method→paper map shared with the marketing site. - Datasets carry HuggingFace subset links (BEIR, BRIGHT, TREC-DL), shown on the dataset cards, dataset detail headers, and beside the dataset selector. - Models render polished display names (e.g. GPT-4.1, Qwen2.5 72B Instruct) with a provider logo icon in the home table LLM column and on model cards. Co-Authored-By: Claude Opus 4.8 --- reproducibility/site/scripts/build-data.ts | 100 +++++++++++++++++- reproducibility/site/src/lib/providerIcon.ts | 26 +++++ .../site/src/pages/datasets/[id].astro | 10 ++ .../site/src/pages/datasets/index.astro | 10 ++ reproducibility/site/src/pages/index.astro | 98 +++++++++++++++-- .../site/src/pages/methods/index.astro | 10 ++ .../site/src/pages/models/index.astro | 6 +- reproducibility/site/src/styles/global.css | 65 ++++++++++++ 8 files changed, 310 insertions(+), 15 deletions(-) create mode 100644 reproducibility/site/src/lib/providerIcon.ts diff --git a/reproducibility/site/scripts/build-data.ts b/reproducibility/site/scripts/build-data.ts index d542cb0..ac29093 100644 --- a/reproducibility/site/scripts/build-data.ts +++ b/reproducibility/site/scripts/build-data.ts @@ -11,9 +11,9 @@ * * Outputs (gitignored, written to src/data/): * - overview.json summary used on / - * - datasets.json [{id, name, run_count, eval_metrics}] - * - methods.json [{id, run_count}] - * - models.json [{id, run_count}] + * - datasets.json [{id, name, run_count, eval_metrics, hf_url?}] + * - methods.json [{id, run_count, display, paper?, paper_url?}] + * - models.json [{id, run_count, display, provider, slug}] * - retrievers.json [{id, display_name, paradigm, run_count}] * - matrix.json flat matrix: rows = (method, model, retriever), values per dataset * - runs.json full run index, keyed by run_id @@ -45,6 +45,17 @@ const RETRIEVER_REGISTRY_YAML = path.join( "reproducibility", "retriever_registry.yaml", ); +// Curated method→paper citations (single source of truth, shared with the +// marketing site). Keyed by logical method id; variant ids (e.g. query2doc-cot) +// fall back to their base id (query2doc). +const METHOD_CITATIONS_JSON = path.join( + REPO_ROOT, + "web", + "site", + "src", + "data", + "methods.json", +); const OUT_DIR = path.join(SITE_ROOT, "src", "data"); const VIEWS_DIR = path.join(OUT_DIR, "views"); @@ -80,8 +91,46 @@ interface DatasetEntry { bm25_weights?: { k1: number; b: number }; eval_metrics: string[]; run_count: number; + hf_url?: string; } +// Canonical HuggingFace dataset/subset pages, keyed by dataset id. BEIR repos +// are first-party (BeIR org); BRIGHT links target the per-domain split in the +// `examples` config of the single xlangai/BRIGHT repo; the MS MARCO / TREC-DL +// links point to the cleanest public HF mirror of each passage task. +const DATASET_HF_URLS: Record = { + "msmarco-v1-passage.trecdl2019": "https://huggingface.co/datasets/whybe-choi/trec-dl-2019", + "msmarco-v1-passage.trecdl2020": "https://huggingface.co/datasets/whybe-choi/trec-dl-2020", + "msmarco-v1-passage.dlhard": "https://huggingface.co/datasets/irds/msmarco-passage_trec-dl-hard", + "beir-v1.0.0-trec-covid": "https://huggingface.co/datasets/BeIR/trec-covid", + "beir-v1.0.0-fiqa": "https://huggingface.co/datasets/BeIR/fiqa", + "beir-v1.0.0-trec-news": "https://huggingface.co/datasets/BeIR/trec-news-generated-queries", + "beir-v1.0.0-arguana": "https://huggingface.co/datasets/BeIR/arguana", + "beir-v1.0.0-dbpedia-entity": "https://huggingface.co/datasets/BeIR/dbpedia-entity", + "beir-v1.0.0-scifact": "https://huggingface.co/datasets/BeIR/scifact", + "bright-biology": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/biology", + "bright-earth-science": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/earth_science", + "bright-economics": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/economics", + "bright-psychology": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/psychology", + "bright-robotics": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/robotics", + "bright-stackoverflow": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/stackoverflow", + "bright-sustainable-living": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/sustainable_living", + "bright-pony": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/pony", + "bright-leetcode": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/leetcode", + "bright-aops": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/aops", + "bright-theoremqa-theorems": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/theoremqa_theorems", + "bright-theoremqa-questions": "https://huggingface.co/datasets/xlangai/BRIGHT/viewer/examples/theoremqa_questions", +}; + +// Polished display name + provider slug per model id. Provider drives the logo +// icon shown next to the model in the UI; falls back to the org prefix. +const MODEL_META: Record = { + "openai/gpt-4.1": { display: "GPT-4.1", provider: "openai" }, + "openai/gpt-4.1-nano": { display: "GPT-4.1 nano", provider: "openai" }, + "Qwen/Qwen2.5-72B-Instruct": { display: "Qwen2.5 72B Instruct", provider: "qwen" }, + "Qwen/Qwen2.5-7B-Instruct": { display: "Qwen2.5 7B Instruct", provider: "qwen" }, +}; + interface RetrieverEntry { id: string; display_name: string; @@ -173,6 +222,29 @@ function readRetrieverRegistry(): Record { + if (!fs.existsSync(METHOD_CITATIONS_JSON)) return {}; + const list = JSON.parse(fs.readFileSync(METHOD_CITATIONS_JSON, "utf-8")) as Array<{ + id: string; + paper?: string; + paper_url?: string; + }>; + const map: Record = {}; + for (const m of list) { + if (m.paper && m.paper_url) map[m.id] = { paper: m.paper, paper_url: m.paper_url }; + } + return map; +} + +// Resolve a (possibly variant) method id to its citation, falling back to the +// base id before the first "-" (e.g. query2doc-cot → query2doc). +function citationFor( + id: string, + cites: Record, +): { paper: string; paper_url: string } | null { + return cites[id] ?? cites[id.split("-")[0]] ?? null; +} + function* iterRunFiles(): Generator { if (!fs.existsSync(RUNS_DIR)) return; const stack = [RUNS_DIR]; @@ -534,10 +606,19 @@ function encodePathSegment(s: string): string { // Strip provider prefix from a model id for display: "openai/gpt-4.1" → "gpt-4.1". // The canonical id stays in the data; this is purely cosmetic. function displayModel(s: string): string { + if (MODEL_META[s]) return MODEL_META[s].display; const i = s.indexOf("/"); return i >= 0 ? s.slice(i + 1) : s; } +function modelProvider(s: string): string { + if (MODEL_META[s]) return MODEL_META[s].provider; + const org = s.includes("/") ? s.slice(0, s.indexOf("/")).toLowerCase() : ""; + if (org.includes("openai")) return "openai"; + if (org.includes("qwen")) return "qwen"; + return ""; +} + // ---------- main ------------------------------------------------------------ function main() { @@ -548,6 +629,7 @@ function main() { const manifest = readManifest(); const datasets = readDatasetRegistry(); const retrieverReg = readRetrieverRegistry(); + const methodCites = readMethodCitations(); const runDetails = readRunDetails(retrieverReg); // Counts. @@ -591,12 +673,21 @@ function main() { const datasetList: DatasetEntry[] = Object.values(datasets).map((d) => ({ ...d, run_count: datasetCounts.get(d.id) ?? 0, + ...(DATASET_HF_URLS[d.id] ? { hf_url: DATASET_HF_URLS[d.id] } : {}), })); datasetList.sort((a, b) => a.id.localeCompare(b.id)); writeJSON(path.join(OUT_DIR, "datasets.json"), datasetList); const methodList = Array.from(methodCounts.entries()) - .map(([id, run_count]) => ({ id, run_count, display: methodDisplay.get(id) ?? id })) + .map(([id, run_count]) => { + const cite = citationFor(id, methodCites); + return { + id, + run_count, + display: methodDisplay.get(id) ?? id, + ...(cite ? { paper: cite.paper, paper_url: cite.paper_url } : {}), + }; + }) .sort((a, b) => a.id.localeCompare(b.id)); writeJSON(path.join(OUT_DIR, "methods.json"), methodList); @@ -604,6 +695,7 @@ function main() { .map(([id, run_count]) => ({ id, display: displayModel(id), + provider: modelProvider(id), run_count, slug: encodePathSegment(id), })) diff --git a/reproducibility/site/src/lib/providerIcon.ts b/reproducibility/site/src/lib/providerIcon.ts new file mode 100644 index 0000000..bc35ff7 --- /dev/null +++ b/reproducibility/site/src/lib/providerIcon.ts @@ -0,0 +1,26 @@ +// Inline monochrome provider logos (currentColor), shared by Astro pages and the +// home-page client script. Returns an empty string for unknown providers so +// callers can render name-only without a placeholder. + +const ICONS: Record = { + // OpenAI mark + openai: + '', + // Qwen mark (Simple Icons) + qwen: + '', +}; + +/** Inline SVG markup for a provider logo, sized to 1em and inheriting color. */ +export function providerIconSvg(provider: string | undefined | null): string { + const body = ICONS[(provider ?? "").toLowerCase()]; + if (!body) return ""; + return ``; +} + +/** provider → inline SVG map, for passing into client scripts via define:vars. */ +export function providerIconMap(): Record { + const out: Record = {}; + for (const k of Object.keys(ICONS)) out[k] = providerIconSvg(k); + return out; +} diff --git a/reproducibility/site/src/pages/datasets/[id].astro b/reproducibility/site/src/pages/datasets/[id].astro index 936d1d4..1ad8a6a 100644 --- a/reproducibility/site/src/pages/datasets/[id].astro +++ b/reproducibility/site/src/pages/datasets/[id].astro @@ -99,6 +99,16 @@ const colCount = 4 + metricCols.length; ← Datasets

{title}

{id} + {datasetMeta?.hf_url && ( + + HuggingFace ↗ + + )}
All results produced by diff --git a/reproducibility/site/src/pages/datasets/index.astro b/reproducibility/site/src/pages/datasets/index.astro index 12ea1d7..1a4b9ac 100644 --- a/reproducibility/site/src/pages/datasets/index.astro +++ b/reproducibility/site/src/pages/datasets/index.astro @@ -23,6 +23,16 @@ import datasets from "../../data/datasets.json";
{d.id}
+ {d.hf_url && ( + + HuggingFace ↗ + + )} )) } diff --git a/reproducibility/site/src/pages/index.astro b/reproducibility/site/src/pages/index.astro index bee0cea..0b49c1d 100644 --- a/reproducibility/site/src/pages/index.astro +++ b/reproducibility/site/src/pages/index.astro @@ -6,7 +6,19 @@ import runs from "../data/runs.json"; import retrievers from "../data/retrievers.json"; import models from "../data/models.json"; import methods from "../data/methods.json"; +import datasets from "../data/datasets.json"; import { buildReproduceCmds, retrieveHint, evaluateHint, type RunLike } from "../lib/reproduce"; +import { providerIconMap } from "../lib/providerIcon"; + +// id → HuggingFace dataset URL, for the link beside the dataset selector. +const datasetHfMap: Record = Object.fromEntries( + (datasets as any[]).filter((d) => d.hf_url).map((d) => [d.id, d.hf_url]), +); +// id → polished display + provider, for icons in the table's LLM column. +const modelMetaMap: Record = Object.fromEntries( + (models as any[]).map((m) => [m.id, { display: m.display ?? m.id, provider: m.provider ?? "" }]), +); +const PROVIDER_ICONS = providerIconMap(); const populated = overview.run_count > 0; @@ -182,6 +194,7 @@ const jsonLd = JSON.stringify({ Dataset {/* populated entirely by JS on init */} + HuggingFace ↗
@@ -208,12 +221,25 @@ const jsonLd = JSON.stringify({
Method - + {/* Custom dropdown: native