diff --git a/CITATION.cff b/CITATION.cff index 74abbde..3eb8c58 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,7 +3,22 @@ title: "VCBench: a capability-stratified benchmark for single-cell foundation mo message: "If you use this benchmark in your research, please cite both the preprint and the software release." type: software authors: - - name: "VCBench contributors" + - family-names: Weidener + given-names: L. + affiliation: "Applied Scientific Intelligence, Inc." + email: lukas@appliedscientific.ai + - family-names: Brkić + given-names: M. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Jovanović + given-names: M. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Ulgac + given-names: E. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Meduri + given-names: A. + affiliation: "Applied Scientific Intelligence, Inc." repository-code: "https://github.com/AppliedScientific/VCBench" license: MIT keywords: @@ -16,9 +31,18 @@ keywords: - reproducibility preferred-citation: type: article - title: "VCBench: a capability-stratified benchmark for single-cell foundation models" + title: "VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models" authors: - - name: "VCBench contributors" - journal: "bioRxiv" + - family-names: Weidener + given-names: L. + - family-names: Brkić + given-names: M. + - family-names: Jovanović + given-names: M. + - family-names: Ulgac + given-names: E. + - family-names: Meduri + given-names: A. year: 2026 - notes: "Preprint DOI and Zenodo software DOI available from the repository." + doi: "10.64898/2026.06.18.733146" + url: "https://doi.org/10.64898/2026.06.18.733146" diff --git a/README.md b/README.md index 41ce84f..887ae52 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ # VCBench +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![DOI](https://img.shields.io/badge/DOI-10.64898%2F2026.06.18.733146-blue.svg)](https://doi.org/10.64898/2026.06.18.733146) +[![Leaderboard](https://img.shields.io/badge/%F0%9F%8F%86%20leaderboard-HF%20Space-ff9d00.svg)](https://huggingface.co/spaces/appliedscientific/vcbench-leaderboard) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20models%20%26%20data-appliedscientific-ffce1c.svg)](https://huggingface.co/appliedscientific) +[![tests](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml/badge.svg)](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml) +[![Applied Scientific Intelligence](https://img.shields.io/badge/lab-appliedscientific.ai-000000.svg)](https://appliedscientific.ai) + VCBench is a capability-stratified benchmark for single-cell foundation models, evaluating five models against pre-registered baselines across five dimensions. > **First-time visitors — start here.** The v1.0.0 release reconciles VCBench's evaluator with upstream cell-eval to numerical precision via an explicit anchor-convention parameter. The Arc State checkpoint lives publicly at [`huggingface.co/appliedscientific/arc-state-norman-gears-corrected`](https://huggingface.co/appliedscientific/arc-state-norman-gears-corrected) — with paste-able reproduction snippets that recover the headline numbers in <5 min on CPU. -> -> *Repository access:* The canonical company-org repository at [`AppliedScientific/VCBench`](https://github.com/AppliedScientific/VCBench) is private during peer review. Contact the corresponding author for read access (URLs of the form `github.com/AppliedScientific/VCBench/...` will return 404 without an invite). The companion HuggingFace artefact + reproduction snippets are public and require no access request. ## What it produces @@ -145,3 +150,28 @@ snapshot_download("appliedscientific/vcbench-embeddings", repo_type="dataset") - **GPU:** A100 80GB recommended (required for UCE; 40GB sufficient for others) - **Baseline construction:** CPU-only, no GPU needed - **Estimated GPU time:** ~75 GPU-hours total + +## License + +VCBench is released under the [MIT License](LICENSE). + +## Citation + +If you use VCBench in your research, please cite both the preprint and the +software release. Machine-readable metadata is in [`CITATION.cff`](CITATION.cff). + +**Preprint** — Weidener, L., Brkić, M., Jovanović, M., Ulgac, E., Meduri, A. +*VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models*, +Applied Scientific Intelligence, Inc. (2026). +[doi:10.64898/2026.06.18.733146](https://doi.org/10.64898/2026.06.18.733146) + +```bibtex +@article{weidener2026vcbench, + title = {VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models}, + author = {Weidener, L. and Brki\'{c}, M. and Jovanovi\'{c}, M. and Ulgac, E. and Meduri, A.}, + year = {2026}, + doi = {10.64898/2026.06.18.733146}, + url = {https://doi.org/10.64898/2026.06.18.733146}, + note = {Preprint} +} +``` diff --git a/pyproject.toml b/pyproject.toml index fed8c75..5682410 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "scipy>=1.10", "pydantic>=2.0", "pyyaml>=6.0", + "tomli>=2.0; python_version < '3.11'", ] [project.optional-dependencies] diff --git a/results/dim_c/geneformer/predicted_edges.csv b/results/dim_c/geneformer/predicted_edges.csv deleted file mode 120000 index f8f2d8f..0000000 --- a/results/dim_c/geneformer/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/geneformer/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/scgpt/predicted_edges.csv b/results/dim_c/scgpt/predicted_edges.csv deleted file mode 120000 index 6408cf2..0000000 --- a/results/dim_c/scgpt/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/scgpt/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/transcriptformer/predicted_edges.csv b/results/dim_c/transcriptformer/predicted_edges.csv deleted file mode 120000 index ff56b52..0000000 --- a/results/dim_c/transcriptformer/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/transcriptformer/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/uce/predicted_edges.csv b/results/dim_c/uce/predicted_edges.csv deleted file mode 120000 index 5f94797..0000000 --- a/results/dim_c/uce/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/uce/predicted_edges.csv \ No newline at end of file diff --git a/src/vcbench/models/arc_state.py b/src/vcbench/models/arc_state.py index 87c0d4c..62e086f 100644 --- a/src/vcbench/models/arc_state.py +++ b/src/vcbench/models/arc_state.py @@ -14,10 +14,14 @@ from __future__ import annotations import os -import tomllib from pathlib import Path from typing import TYPE_CHECKING +try: # tomllib is stdlib on Python >= 3.11; fall back to the tomli backport + import tomllib +except ModuleNotFoundError: # Python 3.9 / 3.10 + import tomli as tomllib + from vcbench.models.base import FoundationModel if TYPE_CHECKING: # pragma: no cover