From b2dfa4f68ea9e54314ecc48962892a27f0a22551 Mon Sep 17 00:00:00 2001 From: Mihailo Jovanovic Date: Wed, 24 Jun 2026 18:08:22 +0200 Subject: [PATCH 1/4] fix(ci): remove dead absolute symlinks to private /root/hf_cache The four results/dim_c/{geneformer,scgpt,transcriptformer,uce}/predicted_edges.csv files were committed as absolute symlinks pointing to /root/hf_cache/embeddings/dim_c//predicted_edges.csv, a path that only exists in the original (root) generation environment. On GitHub Actions, actions/setup-python (cache: pip) globs the repo tree to hash dependency files; that traversal follows these symlinks into /root (mode 700, not searchable by the runner user) and fails with EACCES, killing the 'tests' and 'baselines-drift' jobs before any test runs. The 'docker' job runs as root so it was unaffected. No test depends on these files: the referencing tests in tests/test_correctness.py and tests/test_phase_outputs.py pytest.skip when the CSV is missing, and CI only runs tests/unit/. The derived grn_eval_*.json results remain in place. The raw predicted-edge CSVs for these foundation models were never committed to the public repo (only the private symlink was). --- results/dim_c/geneformer/predicted_edges.csv | 1 - results/dim_c/scgpt/predicted_edges.csv | 1 - results/dim_c/transcriptformer/predicted_edges.csv | 1 - results/dim_c/uce/predicted_edges.csv | 1 - 4 files changed, 4 deletions(-) delete mode 120000 results/dim_c/geneformer/predicted_edges.csv delete mode 120000 results/dim_c/scgpt/predicted_edges.csv delete mode 120000 results/dim_c/transcriptformer/predicted_edges.csv delete mode 120000 results/dim_c/uce/predicted_edges.csv diff --git a/results/dim_c/geneformer/predicted_edges.csv b/results/dim_c/geneformer/predicted_edges.csv deleted file mode 120000 index f8f2d8f..0000000 --- a/results/dim_c/geneformer/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/geneformer/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/scgpt/predicted_edges.csv b/results/dim_c/scgpt/predicted_edges.csv deleted file mode 120000 index 6408cf2..0000000 --- a/results/dim_c/scgpt/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/scgpt/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/transcriptformer/predicted_edges.csv b/results/dim_c/transcriptformer/predicted_edges.csv deleted file mode 120000 index ff56b52..0000000 --- a/results/dim_c/transcriptformer/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/transcriptformer/predicted_edges.csv \ No newline at end of file diff --git a/results/dim_c/uce/predicted_edges.csv b/results/dim_c/uce/predicted_edges.csv deleted file mode 120000 index 5f94797..0000000 --- a/results/dim_c/uce/predicted_edges.csv +++ /dev/null @@ -1 +0,0 @@ -/root/hf_cache/embeddings/dim_c/uce/predicted_edges.csv \ No newline at end of file From 4ea0c3ccca0b1ae9419a5d939ad6cd700789c9a4 Mon Sep 17 00:00:00 2001 From: Mihailo Jovanovic Date: Wed, 24 Jun 2026 18:12:28 +0200 Subject: [PATCH 2/4] fix(py310): tomllib backport + drop stale private-access note - arc_state.py imported tomllib unconditionally, but tomllib is stdlib only on Python >= 3.11, so the unit-test collection crashed on the 3.10 CI matrix leg (ModuleNotFoundError: No module named 'tomllib'). Use the standard try/except fallback to the 'tomli' backport, and declare 'tomli; python_version < 3.11' as a dependency. requires-python is >=3.9 and 3.9/3.10 are advertised in the classifiers, so the backport (not dropping 3.10) is the correct fix. - README: remove the stale 'Repository access' note that described the repo as a private, invite-only company mirror during peer review. The repository is public and open source; that paragraph no longer applies. --- README.md | 2 -- pyproject.toml | 1 + src/vcbench/models/arc_state.py | 6 +++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41ce84f..4c4f27e 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,6 @@ VCBench is a capability-stratified benchmark for single-cell foundation models, evaluating five models against pre-registered baselines across five dimensions. > **First-time visitors — start here.** The v1.0.0 release reconciles VCBench's evaluator with upstream cell-eval to numerical precision via an explicit anchor-convention parameter. The Arc State checkpoint lives publicly at [`huggingface.co/appliedscientific/arc-state-norman-gears-corrected`](https://huggingface.co/appliedscientific/arc-state-norman-gears-corrected) — with paste-able reproduction snippets that recover the headline numbers in <5 min on CPU. -> -> *Repository access:* The canonical company-org repository at [`AppliedScientific/VCBench`](https://github.com/AppliedScientific/VCBench) is private during peer review. Contact the corresponding author for read access (URLs of the form `github.com/AppliedScientific/VCBench/...` will return 404 without an invite). The companion HuggingFace artefact + reproduction snippets are public and require no access request. ## What it produces diff --git a/pyproject.toml b/pyproject.toml index fed8c75..5682410 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "scipy>=1.10", "pydantic>=2.0", "pyyaml>=6.0", + "tomli>=2.0; python_version < '3.11'", ] [project.optional-dependencies] diff --git a/src/vcbench/models/arc_state.py b/src/vcbench/models/arc_state.py index 87c0d4c..62e086f 100644 --- a/src/vcbench/models/arc_state.py +++ b/src/vcbench/models/arc_state.py @@ -14,10 +14,14 @@ from __future__ import annotations import os -import tomllib from pathlib import Path from typing import TYPE_CHECKING +try: # tomllib is stdlib on Python >= 3.11; fall back to the tomli backport + import tomllib +except ModuleNotFoundError: # Python 3.9 / 3.10 + import tomli as tomllib + from vcbench.models.base import FoundationModel if TYPE_CHECKING: # pragma: no cover From 6258116ff07a2b994ede0168995309a0769b928f Mon Sep 17 00:00:00 2001 From: Mihailo Jovanovic Date: Wed, 24 Jun 2026 18:15:47 +0200 Subject: [PATCH 3/4] docs: add README badges + Citation section, wire preprint DOI - README: add shields.io badge row (MIT license, bioRxiv preprint, Hugging Face models & data, CI tests status, appliedscientific.ai lab) mirroring the CardioSafe-benchmark style for org consistency. - README: add ## License and ## Citation sections at the bottom with the preprint DOI (10.64898/2026.06.18.733146) and a BibTeX entry. - CITATION.cff: replace the 'DOI available from the repository' note with the actual preprint doi/url fields. --- CITATION.cff | 3 ++- README.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 74abbde..1de17b4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -21,4 +21,5 @@ preferred-citation: - name: "VCBench contributors" journal: "bioRxiv" year: 2026 - notes: "Preprint DOI and Zenodo software DOI available from the repository." + doi: "10.64898/2026.06.18.733146" + url: "https://doi.org/10.64898/2026.06.18.733146" diff --git a/README.md b/README.md index 4c4f27e..0475d3f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # VCBench +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![Preprint: bioRxiv](https://img.shields.io/badge/preprint-bioRxiv-b31b1b.svg)](https://doi.org/10.64898/2026.06.18.733146) +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20models%20%26%20data-appliedscientific-ffce1c.svg)](https://huggingface.co/appliedscientific) +[![tests](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml/badge.svg)](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml) +[![Applied Scientific Intelligence](https://img.shields.io/badge/lab-appliedscientific.ai-000000.svg)](https://appliedscientific.ai) + VCBench is a capability-stratified benchmark for single-cell foundation models, evaluating five models against pre-registered baselines across five dimensions. > **First-time visitors — start here.** The v1.0.0 release reconciles VCBench's evaluator with upstream cell-eval to numerical precision via an explicit anchor-convention parameter. The Arc State checkpoint lives publicly at [`huggingface.co/appliedscientific/arc-state-norman-gears-corrected`](https://huggingface.co/appliedscientific/arc-state-norman-gears-corrected) — with paste-able reproduction snippets that recover the headline numbers in <5 min on CPU. @@ -143,3 +149,26 @@ snapshot_download("appliedscientific/vcbench-embeddings", repo_type="dataset") - **GPU:** A100 80GB recommended (required for UCE; 40GB sufficient for others) - **Baseline construction:** CPU-only, no GPU needed - **Estimated GPU time:** ~75 GPU-hours total + +## License + +VCBench is released under the [MIT License](LICENSE). + +## Citation + +If you use VCBench in your research, please cite both the preprint and the +software release. Machine-readable metadata is in [`CITATION.cff`](CITATION.cff). + +**Preprint** — *VCBench: a capability-stratified benchmark for single-cell +foundation models* (2026), [doi:10.64898/2026.06.18.733146](https://doi.org/10.64898/2026.06.18.733146). + +```bibtex +@article{vcbench2026, + title = {VCBench: a capability-stratified benchmark for single-cell foundation models}, + author = {VCBench contributors}, + year = {2026}, + journal = {bioRxiv}, + doi = {10.64898/2026.06.18.733146}, + url = {https://doi.org/10.64898/2026.06.18.733146} +} +``` From 2e60022701c716113aeb31202fe530f2eae6fcc5 Mon Sep 17 00:00:00 2001 From: Mihailo Jovanovic Date: Wed, 24 Jun 2026 18:38:25 +0200 Subject: [PATCH 4/4] docs: finalize citation (real authors/title) + DOI & leaderboard badges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Relabel preprint badge bioRxiv -> DOI (per maintainer preference) and add a Hugging Face Leaderboard Space badge (huggingface.co/spaces/appliedscientific/vcbench-leaderboard). - Citation: use the published title 'VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models' and the real author list (Weidener, Brkić, Jovanović, Ulgac, Meduri; Applied Scientific Intelligence, Inc.). Drop the bioRxiv journal field in favour of the DOI. - CITATION.cff: same authors/title; corresponding-author email on Weidener. --- CITATION.cff | 31 +++++++++++++++++++++++++++---- README.md | 19 +++++++++++-------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 1de17b4..3eb8c58 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -3,7 +3,22 @@ title: "VCBench: a capability-stratified benchmark for single-cell foundation mo message: "If you use this benchmark in your research, please cite both the preprint and the software release." type: software authors: - - name: "VCBench contributors" + - family-names: Weidener + given-names: L. + affiliation: "Applied Scientific Intelligence, Inc." + email: lukas@appliedscientific.ai + - family-names: Brkić + given-names: M. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Jovanović + given-names: M. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Ulgac + given-names: E. + affiliation: "Applied Scientific Intelligence, Inc." + - family-names: Meduri + given-names: A. + affiliation: "Applied Scientific Intelligence, Inc." repository-code: "https://github.com/AppliedScientific/VCBench" license: MIT keywords: @@ -16,10 +31,18 @@ keywords: - reproducibility preferred-citation: type: article - title: "VCBench: a capability-stratified benchmark for single-cell foundation models" + title: "VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models" authors: - - name: "VCBench contributors" - journal: "bioRxiv" + - family-names: Weidener + given-names: L. + - family-names: Brkić + given-names: M. + - family-names: Jovanović + given-names: M. + - family-names: Ulgac + given-names: E. + - family-names: Meduri + given-names: A. year: 2026 doi: "10.64898/2026.06.18.733146" url: "https://doi.org/10.64898/2026.06.18.733146" diff --git a/README.md b/README.md index 0475d3f..887ae52 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # VCBench [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Preprint: bioRxiv](https://img.shields.io/badge/preprint-bioRxiv-b31b1b.svg)](https://doi.org/10.64898/2026.06.18.733146) +[![DOI](https://img.shields.io/badge/DOI-10.64898%2F2026.06.18.733146-blue.svg)](https://doi.org/10.64898/2026.06.18.733146) +[![Leaderboard](https://img.shields.io/badge/%F0%9F%8F%86%20leaderboard-HF%20Space-ff9d00.svg)](https://huggingface.co/spaces/appliedscientific/vcbench-leaderboard) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20models%20%26%20data-appliedscientific-ffce1c.svg)](https://huggingface.co/appliedscientific) [![tests](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml/badge.svg)](https://github.com/AppliedScientific/VCBench/actions/workflows/test.yml) [![Applied Scientific Intelligence](https://img.shields.io/badge/lab-appliedscientific.ai-000000.svg)](https://appliedscientific.ai) @@ -159,16 +160,18 @@ VCBench is released under the [MIT License](LICENSE). If you use VCBench in your research, please cite both the preprint and the software release. Machine-readable metadata is in [`CITATION.cff`](CITATION.cff). -**Preprint** — *VCBench: a capability-stratified benchmark for single-cell -foundation models* (2026), [doi:10.64898/2026.06.18.733146](https://doi.org/10.64898/2026.06.18.733146). +**Preprint** — Weidener, L., Brkić, M., Jovanović, M., Ulgac, E., Meduri, A. +*VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models*, +Applied Scientific Intelligence, Inc. (2026). +[doi:10.64898/2026.06.18.733146](https://doi.org/10.64898/2026.06.18.733146) ```bibtex -@article{vcbench2026, - title = {VCBench: a capability-stratified benchmark for single-cell foundation models}, - author = {VCBench contributors}, +@article{weidener2026vcbench, + title = {VCBench: A Multi-Dimensional Benchmark for Single-Cell Foundation Models}, + author = {Weidener, L. and Brki\'{c}, M. and Jovanovi\'{c}, M. and Ulgac, E. and Meduri, A.}, year = {2026}, - journal = {bioRxiv}, doi = {10.64898/2026.06.18.733146}, - url = {https://doi.org/10.64898/2026.06.18.733146} + url = {https://doi.org/10.64898/2026.06.18.733146}, + note = {Preprint} } ```