diff --git a/README.md b/README.md index f2b18f1..27b95e3 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ An API and command line interface (CLI) are provided to allow users to download * Virchow2: https://huggingface.co/paige-ai/Virchow2 * H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0 * H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1 +* H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA * Midnight-12k-SPECTRA: https://huggingface.co/medarc/Midnight-12k-SPECTRA * GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index 87b17f9..13a1496 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml new file mode 100644 index 0000000..c6581e1 --- /dev/null +++ b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml @@ -0,0 +1,8 @@ +model_name: hoptimus0_spectra +type: timm +vlm: false +emb_dim: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/hoptimus0spectra/model.safetensors +hf_tag: hf-hub:medarc/H-optimus-0-SPECTRA + +timm_kwargs: {} diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index 0a37048..22cb32d 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -30,6 +30,10 @@ "paige-ai/Virchow2", "pytorch_model.bin", ), # Virchow2 (https://huggingface.co/paige-ai/Virchow2) + "hoptimus0_spectra": ( + "medarc/H-optimus-0-SPECTRA", + "model.safetensors", + ), # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA) "hoptimus0": ( "bioptimus/H-optimus-0", "pytorch_model.bin", @@ -167,6 +171,7 @@ def download_models(models: Union[List[str], str]) -> None: * virchow_spectra * virchow2 * hoptimus0 + * hoptimus0_spectra * h0mini * hoptimus1 * genbio-pathfm @@ -272,7 +277,7 @@ def download_model(model: str) -> None: ] if local_dir_tag != "plip": extra_files.extend(["merges.txt", "vocab.json"]) - elif local_dir_tag in ["midnight", "midnight12kspectra", "virchowspectra"]: + elif local_dir_tag in ["midnight", "hoptimus0spectra", "midnight12kspectra", "virchowspectra"]: extra_files = ["config.json"] elif local_dir_tag == "titan": extra_files = [ diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index 74f6345..9c8324b 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -377,7 +377,11 @@ def extract_embedding(src, pretrained_model, task_type="linear_probing"): if model_cfg.model_name == "uni2h": nb_reg = 8 - elif model_cfg.model_name in ["hoptimus0", "hoptimus1"]: + elif model_cfg.model_name in [ + "hoptimus0", + "hoptimus1", + "hoptimus0_spectra", + ]: nb_reg = 4 else: nb_reg = 0 @@ -399,6 +403,7 @@ def get_model_from_name(model_name: str, device: str): * virchow_spectra * virchow2 * hoptimus0 + * hoptimus0_spectra * h0mini * hoptimus1 * genbio-pathfm diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index 9c94b81..da84387 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -7,6 +7,7 @@ class ModelConstants(Enum): "hiboub", "hiboul", "hoptimus0", + "hoptimus0_spectra", "h0mini", "hoptimus1", "genbio-pathfm",