diff --git a/README.md b/README.md index 8a2ac4f..d6773e8 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ An API and command line interface (CLI) are provided to allow users to download * Virchow2: https://huggingface.co/paige-ai/Virchow2 * H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0 * H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1 +* OpenMidnight-1.1: https://huggingface.co/medarc/OpenMidnight-1.1 * OpenMidnight-1.1-SPECTRA: https://huggingface.co/medarc/OpenMidnight-1.1-SPECTRA * H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA * Virchow-SPECTRA: https://huggingface.co/medarc/Virchow-SPECTRA diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index b433552..448eb8a 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *openmidnight11*, *openmidnight11_spectra*, *hoptimus0_spectra*, *midnight12k_spectra*, *virchow_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/openmidnight11.yaml b/src/thunder/config/pretrained_model/openmidnight11.yaml new file mode 100644 index 0000000..258c4f9 --- /dev/null +++ b/src/thunder/config/pretrained_model/openmidnight11.yaml @@ -0,0 +1,8 @@ +model_name: openmidnight11 +type: timm +vlm: false +emb_dim: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/openmidnight1.1/model.safetensors +hf_tag: hf-hub:medarc/OpenMidnight-1.1 + +timm_kwargs: {} diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index 8a09a21..b343f0e 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -102,6 +102,10 @@ "kaiko-ai/midnight", "model.safetensors", ), # Midnight-12k (https://huggingface.co/kaiko-ai/midnight) + "openmidnight11": ( + "medarc/OpenMidnight-1.1", + "model.safetensors", + ), # OpenMidnight-1.1 (https://huggingface.co/medarc/OpenMidnight-1.1) "openmidnight11_spectra": ( "medarc/OpenMidnight-1.1-SPECTRA", "model.safetensors", @@ -189,6 +193,7 @@ def download_models(models: Union[List[str], str]) -> None: * midnight * midnight12k_spectra * openmidnight + * openmidnight11 * openmidnight11_spectra * kaiko_vits8 * kaiko_vits16 @@ -284,10 +289,11 @@ def download_model(model: str) -> None: extra_files.extend(["merges.txt", "vocab.json"]) elif local_dir_tag in [ "midnight", + "openmidnight1.1", "openmidnight1.1spectra", - "hoptimus0spectra", - "midnight12kspectra", - "virchowspectra" + "hoptimus0spectra", + "midnight12kspectra", + "virchowspectra", ]: extra_files = ["config.json"] elif local_dir_tag == "titan": diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index 518505f..417633f 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -380,6 +380,7 @@ def extract_embedding(src, pretrained_model, task_type="linear_probing"): elif model_cfg.model_name in [ "hoptimus0", "hoptimus1", + "openmidnight11", "openmidnight11_spectra", "hoptimus0_spectra", ]: @@ -418,6 +419,7 @@ def get_model_from_name(model_name: str, device: str): * midnight * midnight12k_spectra * openmidnight + * openmidnight11 * openmidnight11_spectra * kaiko_vits8 * kaiko_vits16 diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index 9a61837..849ae27 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -18,6 +18,7 @@ class ModelConstants(Enum): "midnight", "midnight12k_spectra", "openmidnight", + "openmidnight11", "openmidnight11_spectra", "phikon", "phikon2",