feat: add nanoVLM model support

scieval/config.py

@@ -1738,6 +1738,11 @@
     "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'),
 }
 
+nanovlm_series = {
+    "nanoVLM-230M-8k": partial(NanoVLM, model_path="lusxvr/nanoVLM-230M-8k"),
+    "nanoVLM-460M-8k": partial(NanoVLM, model_path="lusxvr/nanoVLM-460M-8k"),
+}
+
 
 
 internvl_groups = [
@@ -1762,7 +1767,8 @@
     aria_series, smolvlm_series, sail_series, valley_series, vita_series,
     ross_series, emu_series, ola_series, ursa_series, gemma_series,
     long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, 
-    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series,
+    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series,
+    thyme_series, logics_series, nanovlm_series,
 ]
 
 for grp in model_groups:

scieval/vlm/__init__.py

@@ -110,3 +110,4 @@
     QTuneVLChat,
 )
 from .logics import Logics_Thinking
+from .nanovlm import NanoVLM

scieval/vlm/nanovlm.py

@@ -0,0 +1,128 @@
+import os
+import sys
+import warnings
+
+import torch
+from PIL import Image
+
+from .base import BaseModel
+
+
+_NANOVLM_INSTALL_MESSAGE = (
+    "nanoVLM is not distributed as a Python package. Clone "
+    "https://github.com/huggingface/nanoVLM and set NANOVLM_PATH to the "
+    "checkout directory before running SciEvalKit."
+)
+
+
+def _ensure_nanovlm_importable():
+    nanovlm_path = os.environ.get("NANOVLM_PATH", "")
+    if nanovlm_path and nanovlm_path not in sys.path:
+        sys.path.insert(0, nanovlm_path)
+
+
+class NanoVLM(BaseModel):
+    """Adapter for the pure-PyTorch Hugging Face nanoVLM implementation."""
+
+    INSTALL_REQ = True
+    INTERLEAVE = True
+
+    def __init__(
+        self,
+        model_path="lusxvr/nanoVLM-230M-8k",
+        device=None,
+        **kwargs,
+    ):
+        super().__init__()
+        _ensure_nanovlm_importable()
+        try:
+            from data.processors import get_image_processor, get_tokenizer
+            from models.vision_language_model import VisionLanguageModel
+        except ImportError as exc:
+            raise ImportError(_NANOVLM_INSTALL_MESSAGE) from exc
+
+        self.device = torch.device(device or self._default_device())
+        self.model = VisionLanguageModel.from_pretrained(model_path)
+        self.model = self.model.to(self.device).eval()
+        self.config = self.model.cfg
+
+        self.tokenizer = get_tokenizer(
+            self.config.lm_tokenizer,
+            self.config.vlm_extra_tokens,
+            self.config.lm_chat_template,
+        )
+        self.image_processor = get_image_processor(
+            self.config.max_img_size,
+            self.config.vit_img_size,
+            getattr(self.config, "resize_to_max_side_len", False),
+        )
+
+        generation_kwargs = {"max_new_tokens": 2048, "greedy": True}
+        generation_kwargs.update(kwargs)
+        self.kwargs = generation_kwargs
+        warnings.warn(f"NanoVLM kwargs: {self.kwargs}")
+
+    @staticmethod
+    def _default_device():
+        if torch.cuda.is_available():
+            return "cuda"
+        if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            return "mps"
+        return "cpu"
+
+    @staticmethod
+    def _open_image(path):
+        with Image.open(path) as image:
+            return image.convert("RGB")
+
+    def _prepare_images(self, message):
+        processed_images = []
+        image_ratios = []
+        for item in message:
+            if item["type"] != "image":
+                continue
+            image = self._open_image(item["value"])
+            processed, ratio = self.image_processor(image)
+            if (
+                not hasattr(self.tokenizer, "global_image_token")
+                and ratio[0] * ratio[1] == len(processed) - 1
+            ):
+                processed = processed[1:]
+            processed_images.append(processed.to(self.device))
+            image_ratios.append(ratio)
+        return processed_images, image_ratios
+
+    @staticmethod
+    def _message_text(message):
+        return "\n".join(
+            item["value"].strip() for item in message if item["type"] == "text"
+        )
+
+    def generate_inner(self, message, dataset=None):
+        _ensure_nanovlm_importable()
+        try:
+            from data.processors import get_image_string
+        except ImportError as exc:
+            raise ImportError(_NANOVLM_INSTALL_MESSAGE) from exc
+
+        images, image_ratios = self._prepare_images(message)
+        image_string = get_image_string(
+            self.tokenizer, image_ratios, self.config.mp_image_token_length
+        )
+        prompt = image_string + self._message_text(message)
+        conversation = [{"role": "user", "content": prompt}]
+        encoded_prompt = self.tokenizer.apply_chat_template(
+            [conversation], tokenize=True, add_generation_prompt=True
+        )
+        input_ids = torch.as_tensor(encoded_prompt, device=self.device)
+        if input_ids.ndim == 1:
+            input_ids = input_ids.unsqueeze(0)
+
+        generated_ids = self.model.generate(
+            input_ids,
+            images or None,
+            **self.kwargs,
+        )
+        return self.tokenizer.batch_decode(
+            generated_ids, skip_special_tokens=True
+        )[0].strip()

tests/test_nanovlm.py

@@ -0,0 +1,166 @@
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+from PIL import Image
+
+
+class FakeConfig:
+    lm_tokenizer = "fake-tokenizer"
+    vlm_extra_tokens = {"image_token": "<image>"}
+    lm_chat_template = "fake-template"
+    max_img_size = 2048
+    vit_img_size = 512
+    resize_to_max_side_len = True
+    mp_image_token_length = 64
+
+
+class FakeModel:
+    def __init__(self):
+        self.cfg = FakeConfig()
+        self.to_device = None
+        self.eval_called = False
+        self.input_ids = None
+        self.images = None
+        self.generation_kwargs = None
+
+    def to(self, device):
+        self.to_device = device
+        return self
+
+    def eval(self):
+        self.eval_called = True
+        return self
+
+    def generate(self, input_ids, images, **kwargs):
+        self.input_ids = input_ids
+        self.images = images
+        self.generation_kwargs = kwargs
+        return torch.tensor([[7, 8]])
+
+
+class FakeTokenizer:
+    global_image_token = "<global-image>"
+
+    def __init__(self):
+        self.conversation = None
+        self.decoded_ids = None
+
+    def apply_chat_template(self, conversation, **_kwargs):
+        self.conversation = conversation
+        return [[1, 2, 3]]
+
+    def batch_decode(self, generated_ids, **_kwargs):
+        self.decoded_ids = generated_ids
+        return ["  Cat.  "]
+
+
+@pytest.fixture()
+def nanovlm_module(monkeypatch):
+    model = FakeModel()
+    tokenizer = FakeTokenizer()
+    image_processor_calls = []
+    image_string_calls = []
+
+    vision_language_model = types.ModuleType("models.vision_language_model")
+
+    class VisionLanguageModel:
+        @staticmethod
+        def from_pretrained(_model_path):
+            return model
+
+    vision_language_model.VisionLanguageModel = VisionLanguageModel
+
+    processors = types.ModuleType("data.processors")
+    processors.get_tokenizer = lambda *_args: tokenizer
+
+    def get_image_processor(*args):
+        image_processor_calls.append(args)
+
+        def process(_image):
+            return torch.zeros((1, 3, 2, 2)), (1, 1)
+
+        return process
+
+    def get_image_string(used_tokenizer, ratios, token_length):
+        image_string_calls.append((used_tokenizer, ratios, token_length))
+        return "<image-tokens>" if ratios else ""
+
+    processors.get_image_processor = get_image_processor
+    processors.get_image_string = get_image_string
+
+    models = types.ModuleType("models")
+    models.__path__ = []
+    data = types.ModuleType("data")
+    data.__path__ = []
+    monkeypatch.setitem(sys.modules, "models", models)
+    monkeypatch.setitem(
+        sys.modules, "models.vision_language_model", vision_language_model
+    )
+    monkeypatch.setitem(sys.modules, "data", data)
+    monkeypatch.setitem(sys.modules, "data.processors", processors)
+
+    scieval = types.ModuleType("scieval")
+    scieval.__path__ = []
+    vlm = types.ModuleType("scieval.vlm")
+    vlm.__path__ = []
+    base = types.ModuleType("scieval.vlm.base")
+
+    class BaseModel:
+        def __init__(self):
+            self.dump_image_func = None
+
+    base.BaseModel = BaseModel
+    monkeypatch.setitem(sys.modules, "scieval", scieval)
+    monkeypatch.setitem(sys.modules, "scieval.vlm", vlm)
+    monkeypatch.setitem(sys.modules, "scieval.vlm.base", base)
+
+    module_path = Path(__file__).parents[1] / "scieval" / "vlm" / "nanovlm.py"
+    spec = importlib.util.spec_from_file_location("scieval.vlm.nanovlm", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module, model, tokenizer, image_processor_calls, image_string_calls
+
+
+def test_generate_prepares_image_prompt_and_decodes_output(
+    nanovlm_module, tmp_path
+):
+    module, model, tokenizer, processor_calls, image_string_calls = nanovlm_module
+    image_path = tmp_path / "cat.png"
+    Image.new("RGB", (2, 2), "orange").save(image_path)
+
+    adapter = module.NanoVLM(device="cpu", max_new_tokens=8)
+    answer = adapter.generate_inner(
+        [
+            {"type": "image", "value": str(image_path)},
+            {"type": "text", "value": " What animal is shown? "},
+        ]
+    )
+
+    assert model.to_device == torch.device("cpu")
+    assert model.eval_called
+    assert processor_calls == [(2048, 512, True)]
+    assert image_string_calls == [(tokenizer, [(1, 1)], 64)]
+    assert tokenizer.conversation[0][0]["content"] == (
+        "<image-tokens>What animal is shown?"
+    )
+    assert model.input_ids.shape == (1, 3)
+    assert len(model.images) == 1
+    assert model.images[0].device.type == "cpu"
+    assert model.generation_kwargs == {"max_new_tokens": 8, "greedy": True}
+    assert torch.equal(tokenizer.decoded_ids, torch.tensor([[7, 8]]))
+    assert answer == "Cat."
+
+
+def test_text_only_generation_passes_no_images(nanovlm_module):
+    module, model, tokenizer, _processor_calls, image_string_calls = nanovlm_module
+    adapter = module.NanoVLM(device="cpu")
+
+    adapter.generate_inner([{"type": "text", "value": "hello"}])
+
+    assert image_string_calls == [(tokenizer, [], 64)]
+    assert model.images is None
+    assert tokenizer.conversation[0][0]["content"] == "hello"