From 226e63be85a40eadacf9ab43694a4d9a39df9316 Mon Sep 17 00:00:00 2001 From: genitrix Date: Mon, 17 Aug 2026 08:24:27 +0800 Subject: [PATCH] feat: add nanoVLM model support --- scieval/config.py | 8 +- scieval/vlm/__init__.py | 1 + scieval/vlm/nanovlm.py | 128 +++++++++++++++++++++++++++++++ tests/test_nanovlm.py | 166 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 scieval/vlm/nanovlm.py create mode 100644 tests/test_nanovlm.py diff --git a/scieval/config.py b/scieval/config.py index b8931ff..f69e97b 100644 --- a/scieval/config.py +++ b/scieval/config.py @@ -1738,6 +1738,11 @@ "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'), } +nanovlm_series = { + "nanoVLM-230M-8k": partial(NanoVLM, model_path="lusxvr/nanoVLM-230M-8k"), + "nanoVLM-460M-8k": partial(NanoVLM, model_path="lusxvr/nanoVLM-460M-8k"), +} + internvl_groups = [ @@ -1762,7 +1767,8 @@ aria_series, smolvlm_series, sail_series, valley_series, vita_series, ross_series, emu_series, ola_series, ursa_series, gemma_series, long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, - flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series, + flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, + thyme_series, logics_series, nanovlm_series, ] for grp in model_groups: diff --git a/scieval/vlm/__init__.py b/scieval/vlm/__init__.py index 23002cf..5eb224b 100644 --- a/scieval/vlm/__init__.py +++ b/scieval/vlm/__init__.py @@ -110,3 +110,4 @@ QTuneVLChat, ) from .logics import Logics_Thinking +from .nanovlm import NanoVLM diff --git a/scieval/vlm/nanovlm.py b/scieval/vlm/nanovlm.py new file mode 100644 index 0000000..cf309d0 --- /dev/null +++ b/scieval/vlm/nanovlm.py @@ -0,0 +1,128 @@ +import os +import sys +import warnings + +import torch +from PIL import Image + +from .base import BaseModel + + +_NANOVLM_INSTALL_MESSAGE = ( + "nanoVLM is not distributed as a Python package. Clone " + "https://github.com/huggingface/nanoVLM and set NANOVLM_PATH to the " + "checkout directory before running SciEvalKit." +) + + +def _ensure_nanovlm_importable(): + nanovlm_path = os.environ.get("NANOVLM_PATH", "") + if nanovlm_path and nanovlm_path not in sys.path: + sys.path.insert(0, nanovlm_path) + + +class NanoVLM(BaseModel): + """Adapter for the pure-PyTorch Hugging Face nanoVLM implementation.""" + + INSTALL_REQ = True + INTERLEAVE = True + + def __init__( + self, + model_path="lusxvr/nanoVLM-230M-8k", + device=None, + **kwargs, + ): + super().__init__() + _ensure_nanovlm_importable() + try: + from data.processors import get_image_processor, get_tokenizer + from models.vision_language_model import VisionLanguageModel + except ImportError as exc: + raise ImportError(_NANOVLM_INSTALL_MESSAGE) from exc + + self.device = torch.device(device or self._default_device()) + self.model = VisionLanguageModel.from_pretrained(model_path) + self.model = self.model.to(self.device).eval() + self.config = self.model.cfg + + self.tokenizer = get_tokenizer( + self.config.lm_tokenizer, + self.config.vlm_extra_tokens, + self.config.lm_chat_template, + ) + self.image_processor = get_image_processor( + self.config.max_img_size, + self.config.vit_img_size, + getattr(self.config, "resize_to_max_side_len", False), + ) + + generation_kwargs = {"max_new_tokens": 2048, "greedy": True} + generation_kwargs.update(kwargs) + self.kwargs = generation_kwargs + warnings.warn(f"NanoVLM kwargs: {self.kwargs}") + + @staticmethod + def _default_device(): + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + @staticmethod + def _open_image(path): + with Image.open(path) as image: + return image.convert("RGB") + + def _prepare_images(self, message): + processed_images = [] + image_ratios = [] + for item in message: + if item["type"] != "image": + continue + image = self._open_image(item["value"]) + processed, ratio = self.image_processor(image) + if ( + not hasattr(self.tokenizer, "global_image_token") + and ratio[0] * ratio[1] == len(processed) - 1 + ): + processed = processed[1:] + processed_images.append(processed.to(self.device)) + image_ratios.append(ratio) + return processed_images, image_ratios + + @staticmethod + def _message_text(message): + return "\n".join( + item["value"].strip() for item in message if item["type"] == "text" + ) + + def generate_inner(self, message, dataset=None): + _ensure_nanovlm_importable() + try: + from data.processors import get_image_string + except ImportError as exc: + raise ImportError(_NANOVLM_INSTALL_MESSAGE) from exc + + images, image_ratios = self._prepare_images(message) + image_string = get_image_string( + self.tokenizer, image_ratios, self.config.mp_image_token_length + ) + prompt = image_string + self._message_text(message) + conversation = [{"role": "user", "content": prompt}] + encoded_prompt = self.tokenizer.apply_chat_template( + [conversation], tokenize=True, add_generation_prompt=True + ) + input_ids = torch.as_tensor(encoded_prompt, device=self.device) + if input_ids.ndim == 1: + input_ids = input_ids.unsqueeze(0) + + generated_ids = self.model.generate( + input_ids, + images or None, + **self.kwargs, + ) + return self.tokenizer.batch_decode( + generated_ids, skip_special_tokens=True + )[0].strip() diff --git a/tests/test_nanovlm.py b/tests/test_nanovlm.py new file mode 100644 index 0000000..3bb2610 --- /dev/null +++ b/tests/test_nanovlm.py @@ -0,0 +1,166 @@ +import importlib.util +import sys +import types +from pathlib import Path + +import pytest +import torch +from PIL import Image + + +class FakeConfig: + lm_tokenizer = "fake-tokenizer" + vlm_extra_tokens = {"image_token": ""} + lm_chat_template = "fake-template" + max_img_size = 2048 + vit_img_size = 512 + resize_to_max_side_len = True + mp_image_token_length = 64 + + +class FakeModel: + def __init__(self): + self.cfg = FakeConfig() + self.to_device = None + self.eval_called = False + self.input_ids = None + self.images = None + self.generation_kwargs = None + + def to(self, device): + self.to_device = device + return self + + def eval(self): + self.eval_called = True + return self + + def generate(self, input_ids, images, **kwargs): + self.input_ids = input_ids + self.images = images + self.generation_kwargs = kwargs + return torch.tensor([[7, 8]]) + + +class FakeTokenizer: + global_image_token = "" + + def __init__(self): + self.conversation = None + self.decoded_ids = None + + def apply_chat_template(self, conversation, **_kwargs): + self.conversation = conversation + return [[1, 2, 3]] + + def batch_decode(self, generated_ids, **_kwargs): + self.decoded_ids = generated_ids + return [" Cat. "] + + +@pytest.fixture() +def nanovlm_module(monkeypatch): + model = FakeModel() + tokenizer = FakeTokenizer() + image_processor_calls = [] + image_string_calls = [] + + vision_language_model = types.ModuleType("models.vision_language_model") + + class VisionLanguageModel: + @staticmethod + def from_pretrained(_model_path): + return model + + vision_language_model.VisionLanguageModel = VisionLanguageModel + + processors = types.ModuleType("data.processors") + processors.get_tokenizer = lambda *_args: tokenizer + + def get_image_processor(*args): + image_processor_calls.append(args) + + def process(_image): + return torch.zeros((1, 3, 2, 2)), (1, 1) + + return process + + def get_image_string(used_tokenizer, ratios, token_length): + image_string_calls.append((used_tokenizer, ratios, token_length)) + return "" if ratios else "" + + processors.get_image_processor = get_image_processor + processors.get_image_string = get_image_string + + models = types.ModuleType("models") + models.__path__ = [] + data = types.ModuleType("data") + data.__path__ = [] + monkeypatch.setitem(sys.modules, "models", models) + monkeypatch.setitem( + sys.modules, "models.vision_language_model", vision_language_model + ) + monkeypatch.setitem(sys.modules, "data", data) + monkeypatch.setitem(sys.modules, "data.processors", processors) + + scieval = types.ModuleType("scieval") + scieval.__path__ = [] + vlm = types.ModuleType("scieval.vlm") + vlm.__path__ = [] + base = types.ModuleType("scieval.vlm.base") + + class BaseModel: + def __init__(self): + self.dump_image_func = None + + base.BaseModel = BaseModel + monkeypatch.setitem(sys.modules, "scieval", scieval) + monkeypatch.setitem(sys.modules, "scieval.vlm", vlm) + monkeypatch.setitem(sys.modules, "scieval.vlm.base", base) + + module_path = Path(__file__).parents[1] / "scieval" / "vlm" / "nanovlm.py" + spec = importlib.util.spec_from_file_location("scieval.vlm.nanovlm", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, model, tokenizer, image_processor_calls, image_string_calls + + +def test_generate_prepares_image_prompt_and_decodes_output( + nanovlm_module, tmp_path +): + module, model, tokenizer, processor_calls, image_string_calls = nanovlm_module + image_path = tmp_path / "cat.png" + Image.new("RGB", (2, 2), "orange").save(image_path) + + adapter = module.NanoVLM(device="cpu", max_new_tokens=8) + answer = adapter.generate_inner( + [ + {"type": "image", "value": str(image_path)}, + {"type": "text", "value": " What animal is shown? "}, + ] + ) + + assert model.to_device == torch.device("cpu") + assert model.eval_called + assert processor_calls == [(2048, 512, True)] + assert image_string_calls == [(tokenizer, [(1, 1)], 64)] + assert tokenizer.conversation[0][0]["content"] == ( + "What animal is shown?" + ) + assert model.input_ids.shape == (1, 3) + assert len(model.images) == 1 + assert model.images[0].device.type == "cpu" + assert model.generation_kwargs == {"max_new_tokens": 8, "greedy": True} + assert torch.equal(tokenizer.decoded_ids, torch.tensor([[7, 8]])) + assert answer == "Cat." + + +def test_text_only_generation_passes_no_images(nanovlm_module): + module, model, tokenizer, _processor_calls, image_string_calls = nanovlm_module + adapter = module.NanoVLM(device="cpu") + + adapter.generate_inner([{"type": "text", "value": "hello"}]) + + assert image_string_calls == [(tokenizer, [], 64)] + assert model.images is None + assert tokenizer.conversation[0][0]["content"] == "hello"