From eda8ab43c3430bf8db5fdf1a1bb53712b534164b Mon Sep 17 00:00:00 2001 From: genitrix Date: Mon, 17 Aug 2026 08:32:32 +0800 Subject: [PATCH] feat: add Granite Docling model support --- scieval/config.py | 9 +- scieval/vlm/__init__.py | 1 + scieval/vlm/granite_docling.py | 97 ++++++++++++++++++++ tests/test_granite_docling.py | 160 +++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 scieval/vlm/granite_docling.py create mode 100644 tests/test_granite_docling.py diff --git a/scieval/config.py b/scieval/config.py index b8931ff..23452ba 100644 --- a/scieval/config.py +++ b/scieval/config.py @@ -1738,6 +1738,12 @@ "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'), } +granite_docling_series = { + "granite-docling-258M": partial( + GraniteDocling, model_path="ibm-granite/granite-docling-258M" + ), +} + internvl_groups = [ @@ -1762,7 +1768,8 @@ aria_series, smolvlm_series, sail_series, valley_series, vita_series, ross_series, emu_series, ola_series, ursa_series, gemma_series, long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, - flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series, + flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, + thyme_series, logics_series, granite_docling_series, ] for grp in model_groups: diff --git a/scieval/vlm/__init__.py b/scieval/vlm/__init__.py index 23002cf..398a8e9 100644 --- a/scieval/vlm/__init__.py +++ b/scieval/vlm/__init__.py @@ -110,3 +110,4 @@ QTuneVLChat, ) from .logics import Logics_Thinking +from .granite_docling import GraniteDocling diff --git a/scieval/vlm/granite_docling.py b/scieval/vlm/granite_docling.py new file mode 100644 index 0000000..d260e1c --- /dev/null +++ b/scieval/vlm/granite_docling.py @@ -0,0 +1,97 @@ +import warnings + +import torch +from PIL import Image + +from .base import BaseModel + + +class GraniteDocling(BaseModel): + """Transformers adapter for IBM Granite Docling document conversion.""" + + INSTALL_REQ = True + INTERLEAVE = True + + def __init__( + self, + model_path="ibm-granite/granite-docling-258M", + device=None, + model_kwargs=None, + skip_special_tokens=False, + **kwargs, + ): + super().__init__() + try: + from transformers import AutoModelForImageTextToText, AutoProcessor + except ImportError as exc: + raise ImportError( + "Granite Docling requires a recent Transformers release." + ) from exc + + self.device = device or self._default_device() + self.skip_special_tokens = skip_special_tokens + + load_kwargs = dict(model_kwargs or {}) + load_kwargs.setdefault( + "dtype", torch.bfloat16 if self.device == "cuda" else torch.float32 + ) + load_kwargs.setdefault("_attn_implementation", "sdpa") + + self.processor = AutoProcessor.from_pretrained(model_path) + self.model = AutoModelForImageTextToText.from_pretrained( + model_path, **load_kwargs + ) + if "device_map" not in load_kwargs: + self.model = self.model.to(self.device) + self.model = self.model.eval() + + generation_kwargs = {"max_new_tokens": 1024, "use_cache": False} + generation_kwargs.update(kwargs) + self.kwargs = generation_kwargs + warnings.warn( + f"Following kwargs received: {self.kwargs}, will use as generation config." + ) + + @staticmethod + def _default_device(): + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + @staticmethod + def _open_image(path): + with Image.open(path) as image: + return image.convert("RGB") + + def message_to_chat_messages(self, message): + content = [] + images = [] + for item in message: + if item["type"] == "image": + images.append(self._open_image(item["value"])) + content.append({"type": "image"}) + elif item["type"] == "text": + content.append({"type": "text", "text": item["value"].strip()}) + return [{"role": "user", "content": content}], images + + def generate_inner(self, message, dataset=None): + conversation, images = self.message_to_chat_messages(message) + prompt = self.processor.apply_chat_template( + conversation, add_generation_prompt=True + ) + inputs = self.processor( + text=prompt, + images=images or None, + return_tensors="pt", + ).to(self.model.device) + input_length = inputs["input_ids"].shape[-1] + + with torch.inference_mode(): + generated_ids = self.model.generate(**inputs, **self.kwargs) + + return self.processor.batch_decode( + generated_ids[:, input_length:], + skip_special_tokens=self.skip_special_tokens, + )[0].strip() diff --git a/tests/test_granite_docling.py b/tests/test_granite_docling.py new file mode 100644 index 0000000..c2b7864 --- /dev/null +++ b/tests/test_granite_docling.py @@ -0,0 +1,160 @@ +import importlib.util +import sys +import types +from pathlib import Path + +import pytest +import torch +from PIL import Image + + +class FakeBatch(dict): + def __init__(self): + super().__init__(input_ids=torch.tensor([[1, 2, 3]])) + self.to_device = None + + def to(self, device): + self.to_device = device + return self + + +class FakeProcessor: + def __init__(self): + self.conversation = None + self.prompt = None + self.images = None + self.decoded_ids = None + self.skip_special_tokens = None + + def apply_chat_template(self, conversation, **_kwargs): + self.conversation = conversation + return "formatted prompt" + + def __call__(self, text, images, return_tensors): + self.prompt = text + self.images = images + assert return_tensors == "pt" + return FakeBatch() + + def batch_decode(self, generated_ids, skip_special_tokens): + self.decoded_ids = generated_ids + self.skip_special_tokens = skip_special_tokens + return [" Science "] + + +class FakeModel: + def __init__(self): + self.device = torch.device("cpu") + self.to_device = None + self.eval_called = False + self.generation_kwargs = None + + def to(self, device): + self.to_device = device + self.device = torch.device(device) + return self + + def eval(self): + self.eval_called = True + return self + + def generate(self, **kwargs): + self.generation_kwargs = kwargs + return torch.tensor([[1, 2, 3, 8, 9]]) + + +@pytest.fixture() +def granite_docling_module(monkeypatch): + processor = FakeProcessor() + model = FakeModel() + model_load_calls = [] + + processor_auto = types.SimpleNamespace( + from_pretrained=lambda _model_path: processor + ) + + def load_model(model_path, **kwargs): + model_load_calls.append((model_path, kwargs)) + return model + + model_auto = types.SimpleNamespace(from_pretrained=load_model) + transformers = types.ModuleType("transformers") + transformers.AutoProcessor = processor_auto + transformers.AutoModelForImageTextToText = model_auto + monkeypatch.setitem(sys.modules, "transformers", transformers) + + scieval = types.ModuleType("scieval") + scieval.__path__ = [] + vlm = types.ModuleType("scieval.vlm") + vlm.__path__ = [] + base = types.ModuleType("scieval.vlm.base") + + class BaseModel: + def __init__(self): + self.dump_image_func = None + + base.BaseModel = BaseModel + monkeypatch.setitem(sys.modules, "scieval", scieval) + monkeypatch.setitem(sys.modules, "scieval.vlm", vlm) + monkeypatch.setitem(sys.modules, "scieval.vlm.base", base) + + module_path = ( + Path(__file__).parents[1] / "scieval" / "vlm" / "granite_docling.py" + ) + spec = importlib.util.spec_from_file_location( + "scieval.vlm.granite_docling", module_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, processor, model, model_load_calls + + +def test_generate_preserves_multimodal_order_and_decodes_new_tokens( + granite_docling_module, tmp_path +): + module, processor, model, model_load_calls = granite_docling_module + image_path = tmp_path / "page.png" + Image.new("RGB", (2, 2), "white").save(image_path) + + adapter = module.GraniteDocling(device="cpu", max_new_tokens=12) + result = adapter.generate_inner( + [ + {"type": "text", "value": " Convert this page. "}, + {"type": "image", "value": str(image_path)}, + ] + ) + + assert model_load_calls == [ + ( + "ibm-granite/granite-docling-258M", + {"dtype": torch.float32, "_attn_implementation": "sdpa"}, + ) + ] + assert model.to_device == "cpu" + assert model.eval_called + assert processor.conversation == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Convert this page."}, + {"type": "image"}, + ], + } + ] + assert processor.prompt == "formatted prompt" + assert len(processor.images) == 1 + assert isinstance(processor.images[0], Image.Image) + assert model.generation_kwargs["max_new_tokens"] == 12 + assert torch.equal(processor.decoded_ids, torch.tensor([[8, 9]])) + assert processor.skip_special_tokens is False + assert result == "Science" + + +def test_device_map_skips_explicit_model_move(granite_docling_module): + module, _processor, model, _model_load_calls = granite_docling_module + adapter = module.GraniteDocling( + device="cpu", model_kwargs={"device_map": "auto", "dtype": "auto"} + ) + + assert adapter.device == "cpu" + assert model.to_device is None