feat: add Granite Docling model support

scieval/config.py

@@ -1738,6 +1738,12 @@
     "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'),
 }
 
+granite_docling_series = {
+    "granite-docling-258M": partial(
+        GraniteDocling, model_path="ibm-granite/granite-docling-258M"
+    ),
+}
+
 
 
 internvl_groups = [
@@ -1762,7 +1768,8 @@
     aria_series, smolvlm_series, sail_series, valley_series, vita_series,
     ross_series, emu_series, ola_series, ursa_series, gemma_series,
     long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, 
-    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series,
+    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series,
+    thyme_series, logics_series, granite_docling_series,
 ]
 
 for grp in model_groups:

scieval/vlm/__init__.py

@@ -110,3 +110,4 @@
     QTuneVLChat,
 )
 from .logics import Logics_Thinking
+from .granite_docling import GraniteDocling

scieval/vlm/granite_docling.py

@@ -0,0 +1,97 @@
+import warnings
+
+import torch
+from PIL import Image
+
+from .base import BaseModel
+
+
+class GraniteDocling(BaseModel):
+    """Transformers adapter for IBM Granite Docling document conversion."""
+
+    INSTALL_REQ = True
+    INTERLEAVE = True
+
+    def __init__(
+        self,
+        model_path="ibm-granite/granite-docling-258M",
+        device=None,
+        model_kwargs=None,
+        skip_special_tokens=False,
+        **kwargs,
+    ):
+        super().__init__()
+        try:
+            from transformers import AutoModelForImageTextToText, AutoProcessor
+        except ImportError as exc:
+            raise ImportError(
+                "Granite Docling requires a recent Transformers release."
+            ) from exc
+
+        self.device = device or self._default_device()
+        self.skip_special_tokens = skip_special_tokens
+
+        load_kwargs = dict(model_kwargs or {})
+        load_kwargs.setdefault(
+            "dtype", torch.bfloat16 if self.device == "cuda" else torch.float32
+        )
+        load_kwargs.setdefault("_attn_implementation", "sdpa")
+
+        self.processor = AutoProcessor.from_pretrained(model_path)
+        self.model = AutoModelForImageTextToText.from_pretrained(
+            model_path, **load_kwargs
+        )
+        if "device_map" not in load_kwargs:
+            self.model = self.model.to(self.device)
+        self.model = self.model.eval()
+
+        generation_kwargs = {"max_new_tokens": 1024, "use_cache": False}
+        generation_kwargs.update(kwargs)
+        self.kwargs = generation_kwargs
+        warnings.warn(
+            f"Following kwargs received: {self.kwargs}, will use as generation config."
+        )
+
+    @staticmethod
+    def _default_device():
+        if torch.cuda.is_available():
+            return "cuda"
+        if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            return "mps"
+        return "cpu"
+
+    @staticmethod
+    def _open_image(path):
+        with Image.open(path) as image:
+            return image.convert("RGB")
+
+    def message_to_chat_messages(self, message):
+        content = []
+        images = []
+        for item in message:
+            if item["type"] == "image":
+                images.append(self._open_image(item["value"]))
+                content.append({"type": "image"})
+            elif item["type"] == "text":
+                content.append({"type": "text", "text": item["value"].strip()})
+        return [{"role": "user", "content": content}], images
+
+    def generate_inner(self, message, dataset=None):
+        conversation, images = self.message_to_chat_messages(message)
+        prompt = self.processor.apply_chat_template(
+            conversation, add_generation_prompt=True
+        )
+        inputs = self.processor(
+            text=prompt,
+            images=images or None,
+            return_tensors="pt",
+        ).to(self.model.device)
+        input_length = inputs["input_ids"].shape[-1]
+
+        with torch.inference_mode():
+            generated_ids = self.model.generate(**inputs, **self.kwargs)
+
+        return self.processor.batch_decode(
+            generated_ids[:, input_length:],
+            skip_special_tokens=self.skip_special_tokens,
+        )[0].strip()

tests/test_granite_docling.py

@@ -0,0 +1,160 @@
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+from PIL import Image
+
+
+class FakeBatch(dict):
+    def __init__(self):
+        super().__init__(input_ids=torch.tensor([[1, 2, 3]]))
+        self.to_device = None
+
+    def to(self, device):
+        self.to_device = device
+        return self
+
+
+class FakeProcessor:
+    def __init__(self):
+        self.conversation = None
+        self.prompt = None
+        self.images = None
+        self.decoded_ids = None
+        self.skip_special_tokens = None
+
+    def apply_chat_template(self, conversation, **_kwargs):
+        self.conversation = conversation
+        return "formatted prompt"
+
+    def __call__(self, text, images, return_tensors):
+        self.prompt = text
+        self.images = images
+        assert return_tensors == "pt"
+        return FakeBatch()
+
+    def batch_decode(self, generated_ids, skip_special_tokens):
+        self.decoded_ids = generated_ids
+        self.skip_special_tokens = skip_special_tokens
+        return ["  <doctag><text>Science</text>  "]
+
+
+class FakeModel:
+    def __init__(self):
+        self.device = torch.device("cpu")
+        self.to_device = None
+        self.eval_called = False
+        self.generation_kwargs = None
+
+    def to(self, device):
+        self.to_device = device
+        self.device = torch.device(device)
+        return self
+
+    def eval(self):
+        self.eval_called = True
+        return self
+
+    def generate(self, **kwargs):
+        self.generation_kwargs = kwargs
+        return torch.tensor([[1, 2, 3, 8, 9]])
+
+
+@pytest.fixture()
+def granite_docling_module(monkeypatch):
+    processor = FakeProcessor()
+    model = FakeModel()
+    model_load_calls = []
+
+    processor_auto = types.SimpleNamespace(
+        from_pretrained=lambda _model_path: processor
+    )
+
+    def load_model(model_path, **kwargs):
+        model_load_calls.append((model_path, kwargs))
+        return model
+
+    model_auto = types.SimpleNamespace(from_pretrained=load_model)
+    transformers = types.ModuleType("transformers")
+    transformers.AutoProcessor = processor_auto
+    transformers.AutoModelForImageTextToText = model_auto
+    monkeypatch.setitem(sys.modules, "transformers", transformers)
+
+    scieval = types.ModuleType("scieval")
+    scieval.__path__ = []
+    vlm = types.ModuleType("scieval.vlm")
+    vlm.__path__ = []
+    base = types.ModuleType("scieval.vlm.base")
+
+    class BaseModel:
+        def __init__(self):
+            self.dump_image_func = None
+
+    base.BaseModel = BaseModel
+    monkeypatch.setitem(sys.modules, "scieval", scieval)
+    monkeypatch.setitem(sys.modules, "scieval.vlm", vlm)
+    monkeypatch.setitem(sys.modules, "scieval.vlm.base", base)
+
+    module_path = (
+        Path(__file__).parents[1] / "scieval" / "vlm" / "granite_docling.py"
+    )
+    spec = importlib.util.spec_from_file_location(
+        "scieval.vlm.granite_docling", module_path
+    )
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module, processor, model, model_load_calls
+
+
+def test_generate_preserves_multimodal_order_and_decodes_new_tokens(
+    granite_docling_module, tmp_path
+):
+    module, processor, model, model_load_calls = granite_docling_module
+    image_path = tmp_path / "page.png"
+    Image.new("RGB", (2, 2), "white").save(image_path)
+
+    adapter = module.GraniteDocling(device="cpu", max_new_tokens=12)
+    result = adapter.generate_inner(
+        [
+            {"type": "text", "value": " Convert this page. "},
+            {"type": "image", "value": str(image_path)},
+        ]
+    )
+
+    assert model_load_calls == [
+        (
+            "ibm-granite/granite-docling-258M",
+            {"dtype": torch.float32, "_attn_implementation": "sdpa"},
+        )
+    ]
+    assert model.to_device == "cpu"
+    assert model.eval_called
+    assert processor.conversation == [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Convert this page."},
+                {"type": "image"},
+            ],
+        }
+    ]
+    assert processor.prompt == "formatted prompt"
+    assert len(processor.images) == 1
+    assert isinstance(processor.images[0], Image.Image)
+    assert model.generation_kwargs["max_new_tokens"] == 12
+    assert torch.equal(processor.decoded_ids, torch.tensor([[8, 9]]))
+    assert processor.skip_special_tokens is False
+    assert result == "<doctag><text>Science</text>"
+
+
+def test_device_map_skips_explicit_model_move(granite_docling_module):
+    module, _processor, model, _model_load_calls = granite_docling_module
+    adapter = module.GraniteDocling(
+        device="cpu", model_kwargs={"device_map": "auto", "dtype": "auto"}
+    )
+
+    assert adapter.device == "cpu"
+    assert model.to_device is None