feat: add LFM2-VL model support

scieval/config.py

@@ -1738,6 +1738,12 @@
     "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'),
 }
 
+lfm2_vl_series = {
+    "LFM2-VL-450M": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-450M"),
+    "LFM2-VL-1.6B": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-1.6B"),
+    "LFM2-VL-3B": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-3B"),
+}
+
 
 
 internvl_groups = [
@@ -1762,7 +1768,8 @@
     aria_series, smolvlm_series, sail_series, valley_series, vita_series,
     ross_series, emu_series, ola_series, ursa_series, gemma_series,
     long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, 
-    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series,
+    flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series,
+    thyme_series, logics_series, lfm2_vl_series,
 ]
 
 for grp in model_groups:

scieval/vlm/__init__.py

@@ -110,3 +110,4 @@
     QTuneVLChat,
 )
 from .logics import Logics_Thinking
+from .liquid import LFM2VL

scieval/vlm/liquid.py

@@ -0,0 +1,121 @@
+import warnings
+
+import torch
+from PIL import Image
+
+from .base import BaseModel
+
+
+class LFM2VL(BaseModel):
+    """Hugging Face Transformers adapter for the LiquidAI LFM2-VL family."""
+
+    INSTALL_REQ = True
+    INTERLEAVE = True
+
+    _NO_BRIEF_INSTRUCTION = {"MathVista_MINI", "MM-IFEval", "MMVet"}
+    _BRIEF_INSTRUCTION = (
+        "\nPlease answer directly with only the final answer, "
+        "do not give any explanation."
+    )
+
+    def __init__(
+        self,
+        model_path="LiquidAI/LFM2-VL-450M",
+        device=None,
+        model_kwargs=None,
+        use_default_instruction=True,
+        **kwargs,
+    ):
+        super().__init__()
+
+        try:
+            from transformers import AutoModelForImageTextToText, AutoProcessor
+        except ImportError as exc:
+            raise ImportError(
+                "LFM2-VL requires a recent Transformers release "
+                "(version 4.57 or newer)."
+            ) from exc
+
+        self.device = device or self._default_device()
+        self.use_default_instruction = use_default_instruction
+
+        load_kwargs = dict(model_kwargs or {})
+        load_kwargs.setdefault(
+            "dtype", torch.bfloat16 if self.device == "cuda" else torch.float32
+        )
+
+        self.processor = AutoProcessor.from_pretrained(model_path)
+        self.model = AutoModelForImageTextToText.from_pretrained(
+            model_path, **load_kwargs
+        )
+        if "device_map" not in load_kwargs:
+            self.model = self.model.to(self.device)
+        self.model = self.model.eval()
+
+        generation_kwargs = {"max_new_tokens": 1024, "use_cache": True}
+        generation_kwargs.update(kwargs)
+        self.kwargs = generation_kwargs
+        warnings.warn(
+            f"Following kwargs received: {self.kwargs}, will use as generation config."
+        )
+
+    @staticmethod
+    def _default_device():
+        if torch.cuda.is_available():
+            return "cuda"
+        if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+            return "mps"
+        return "cpu"
+
+    def custom_instruction_prompt_by_dataset(self, dataset):
+        if not self.use_default_instruction or dataset in self._NO_BRIEF_INSTRUCTION:
+            return ""
+        return self._BRIEF_INSTRUCTION
+
+    @staticmethod
+    def _load_image(path):
+        with Image.open(path) as image:
+            return image.convert("RGB")
+
+    def message_to_chat_messages(self, message, instruction_prompt, dataset=None):
+        content = []
+        for item in message:
+            if item["type"] == "image":
+                content.append(
+                    {"type": "image", "image": self._load_image(item["value"])}
+                )
+            elif item["type"] == "text":
+                content.append({"type": "text", "text": item["value"]})
+
+        if instruction_prompt:
+            content.append({"type": "text", "text": instruction_prompt})
+
+        if dataset == "MM-IFEval":
+            images = [item for item in content if item["type"] == "image"]
+            texts = [item for item in content if item["type"] != "image"]
+            content = images + texts
+
+        return [{"role": "user", "content": content}]
+
+    def generate_inner(self, message, dataset=None):
+        instruction = self.custom_instruction_prompt_by_dataset(dataset)
+        conversation = self.message_to_chat_messages(message, instruction, dataset)
+        inputs = self.processor.apply_chat_template(
+            conversation,
+            add_generation_prompt=True,
+            return_tensors="pt",
+            return_dict=True,
+            tokenize=True,
+        ).to(self.model.device)
+        input_length = inputs["input_ids"].shape[-1]
+
+        with torch.inference_mode():
+            outputs = self.model.generate(**inputs, **self.kwargs)
+
+        generated_ids = outputs[:, input_length:]
+        return self.processor.batch_decode(
+            generated_ids, skip_special_tokens=True
+        )[0].strip()
+
+    def chat_inner(self, message, dataset=None):
+        return self.generate_inner(message, dataset)

tests/test_lfm2_vl.py

@@ -0,0 +1,157 @@
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+import pytest
+import torch
+from PIL import Image
+
+
+class FakeBatch(dict):
+    def __init__(self):
+        super().__init__(input_ids=torch.tensor([[1, 2, 3]]))
+        self.device = None
+
+    def to(self, device):
+        self.device = device
+        return self
+
+
+class FakeProcessor:
+    def __init__(self):
+        self.conversation = None
+        self.template_kwargs = None
+        self.decoded_ids = None
+
+    def apply_chat_template(self, conversation, **kwargs):
+        self.conversation = conversation
+        self.template_kwargs = kwargs
+        return FakeBatch()
+
+    def batch_decode(self, generated_ids, **kwargs):
+        self.decoded_ids = generated_ids
+        return ["  final answer  "]
+
+
+class FakeModel:
+    def __init__(self):
+        self.device = torch.device("cpu")
+        self.to_device = None
+        self.eval_called = False
+        self.generation_kwargs = None
+
+    def to(self, device):
+        self.to_device = device
+        self.device = torch.device(device)
+        return self
+
+    def eval(self):
+        self.eval_called = True
+        return self
+
+    def generate(self, **kwargs):
+        self.generation_kwargs = kwargs
+        return torch.tensor([[1, 2, 3, 8, 9]])
+
+
+@pytest.fixture()
+def liquid_module(monkeypatch):
+    processor = FakeProcessor()
+    model = FakeModel()
+    processor_auto = types.SimpleNamespace(
+        from_pretrained=lambda _model_path: processor
+    )
+    model_auto = types.SimpleNamespace(
+        from_pretrained=lambda _model_path, **_kwargs: model
+    )
+    transformers = types.ModuleType("transformers")
+    transformers.AutoProcessor = processor_auto
+    transformers.AutoModelForImageTextToText = model_auto
+    monkeypatch.setitem(sys.modules, "transformers", transformers)
+
+    scieval = types.ModuleType("scieval")
+    scieval.__path__ = []
+    vlm = types.ModuleType("scieval.vlm")
+    vlm.__path__ = []
+    base = types.ModuleType("scieval.vlm.base")
+
+    class BaseModel:
+        def __init__(self):
+            self.dump_image_func = None
+
+    base.BaseModel = BaseModel
+    monkeypatch.setitem(sys.modules, "scieval", scieval)
+    monkeypatch.setitem(sys.modules, "scieval.vlm", vlm)
+    monkeypatch.setitem(sys.modules, "scieval.vlm.base", base)
+
+    module_path = Path(__file__).parents[1] / "scieval" / "vlm" / "liquid.py"
+    spec = importlib.util.spec_from_file_location("scieval.vlm.liquid", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module, processor, model
+
+
+def test_generate_uses_multimodal_chat_template_and_decodes_new_tokens(
+    liquid_module, tmp_path
+):
+    module, processor, model = liquid_module
+    image_path = tmp_path / "sample.png"
+    Image.new("RGB", (2, 2), "red").save(image_path)
+
+    adapter = module.LFM2VL(device="cpu", max_new_tokens=12)
+    result = adapter.generate_inner(
+        [
+            {"type": "text", "value": "before"},
+            {"type": "image", "value": str(image_path)},
+            {"type": "text", "value": "after"},
+        ]
+    )
+
+    content = processor.conversation[0]["content"]
+    assert [item["type"] for item in content] == ["text", "image", "text", "text"]
+    assert isinstance(content[1]["image"], Image.Image)
+    assert content[-1]["text"] == adapter._BRIEF_INSTRUCTION
+    assert processor.template_kwargs == {
+        "add_generation_prompt": True,
+        "return_tensors": "pt",
+        "return_dict": True,
+        "tokenize": True,
+    }
+    assert model.to_device == "cpu"
+    assert model.eval_called
+    assert model.generation_kwargs["max_new_tokens"] == 12
+    assert torch.equal(processor.decoded_ids, torch.tensor([[8, 9]]))
+    assert result == "final answer"
+
+
+def test_mm_ifeval_moves_images_first_without_adding_instruction(
+    liquid_module, tmp_path
+):
+    module, processor, _model = liquid_module
+    image_path = tmp_path / "sample.png"
+    Image.new("RGB", (2, 2), "blue").save(image_path)
+    adapter = module.LFM2VL(device="cpu")
+
+    adapter.generate_inner(
+        [
+            {"type": "text", "value": "first"},
+            {"type": "image", "value": str(image_path)},
+            {"type": "text", "value": "last"},
+        ],
+        dataset="MM-IFEval",
+    )
+
+    content = processor.conversation[0]["content"]
+    assert [item["type"] for item in content] == ["image", "text", "text"]
+    assert [item["text"] for item in content[1:]] == ["first", "last"]
+
+
+def test_device_map_skips_explicit_model_move(liquid_module):
+    module, _processor, model = liquid_module
+    adapter = module.LFM2VL(
+        device="cpu", model_kwargs={"device_map": "auto", "dtype": "auto"}
+    )
+
+    assert adapter.device == "cpu"
+    assert model.to_device is None