diff --git a/scieval/config.py b/scieval/config.py index b8931ff..053431b 100644 --- a/scieval/config.py +++ b/scieval/config.py @@ -1738,6 +1738,12 @@ "Logics-Thinking-32B": partial(Logics_Thinking,model_path='Logics-MLLM/Logics-Thinking-32B'), } +lfm2_vl_series = { + "LFM2-VL-450M": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-450M"), + "LFM2-VL-1.6B": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-1.6B"), + "LFM2-VL-3B": partial(LFM2VL, model_path="LiquidAI/LFM2-VL-3B"), +} + internvl_groups = [ @@ -1762,7 +1768,8 @@ aria_series, smolvlm_series, sail_series, valley_series, vita_series, ross_series, emu_series, ola_series, ursa_series, gemma_series, long_vita_series, ristretto_series, kimi_series, aguvis_series, hawkvl_series, - flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, thyme_series,logics_series, + flash_vl, kimi_vllm_series, oryx_series, treevgr_series, varco_vision_series, qtunevl_series, xvl_series, + thyme_series, logics_series, lfm2_vl_series, ] for grp in model_groups: diff --git a/scieval/vlm/__init__.py b/scieval/vlm/__init__.py index 23002cf..cba2df0 100644 --- a/scieval/vlm/__init__.py +++ b/scieval/vlm/__init__.py @@ -110,3 +110,4 @@ QTuneVLChat, ) from .logics import Logics_Thinking +from .liquid import LFM2VL diff --git a/scieval/vlm/liquid.py b/scieval/vlm/liquid.py new file mode 100644 index 0000000..c25780a --- /dev/null +++ b/scieval/vlm/liquid.py @@ -0,0 +1,121 @@ +import warnings + +import torch +from PIL import Image + +from .base import BaseModel + + +class LFM2VL(BaseModel): + """Hugging Face Transformers adapter for the LiquidAI LFM2-VL family.""" + + INSTALL_REQ = True + INTERLEAVE = True + + _NO_BRIEF_INSTRUCTION = {"MathVista_MINI", "MM-IFEval", "MMVet"} + _BRIEF_INSTRUCTION = ( + "\nPlease answer directly with only the final answer, " + "do not give any explanation." + ) + + def __init__( + self, + model_path="LiquidAI/LFM2-VL-450M", + device=None, + model_kwargs=None, + use_default_instruction=True, + **kwargs, + ): + super().__init__() + + try: + from transformers import AutoModelForImageTextToText, AutoProcessor + except ImportError as exc: + raise ImportError( + "LFM2-VL requires a recent Transformers release " + "(version 4.57 or newer)." + ) from exc + + self.device = device or self._default_device() + self.use_default_instruction = use_default_instruction + + load_kwargs = dict(model_kwargs or {}) + load_kwargs.setdefault( + "dtype", torch.bfloat16 if self.device == "cuda" else torch.float32 + ) + + self.processor = AutoProcessor.from_pretrained(model_path) + self.model = AutoModelForImageTextToText.from_pretrained( + model_path, **load_kwargs + ) + if "device_map" not in load_kwargs: + self.model = self.model.to(self.device) + self.model = self.model.eval() + + generation_kwargs = {"max_new_tokens": 1024, "use_cache": True} + generation_kwargs.update(kwargs) + self.kwargs = generation_kwargs + warnings.warn( + f"Following kwargs received: {self.kwargs}, will use as generation config." + ) + + @staticmethod + def _default_device(): + if torch.cuda.is_available(): + return "cuda" + if hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + return "mps" + return "cpu" + + def custom_instruction_prompt_by_dataset(self, dataset): + if not self.use_default_instruction or dataset in self._NO_BRIEF_INSTRUCTION: + return "" + return self._BRIEF_INSTRUCTION + + @staticmethod + def _load_image(path): + with Image.open(path) as image: + return image.convert("RGB") + + def message_to_chat_messages(self, message, instruction_prompt, dataset=None): + content = [] + for item in message: + if item["type"] == "image": + content.append( + {"type": "image", "image": self._load_image(item["value"])} + ) + elif item["type"] == "text": + content.append({"type": "text", "text": item["value"]}) + + if instruction_prompt: + content.append({"type": "text", "text": instruction_prompt}) + + if dataset == "MM-IFEval": + images = [item for item in content if item["type"] == "image"] + texts = [item for item in content if item["type"] != "image"] + content = images + texts + + return [{"role": "user", "content": content}] + + def generate_inner(self, message, dataset=None): + instruction = self.custom_instruction_prompt_by_dataset(dataset) + conversation = self.message_to_chat_messages(message, instruction, dataset) + inputs = self.processor.apply_chat_template( + conversation, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, + tokenize=True, + ).to(self.model.device) + input_length = inputs["input_ids"].shape[-1] + + with torch.inference_mode(): + outputs = self.model.generate(**inputs, **self.kwargs) + + generated_ids = outputs[:, input_length:] + return self.processor.batch_decode( + generated_ids, skip_special_tokens=True + )[0].strip() + + def chat_inner(self, message, dataset=None): + return self.generate_inner(message, dataset) diff --git a/tests/test_lfm2_vl.py b/tests/test_lfm2_vl.py new file mode 100644 index 0000000..e517d3f --- /dev/null +++ b/tests/test_lfm2_vl.py @@ -0,0 +1,157 @@ +import importlib.util +import sys +import types +from pathlib import Path + +import pytest +import torch +from PIL import Image + + +class FakeBatch(dict): + def __init__(self): + super().__init__(input_ids=torch.tensor([[1, 2, 3]])) + self.device = None + + def to(self, device): + self.device = device + return self + + +class FakeProcessor: + def __init__(self): + self.conversation = None + self.template_kwargs = None + self.decoded_ids = None + + def apply_chat_template(self, conversation, **kwargs): + self.conversation = conversation + self.template_kwargs = kwargs + return FakeBatch() + + def batch_decode(self, generated_ids, **kwargs): + self.decoded_ids = generated_ids + return [" final answer "] + + +class FakeModel: + def __init__(self): + self.device = torch.device("cpu") + self.to_device = None + self.eval_called = False + self.generation_kwargs = None + + def to(self, device): + self.to_device = device + self.device = torch.device(device) + return self + + def eval(self): + self.eval_called = True + return self + + def generate(self, **kwargs): + self.generation_kwargs = kwargs + return torch.tensor([[1, 2, 3, 8, 9]]) + + +@pytest.fixture() +def liquid_module(monkeypatch): + processor = FakeProcessor() + model = FakeModel() + processor_auto = types.SimpleNamespace( + from_pretrained=lambda _model_path: processor + ) + model_auto = types.SimpleNamespace( + from_pretrained=lambda _model_path, **_kwargs: model + ) + transformers = types.ModuleType("transformers") + transformers.AutoProcessor = processor_auto + transformers.AutoModelForImageTextToText = model_auto + monkeypatch.setitem(sys.modules, "transformers", transformers) + + scieval = types.ModuleType("scieval") + scieval.__path__ = [] + vlm = types.ModuleType("scieval.vlm") + vlm.__path__ = [] + base = types.ModuleType("scieval.vlm.base") + + class BaseModel: + def __init__(self): + self.dump_image_func = None + + base.BaseModel = BaseModel + monkeypatch.setitem(sys.modules, "scieval", scieval) + monkeypatch.setitem(sys.modules, "scieval.vlm", vlm) + monkeypatch.setitem(sys.modules, "scieval.vlm.base", base) + + module_path = Path(__file__).parents[1] / "scieval" / "vlm" / "liquid.py" + spec = importlib.util.spec_from_file_location("scieval.vlm.liquid", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, processor, model + + +def test_generate_uses_multimodal_chat_template_and_decodes_new_tokens( + liquid_module, tmp_path +): + module, processor, model = liquid_module + image_path = tmp_path / "sample.png" + Image.new("RGB", (2, 2), "red").save(image_path) + + adapter = module.LFM2VL(device="cpu", max_new_tokens=12) + result = adapter.generate_inner( + [ + {"type": "text", "value": "before"}, + {"type": "image", "value": str(image_path)}, + {"type": "text", "value": "after"}, + ] + ) + + content = processor.conversation[0]["content"] + assert [item["type"] for item in content] == ["text", "image", "text", "text"] + assert isinstance(content[1]["image"], Image.Image) + assert content[-1]["text"] == adapter._BRIEF_INSTRUCTION + assert processor.template_kwargs == { + "add_generation_prompt": True, + "return_tensors": "pt", + "return_dict": True, + "tokenize": True, + } + assert model.to_device == "cpu" + assert model.eval_called + assert model.generation_kwargs["max_new_tokens"] == 12 + assert torch.equal(processor.decoded_ids, torch.tensor([[8, 9]])) + assert result == "final answer" + + +def test_mm_ifeval_moves_images_first_without_adding_instruction( + liquid_module, tmp_path +): + module, processor, _model = liquid_module + image_path = tmp_path / "sample.png" + Image.new("RGB", (2, 2), "blue").save(image_path) + adapter = module.LFM2VL(device="cpu") + + adapter.generate_inner( + [ + {"type": "text", "value": "first"}, + {"type": "image", "value": str(image_path)}, + {"type": "text", "value": "last"}, + ], + dataset="MM-IFEval", + ) + + content = processor.conversation[0]["content"] + assert [item["type"] for item in content] == ["image", "text", "text"] + assert [item["text"] for item in content[1:]] == ["first", "last"] + + +def test_device_map_skips_explicit_model_move(liquid_module): + module, _processor, model = liquid_module + adapter = module.LFM2VL( + device="cpu", model_kwargs={"device_map": "auto", "dtype": "auto"} + ) + + assert adapter.device == "cpu" + assert model.to_device is None