diff --git a/packages/img2vql/pyproject.toml b/packages/img2vql/pyproject.toml index 497a395..81222e0 100644 --- a/packages/img2vql/pyproject.toml +++ b/packages/img2vql/pyproject.toml @@ -19,6 +19,9 @@ dependencies = [ [project.optional-dependencies] opencv = ["opencv-python-headless>=4.8.0"] ocr = ["rapidocr-onnxruntime>=1.3.0"] +llm = [ + "subactor-subllm @ git+https://github.com/subactor/subllm.git@40726bb9399eb5cf9dfc038c1c948b7d2fbdca3b", +] [project.scripts] img2vql = "img2vql.cli:main" diff --git a/packages/img2vql/src/img2vql/pipeline/llm_client.py b/packages/img2vql/src/img2vql/pipeline/llm_client.py index 8b74185..ff74d65 100644 --- a/packages/img2vql/src/img2vql/pipeline/llm_client.py +++ b/packages/img2vql/src/img2vql/pipeline/llm_client.py @@ -7,12 +7,18 @@ import os import urllib.error import urllib.request +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any from img2vql.contracts import response_format from img2vql.pipeline.config import DEFAULT_VQL_VISION_MODEL, PipelineLLMConfig +try: + from subllm import complete as subllm_complete +except ImportError: + subllm_complete = None + class LLMClientError(RuntimeError): pass @@ -29,6 +35,17 @@ def _image_to_data_url(path: str | Path) -> str: return f"data:image/{mime};base64,{b64}" +def _messages_have_image(messages: Sequence[Mapping[str, Any]]) -> bool: + for message in messages: + content = message.get("content") + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, Mapping) and part.get("type") == "image_url": + return True + return False + + def chat_completion( config: PipelineLLMConfig, messages: list[dict[str, Any]], @@ -38,6 +55,27 @@ def chat_completion( "LLM not configured: set VQL_LLM_ENABLED=1 and OPENROUTER_API_KEY in .env" ) + if _messages_have_image(messages): + if subllm_complete is None: + raise LLMClientError("subactor-subllm is not installed") + try: + response = subllm_complete( + "autogrammar-vql", + "vision", + messages, + response_format=response_format(), + timeout_seconds=float(config.timeout_s), + credentials={"openrouter": config.api_key}, + ) + except Exception as exc: + raise LLMClientError(f"SubLLM vision request failed: {exc}") from exc + return { + "content": response.content, + "model": response.model, + "usage": dict(response.usage), + "raw": {}, + } + body = { "model": config.model, "messages": messages, diff --git a/packages/img2vql/tests/test_llm_contract.py b/packages/img2vql/tests/test_llm_contract.py index 46d0d67..491738e 100644 --- a/packages/img2vql/tests/test_llm_contract.py +++ b/packages/img2vql/tests/test_llm_contract.py @@ -116,6 +116,42 @@ def fake_urlopen(request, timeout): ) +def test_image_messages_use_central_subllm_vision(monkeypatch) -> None: + captured: dict = {} + + def fake_complete(application, function, messages, **kwargs): + captured.update(application=application, function=function, kwargs=kwargs) + return type("Response", (), {"content": '{"ok":true}', "model": "z-ai/glm-4.5v", "usage": {}})() + + monkeypatch.setattr("img2vql.pipeline.llm_client.subllm_complete", fake_complete) + config = PipelineLLMConfig( + enabled=True, + api_key="test-key", + model="ignored-by-policy", + base_url="https://openrouter.ai/api/v1", + vision=True, + temperature=0, + max_tokens=1000, + timeout_s=30, + ) + result = chat_completion( + config, + [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aaa"}}, + {"type": "text", "text": "extract"}, + ], + } + ], + ) + assert result["content"] == '{"ok":true}' + assert captured["application"] == "autogrammar-vql" + assert captured["function"] == "vision" + assert captured["kwargs"]["credentials"] == {"openrouter": "test-key"} + + def test_manifest_and_models_share_versions_and_artifacts() -> None: manifest = json.loads((CONTRACTS / "manifest.json").read_text(encoding="utf-8")) proto = (CONTRACTS / "vql-program.proto").read_text(encoding="utf-8")