From 7549d8db16000dfe52061274d511550ca2273556 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Sat, 29 Aug 2026 06:45:17 +0200 Subject: [PATCH 1/4] feat: refine screenshot catalogs through SubLLM vision Interactive element detection uses autogrammar-imgl/vision instead of a private LiteLLM model slug. Text agent planning is unchanged. Co-authored-by: Koru Agent --- imgl/cli.py | 2 +- imgl/llm_catalog.py | 29 ++++++++++++++++------------- pyproject.toml | 2 +- tests/test_llm_catalog.py | 23 +++++++++++++++++++++++ 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/imgl/cli.py b/imgl/cli.py index 5584e47..329adf7 100644 --- a/imgl/cli.py +++ b/imgl/cli.py @@ -331,7 +331,7 @@ def build_parser() -> argparse.ArgumentParser: interact_parser.add_argument( "--llm", action="store_true", - help="Use vision LLM catalog (requires OPENROUTER_API_KEY, pip install litellm)", + help="Use vision LLM catalog (requires OPENROUTER_API_KEY, pip install -e '.[llm]')", ) interact_parser.add_argument( "--no-filter", diff --git a/imgl/llm_catalog.py b/imgl/llm_catalog.py index d402dbd..c9a616d 100644 --- a/imgl/llm_catalog.py +++ b/imgl/llm_catalog.py @@ -87,12 +87,18 @@ def llm_available() -> bool: def llm_dependencies_ok() -> tuple[bool, str | None]: try: - import litellm # type: ignore # noqa: F401 + _subllm_complete() except ImportError: - return False, "litellm not installed (pip install -e '.[llm]')" + return False, "subactor-subllm not installed (pip install -e '.[llm]')" return True, None +def _subllm_complete(): + from subllm import complete as subllm_complete + + return subllm_complete + + def refine_catalog_with_llm( scene: Scene, *, @@ -219,12 +225,7 @@ def _call_vision_llm( crop_bbox: BBox | None = None, window_title: str | None = None, ) -> dict[str, Any]: - os.environ.setdefault("LITELLM_LOG", "ERROR") - import litellm # type: ignore - - litellm.set_verbose = False - if hasattr(litellm, "suppress_debug_info"): - litellm.suppress_debug_info = True + subllm_complete = _subllm_complete() image_b64 = _image_to_base64(image_path, crop_bbox=crop_bbox) scope_hint = ( f"This image shows one application window{f' ({window_title})' if window_title else ''}. " @@ -251,13 +252,15 @@ def _call_vision_llm( ], }, ] - response = litellm.completion( - model=model, - messages=messages, - temperature=0.1, + api_key = os.getenv("OPENROUTER_API_KEY", "").strip() + response = subllm_complete( + "autogrammar-imgl", + "vision", + messages, response_format={"type": "json_object"}, + credentials={"openrouter": api_key} if api_key else None, ) - content = (response.choices[0].message.content or "").strip() + content = (response.content or "").strip() return _parse_json_payload(content) diff --git a/pyproject.toml b/pyproject.toml index 8cb587b..89c2391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,8 @@ diagnose = [ "numpy>=1.24", ] llm = [ - "litellm>=1.30", "python-dotenv>=1.0", + "subactor-subllm @ git+https://github.com/subactor/subllm.git@067732fcd2fab3b36453160bb4497012b13bc298", ] web = [ "fastapi>=0.110", diff --git a/tests/test_llm_catalog.py b/tests/test_llm_catalog.py index 823d383..d40bf8f 100644 --- a/tests/test_llm_catalog.py +++ b/tests/test_llm_catalog.py @@ -124,3 +124,26 @@ def test_merge_heuristic_inputs_appends_missing_fields(): for opt in inputs for term in ("Chat", "Terminal", "Pole", "Editor", "Input") ) + + +def test_call_vision_llm_uses_central_subllm(tmp_path: Path, monkeypatch): + from imgl import llm_catalog + + captured: dict = {} + from PIL import Image + + image = tmp_path / "shot.png" + Image.new("RGB", (8, 8), color=(255, 255, 255)).save(image) + + def fake_complete(application, function, messages, **kwargs): + captured.update(application=application, function=function, messages=messages, kwargs=kwargs) + return type("Response", (), {"content": '{"elements":[]}'})() + + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test") + monkeypatch.setattr(llm_catalog, "_subllm_complete", lambda: fake_complete) + payload = llm_catalog._call_vision_llm(str(image), model="ignored", max_elements=5) + assert payload == {"elements": []} + assert captured["application"] == "autogrammar-imgl" + assert captured["function"] == "vision" + assert captured["messages"][1]["content"][0]["type"] == "image_url" + assert captured["kwargs"]["credentials"] == {"openrouter": "sk-test"} From 831afebca47a6493c1066964c819fee42b2a146c Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 31 Aug 2026 20:22:02 +0200 Subject: [PATCH 2/4] fix: gate SubLLM vision on supported Python Co-authored-by: Koru Agent --- imgl/llm_catalog.py | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/imgl/llm_catalog.py b/imgl/llm_catalog.py index c9a616d..3b783ae 100644 --- a/imgl/llm_catalog.py +++ b/imgl/llm_catalog.py @@ -6,6 +6,7 @@ import json import os import re +import sys from io import BytesIO from pathlib import Path from typing import Any @@ -86,6 +87,8 @@ def llm_available() -> bool: def llm_dependencies_ok() -> tuple[bool, str | None]: + if sys.version_info < (3, 11): + return False, "SubLLM vision requires Python 3.11+" try: _subllm_complete() except ImportError: diff --git a/pyproject.toml b/pyproject.toml index 89c2391..7c813aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ diagnose = [ ] llm = [ "python-dotenv>=1.0", - "subactor-subllm @ git+https://github.com/subactor/subllm.git@067732fcd2fab3b36453160bb4497012b13bc298", + "subactor-subllm @ git+https://github.com/subactor/subllm.git@067732fcd2fab3b36453160bb4497012b13bc298 ; python_version >= '3.11'", ] web = [ "fastapi>=0.110", From 126d39034bcc02d2b7f0326a78f071edf8839c1c Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 31 Aug 2026 21:24:13 +0200 Subject: [PATCH 3/4] ci: gate pull requests with full tests Co-authored-by: Koru Agent --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..eee38d1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Install test subjects + run: | + python -m pip install --upgrade pip + python -m pip install -e . + python -m pip install -e packages/mcp2imgl --no-deps + - name: Run tests + run: python -m pytest tests -q From 51eb3a7e5d77d840b500180875015fc70fb78d6d Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 31 Aug 2026 21:33:33 +0200 Subject: [PATCH 4/4] fix(ci): install pytest before running checks Co-authored-by: Koru Agent --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eee38d1..e05d833 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e . + python -m pip install 'pytest>=8.0' python -m pip install -e packages/mcp2imgl --no-deps - name: Run tests run: python -m pytest tests -q