From b403738f674f925d4c48c6373413dd5ae4594427 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:25:21 +0000 Subject: [PATCH 01/14] [P1] tree cache + input invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tree_cache.py: a thread-safe per-window_uid TTL+LRU cache of accessibility-tree captures (TTL from tree.cache_ttl_s, default 2.0s; LRU capped at 8 windows), attached to session.Session like tree_tokens. ScreenObserver.get_element_tree() gains optional window_uid/use_cache parameters (existing hwnd-only callers keep the uncached behavior); a fresh cached capture is returned without an adapter walk, a miss walks and stores. get_element_tree_with_meta() additionally reports {cache: hit|miss|bypass, capture_ms, node_count} for later perf telemetry. tools.dispatch() invalidates the target window's entry after any input tool (bring_to_foreground included) — the single choke point — and post-action re-reads (ActionReceipt after-state, wait_for/ wait_idle polling, select_option re-walk, trace after-hash) bypass the cache so they always observe fresh state. load_scenario invalidates the whole cache since it replaces the mock world. MockAdapter gains test hooks: capture_count and tree_mutator. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 3 +- config.json.example | 5 +- main.py | 2 +- observer.py | 93 +++++++++++++++++- session.py | 4 + tests/test_tree_cache.py | 207 +++++++++++++++++++++++++++++++++++++++ tools.py | 87 ++++++++++++---- tree_cache.py | 150 ++++++++++++++++++++++++++++ 8 files changed, 525 insertions(+), 26 deletions(-) create mode 100644 tests/test_tree_cache.py create mode 100644 tree_cache.py diff --git a/README.md b/README.md index c0d278f..c2acadd 100644 --- a/README.md +++ b/README.md @@ -521,7 +521,8 @@ The following shows the built-in defaults (when no `config.json` is provided). T "unicode_box": true // false → plain ASCII +/-/| instead of ┌─┐│└┘ }, "tree": { - "max_depth": 8 // maximum depth to traverse (Windows only) + "max_depth": 8, // hard cap on traversal depth + "cache_ttl_s": 2.0 // per-window tree cache TTL; input actions invalidate automatically }, "logging": { "level": "INFO" // DEBUG / INFO / WARNING / ERROR diff --git a/config.json.example b/config.json.example index b395462..1784cff 100644 --- a/config.json.example +++ b/config.json.example @@ -79,9 +79,10 @@ "vlm_fallback": false }, - "_tree": "Accessibility-tree walk limits. 'max_depth' caps recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval. Raise it if your target app legitimately nests deeper.", + "_tree": "Accessibility-tree walk limits. 'max_depth' caps recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval. Raise it if your target app legitimately nests deeper. 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse.", "tree": { - "max_depth": 8 + "max_depth": 8, + "cache_ttl_s": 2.0 }, "_logging": "Python logging level for the OSScreenObserver loggers. Use DEBUG to trace adapter behavior, WARNING to silence routine info.", diff --git a/main.py b/main.py index 8275423..6ce1d7a 100644 --- a/main.py +++ b/main.py @@ -63,7 +63,7 @@ "model": None, "max_tokens": 1500}, "ascii_sketch": {"grid_width": 110, "grid_height": 38, "unicode_box": True}, - "tree": {"max_depth": 8}, + "tree": {"max_depth": 8, "cache_ttl_s": 2.0}, "logging": {"level": "INFO"}, "mock": False, "platform": "auto", diff --git a/observer.py b/observer.py index 6ea8b3d..3b73cc4 100644 --- a/observer.py +++ b/observer.py @@ -17,9 +17,10 @@ import logging import os import platform +import time import traceback from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Tuple logger = logging.getLogger(__name__) @@ -191,6 +192,12 @@ def __init__(self) -> None: # --scenario is supplied; methods route through the scenario when # active so that input actions can drive state transitions. self.scenario: Optional[Any] = None + # Test hooks (P1 perf work): capture_count increments on every tree + # walk so tests can assert cache hits avoided adapter work; + # tree_mutator, when set, post-processes (or replaces) each captured + # tree so tests can simulate UI changes between captures. + self.capture_count: int = 0 + self.tree_mutator: Optional[Callable[[UIElement], UIElement]] = None def get_windows_above_bounds(self, hwnd) -> List["Bounds"]: return [] # Mock assumes the target window is on top @@ -211,6 +218,15 @@ def list_windows(self) -> List[WindowInfo]: ] def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + self.capture_count += 1 + tree = self._build_tree(hwnd) + if tree is not None and self.tree_mutator is not None: + mutated = self.tree_mutator(tree) + if mutated is not None: + tree = mutated + return tree + + def _build_tree(self, hwnd=None) -> Optional[UIElement]: if self.scenario is not None: return self.scenario.get_element_tree(hwnd) root = UIElement("root", "Untitled — Notepad", "Window", @@ -1173,8 +1189,79 @@ def is_mock(self) -> bool: def list_windows(self) -> List[WindowInfo]: return self._adapter.list_windows() - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - return self._adapter.get_element_tree(hwnd) + def get_element_tree(self, hwnd=None, window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return the accessibility tree for a window. + + When *window_uid* is supplied the per-window tree cache is consulted: + a fresh cached capture (within ``tree.cache_ttl_s``) is returned + without walking the adapter; a miss walks and stores. Pass + ``use_cache=False`` to force a fresh walk (post-action re-reads); + the fresh capture still refreshes the cache. Calls without a + *window_uid* always walk and are never cached. + """ + tree, _meta = self.get_element_tree_with_meta( + hwnd, window_uid=window_uid, use_cache=use_cache) + return tree + + def get_element_tree_with_meta( + self, hwnd=None, *, window_uid: Optional[str] = None, + use_cache: bool = True, + ) -> Tuple[Optional[UIElement], Dict[str, Any]]: + """get_element_tree plus capture metadata. + + Returns (tree, meta) where meta is + ``{"cache": "hit"|"miss"|"bypass", "capture_ms": int, + "node_count": int}``. + """ + tree_cfg = self.config.get("tree", {}) or {} + ttl = float(tree_cfg.get("cache_ttl_s", 2.0)) + cache = self._tree_cache() + + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, ttl_s=ttl) + if entry is not None: + return entry.tree, { + "cache": "hit", + "capture_ms": 0, + "node_count": entry.node_count, + } + + started = time.time() + tree = self._adapter.get_element_tree(hwnd) + capture_ms = int((time.time() - started) * 1000) + node_count = len(tree.flat_list()) if tree is not None else 0 + + if tree is not None and window_uid and cache is not None: + from hashing import tree_hash as _tree_hash + named = sum( + 1 for e in tree.flat_list()[1:] if (e.name or "").strip() + ) + cache.put( + window_uid, + tree=tree, + serialized=tree.to_dict(), + tree_hash=_tree_hash(tree), + max_depth=int(tree_cfg.get("max_depth", 8)), + capture_ms=capture_ms, + node_count=node_count, + named_node_count=named, + ) + + return tree, { + "cache": "bypass" if not use_cache else "miss", + "capture_ms": capture_ms, + "node_count": node_count, + } + + @staticmethod + def _tree_cache(): + """The session-scoped TreeCache (lazy import avoids cycles).""" + try: + from session import get_session + return get_session().tree_cache + except Exception: + return None def get_screenshot(self, hwnd=None) -> Optional[bytes]: return self._adapter.get_screenshot(hwnd) diff --git a/session.py b/session.py index 6d55834..26ee0ca 100644 --- a/session.py +++ b/session.py @@ -4,6 +4,7 @@ Holds per-process state shared between the REST and MCP interfaces: - step_id counter and last input step (for caused_by_step_id) - tree token ring buffer per window_uid (for since= diffs) + - per-window tree cache (avoids re-walking the accessibility tree) - snapshots (TTL-bounded, LRU) - confirmation tokens (TTL-bounded) - active trace handle (set by tracing.py) @@ -20,6 +21,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple +from tree_cache import TreeCache + # ─── Tunables (overridable from config) ────────────────────────────────────── @@ -234,6 +237,7 @@ def uptime_s(self) -> float: @dataclass class Session: tree_tokens: _TreeTokenStore = field(default_factory=_TreeTokenStore) + tree_cache: TreeCache = field(default_factory=TreeCache) snapshots: _SnapshotStore = field(default_factory=_SnapshotStore) confirms: _ConfirmStore = field(default_factory=_ConfirmStore) steps: _StepCounter = field(default_factory=_StepCounter) diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py new file mode 100644 index 0000000..d153b8c --- /dev/null +++ b/tests/test_tree_cache.py @@ -0,0 +1,207 @@ +"""Tests for the per-window tree cache (P1 perception performance).""" +from __future__ import annotations + +import time + +import pytest + +import tools as _tools +from observer import Bounds, UIElement +from session import get_session +from tree_cache import TreeCache + + +# ─── Helpers / fixtures ─────────────────────────────────────────────────────── + +def _make_tree(name: str = "T") -> UIElement: + return UIElement("root", name, "Window", bounds=Bounds(0, 0, 10, 10)) + + +def _entry_kwargs(name: str = "T") -> dict: + t = _make_tree(name) + return dict(tree=t, serialized=t.to_dict(), tree_hash=f"sha1:{name}", + max_depth=8, capture_ms=1, node_count=1, named_node_count=1) + + +@pytest.fixture() +def ctx(config, observer, renderer, describer): + return _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + +def _first_window(observer): + w = observer.list_windows()[0] + return w.handle, w.window_uid + + +# ─── TreeCache unit tests ───────────────────────────────────────────────────── + +def test_cache_put_get_hit(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs("A")) + e = c.get("w1") + assert e is not None + assert e.tree_hash == "sha1:A" + assert c.get("nope") is None + + +def test_cache_ttl_expiry(): + c = TreeCache(ttl_s=60) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 120 # age the entry past the TTL + assert c.get("w1") is None # expired entries are dropped + assert "w1" not in c + + +def test_cache_per_call_ttl_override(): + c = TreeCache(ttl_s=0.0) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 1 + assert c.get("w1", ttl_s=60) is not None + + +def test_cache_peek_ignores_ttl(): + c = TreeCache(ttl_s=60) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 120 + assert c.peek("w1") is not None # baseline lookups ignore TTL + + +def test_cache_invalidate(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs()) + assert c.invalidate("w1") is True + assert c.get("w1") is None + assert c.invalidate("w1") is False + + +def test_cache_lru_eviction(): + c = TreeCache(ttl_s=60, max_windows=3) + for i in range(3): + c.put(f"w{i}", **_entry_kwargs()) + c.get("w0") # touch w0 → w1 becomes LRU + c.put("w3", **_entry_kwargs()) + assert "w0" in c and "w2" in c and "w3" in c + assert "w1" not in c + assert len(c) == 3 + + +def test_cache_stats_survive_invalidation(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs()) + c.invalidate("w1") + stats = c.stats() + assert "w1" in stats + assert stats["w1"]["node_count"] == 1 + assert stats["w1"]["named_node_count"] == 1 + + +# ─── Observer-level caching ─────────────────────────────────────────────────── + +def test_observer_cache_hit_avoids_walk(observer): + hwnd, uid = _first_window(observer) + adapter = observer._adapter + t1 = observer.get_element_tree(hwnd, window_uid=uid) + assert adapter.capture_count == 1 + t2 = observer.get_element_tree(hwnd, window_uid=uid) + assert adapter.capture_count == 1 # served from cache + assert t2 is t1 + + +def test_observer_cache_meta_hit_miss_bypass(observer): + hwnd, uid = _first_window(observer) + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid) + assert meta["cache"] == "miss" + assert meta["node_count"] > 1 + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid) + assert meta["cache"] == "hit" + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid, + use_cache=False) + assert meta["cache"] == "bypass" + assert observer._adapter.capture_count == 2 + + +def test_observer_cache_ttl_expiry_rewalks(observer): + hwnd, uid = _first_window(observer) + observer.get_element_tree(hwnd, window_uid=uid) + entry = get_session().tree_cache.peek(uid) + entry.captured_at = time.time() - 100 # simulate TTL expiry + observer.get_element_tree(hwnd, window_uid=uid) + assert observer._adapter.capture_count == 2 + + +def test_observer_no_uid_is_never_cached(observer): + hwnd, _ = _first_window(observer) + observer.get_element_tree(hwnd) + observer.get_element_tree(hwnd) + assert observer._adapter.capture_count == 2 + assert len(get_session().tree_cache) == 0 + + +def test_observer_bypass_still_refreshes_cache(observer): + hwnd, uid = _first_window(observer) + observer.get_element_tree(hwnd, window_uid=uid, use_cache=False) + assert get_session().tree_cache.peek(uid) is not None + + +# ─── Invalidation through tools.dispatch ───────────────────────────────────── + +def test_input_tool_invalidates_target_window(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert uid in get_session().tree_cache + + r = _tools.dispatch(ctx, "click_element", { + "window_index": 0, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }) + assert r["ok"] is True + assert uid not in get_session().tree_cache + + +def test_legacy_input_tool_invalidates_all(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert uid in get_session().tree_cache + + _tools.dispatch(ctx, "type_text", {"text": "hi"}) # no window binding + assert len(get_session().tree_cache) == 0 + + +def test_read_only_tool_does_not_invalidate(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + before = observer._adapter.capture_count + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert observer._adapter.capture_count == before # cache hit + assert uid in get_session().tree_cache + + +def test_receipt_after_state_bypasses_cache(ctx, observer): + """A mutation applied after the cache is warmed must be visible in the + ActionReceipt's after-state (post-action reads bypass the cache).""" + adapter = observer._adapter + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + + def _rename(tree): + tree.name = "MUTATED TITLE" + return tree + + adapter.tree_mutator = _rename + r = _tools.dispatch(ctx, "click_element", { + "window_index": 0, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }) + assert r["ok"] is True + # before came from the warm cache; after re-walked and saw the mutation. + assert r["before"]["tree_hash"] != r["after"]["tree_hash"] + assert r["changed"] is True + + +def test_mock_adapter_mutation_hook(observer): + adapter = observer._adapter + adapter.tree_mutator = lambda t: UIElement( + "root", "Replaced", "Window", bounds=Bounds(0, 0, 1, 1)) + tree = observer.get_element_tree(None) + assert tree.name == "Replaced" + assert adapter.capture_count == 1 diff --git a/tools.py b/tools.py index 27ef2e1..2a8fbfb 100644 --- a/tools.py +++ b/tools.py @@ -204,7 +204,8 @@ def find_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) @@ -291,7 +292,8 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) @@ -336,7 +338,11 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An duration_ms = int((time.time() - started) * 1000) after_windows = ctx.observer.list_windows() info_after = ctx.observer.window_by_uid(after_windows, info.window_uid) - after_tree = (ctx.observer.get_element_tree(info_after.handle) + # Post-action re-read must bypass the tree cache so the receipt's + # after-state reflects reality (the fresh capture refreshes the cache). + after_tree = (ctx.observer.get_element_tree(info_after.handle, + window_uid=info_after.window_uid, + use_cache=False) if info_after else None) ok = bool(executor_result.get("success", True)) @@ -517,8 +523,11 @@ def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] "button": "left", "double": False}, hwnd=info.handle, ) - # Re-walk to see the now-visible option list. - new_tree = ctx.observer.get_element_tree(info.handle) + # Re-walk to see the now-visible option list (bypass the cache — + # the click above just changed the UI). + new_tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) target: Optional[UIElement] = None if new_tree is not None: descendants = [] @@ -592,7 +601,8 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) @@ -704,7 +714,8 @@ def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, ) -> Tuple[Optional[UIElement], Dict[str, Any]]: - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return None, {"error": "no tree"} serialized = tree.to_dict() @@ -741,7 +752,8 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: return full entry = get_session().tree_tokens.get(since) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) @@ -786,7 +798,8 @@ def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: hashes: Dict[str, str] = {} for w in windows: try: - t = ctx.observer.get_element_tree(w.handle) + t = ctx.observer.get_element_tree(w.handle, + window_uid=w.window_uid) if t is not None and w.window_uid: trees[w.window_uid] = t.to_dict() hashes[w.window_uid] = tree_hash(t) @@ -907,12 +920,17 @@ def _check_condition(ctx: ToolContext, cond: Dict[str, Any], entry = sess.tree_tokens.get(token) if token else None if entry is None or info is None: return False, {} - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) return tree is not None and tree_hash(tree) != entry.tree_hash, {} if info is None: return False, {} - tree = ctx.observer.get_element_tree(info.handle) + # Polling must observe fresh state — bypass the tree cache. + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) if tree is None: return False, {} @@ -1009,7 +1027,9 @@ def wait_idle(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: last_hash = None last_change_at = time.time() while (time.time() - started) * 1000 < timeout_ms: - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) if tree is None: time.sleep(poll_ms / 1000.0) continue @@ -1238,7 +1258,8 @@ def get_screenshot_cropped(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, max_width: Optional[int] = args.get("max_width") if bbox is None and element_id and info is not None: - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is not None: elem = _find_by_id(tree, element_id) if elem is not None: @@ -1321,7 +1342,8 @@ def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: bbox = args.get("bbox") element_id = args.get("element_id") if element_id and not bbox: - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is not None: elem = _find_by_id(tree, element_id) if elem is not None: @@ -1388,7 +1410,8 @@ def get_screen_description(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id) @@ -1673,6 +1696,8 @@ def load_scenario(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: try: sc = _scn.load(path) _scn.attach_to_observer(sc, ctx.observer) + # The scenario replaces the mock world — cached trees are stale. + get_session().tree_cache.invalidate_all() except _scn.ScenarioError as e: return error_dict(Code.SCENARIO_INVALID, str(e), step_id=step_id, path=path) @@ -1752,7 +1777,8 @@ def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id) @@ -1853,7 +1879,9 @@ def drag(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: # Resolve element references on either end. windows, res = _resolve_window(ctx, args) info = res.info or _focused_window(windows) - tree = ctx.observer.get_element_tree(info.handle) if info else None + tree = (ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if info else None) def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: if "x" in spec and "y" in spec: @@ -2002,7 +2030,8 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any windows0 = ctx.observer.list_windows() focused0 = next((w for w in windows0 if w.is_focused), None) if focused0: - t = ctx.observer.get_element_tree(focused0.handle) + t = ctx.observer.get_element_tree( + focused0.handle, window_uid=focused0.window_uid) if t: tree_before = tree_hash(t) except Exception: @@ -2016,6 +2045,24 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any duration_ms = int((time.time() - started) * 1000) + # P1 tree cache invalidation — the single choke point. After any input + # tool (including bring_to_foreground) the affected window's cached tree + # is stale; drop it so subsequent reads re-walk. When the target window + # cannot be determined (legacy coordinate tools), drop everything. + if _is_input_tool(name): + try: + uid = None + if isinstance(result, dict): + uid = ((result.get("target") or {}).get("window_uid") + or result.get("window_uid")) + uid = uid or (args or {}).get("window_uid") + if uid: + sess.tree_cache.invalidate(uid) + else: + sess.tree_cache.invalidate_all() + except Exception: + logger.exception("tree cache invalidation failed") + # Recurse-safety: don't trace meta tools that would recurse forever. if name not in {"trace_start", "trace_stop", "trace_status", "replay_start", "replay_step", "replay_status", @@ -2038,7 +2085,9 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any after_w = ctx.observer.list_windows() f = next((w for w in after_w if w.is_focused), None) if f: - t = ctx.observer.get_element_tree(f.handle) + t = ctx.observer.get_element_tree( + f.handle, window_uid=f.window_uid, + use_cache=False) if t: tree_after = tree_hash(t) except Exception: diff --git a/tree_cache.py b/tree_cache.py new file mode 100644 index 0000000..c9d4a4a --- /dev/null +++ b/tree_cache.py @@ -0,0 +1,150 @@ +""" +tree_cache.py — Per-window accessibility-tree cache (perception performance P1). + +Walking the accessibility tree is by far the most expensive observation +primitive (multi-second on complex Windows apps). Historically every tool +call re-walked the full tree; this module caches the most recent capture per +``window_uid`` so read-only tools within a short TTL reuse it. + +Coherence model +--------------- + - Entries expire after ``tree.cache_ttl_s`` seconds (default 2.0). + - ``tools.dispatch()`` invalidates a window's entry after any input tool + (clicks, typing, bring_to_foreground, …) executes, so post-action reads + never observe pre-action state. + - Post-action re-reads (ActionReceipt ``after`` captures) bypass the cache + explicitly via ``use_cache=False``. + +The cache also remembers lightweight per-window last-capture statistics +(node counts) which survive invalidation; ``get_capabilities`` surfaces them +so agents can detect accessibility-dark windows. +""" + +from __future__ import annotations + +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any, Dict, Optional + +# Defaults (overridable via config / constructor). +DEFAULT_TTL_S = 2.0 +DEFAULT_MAX_WINDOWS = 8 +_MAX_STATS = 64 # per-window last-capture stats retained + + +@dataclass +class TreeCacheEntry: + window_uid: str + tree: Any # UIElement root of the capture + serialized: Dict[str, Any] # tree.to_dict() of the same capture + tree_hash: str + captured_at: float + max_depth: int # depth cap in effect when captured + capture_ms: int = 0 + node_count: int = 0 + named_node_count: int = 0 + + def age_s(self, now: Optional[float] = None) -> float: + return (now if now is not None else time.time()) - self.captured_at + + +class TreeCache: + """Thread-safe TTL + LRU cache of per-window accessibility trees.""" + + def __init__(self, ttl_s: float = DEFAULT_TTL_S, + max_windows: int = DEFAULT_MAX_WINDOWS) -> None: + self.ttl_s = float(ttl_s) + self.max_windows = int(max_windows) + self._lock = threading.RLock() + self._entries: "OrderedDict[str, TreeCacheEntry]" = OrderedDict() + # Last-capture stats per window; kept across invalidation so + # capability reporting works even when the cache is cold. + self._stats: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + + # ── Read ───────────────────────────────────────────────────────────────── + + def get(self, window_uid: str, + ttl_s: Optional[float] = None) -> Optional[TreeCacheEntry]: + """Return a fresh entry for *window_uid*, or None (expired entries + are dropped). *ttl_s* overrides the instance TTL for this lookup.""" + limit = self.ttl_s if ttl_s is None else float(ttl_s) + with self._lock: + entry = self._entries.get(window_uid) + if entry is None: + return None + if entry.age_s() > limit: + self._entries.pop(window_uid, None) + return None + self._entries.move_to_end(window_uid) + return entry + + def peek(self, window_uid: str) -> Optional[TreeCacheEntry]: + """Return the last capture regardless of TTL (no LRU touch). + Used as the baseline for changed_only comparisons.""" + with self._lock: + return self._entries.get(window_uid) + + # ── Write ──────────────────────────────────────────────────────────────── + + def put(self, window_uid: str, *, tree: Any, serialized: Dict[str, Any], + tree_hash: str, max_depth: int, capture_ms: int = 0, + node_count: int = 0, named_node_count: int = 0) -> TreeCacheEntry: + entry = TreeCacheEntry( + window_uid=window_uid, tree=tree, serialized=serialized, + tree_hash=tree_hash, captured_at=time.time(), + max_depth=int(max_depth), capture_ms=int(capture_ms), + node_count=int(node_count), + named_node_count=int(named_node_count), + ) + with self._lock: + self._entries[window_uid] = entry + self._entries.move_to_end(window_uid) + while len(self._entries) > self.max_windows: + self._entries.popitem(last=False) + self._stats[window_uid] = { + "captured_at": entry.captured_at, + "capture_ms": entry.capture_ms, + "node_count": entry.node_count, + "named_node_count": entry.named_node_count, + } + self._stats.move_to_end(window_uid) + while len(self._stats) > _MAX_STATS: + self._stats.popitem(last=False) + return entry + + # ── Invalidation ───────────────────────────────────────────────────────── + + def invalidate(self, window_uid: str) -> bool: + """Drop the entry for one window. Returns True when one existed.""" + with self._lock: + return self._entries.pop(window_uid, None) is not None + + def invalidate_all(self) -> None: + with self._lock: + self._entries.clear() + + # ── Introspection ──────────────────────────────────────────────────────── + + def __len__(self) -> int: + with self._lock: + return len(self._entries) + + def __contains__(self, window_uid: str) -> bool: + with self._lock: + return window_uid in self._entries + + def stats(self) -> Dict[str, Dict[str, Any]]: + """Per-window last-capture statistics (survive invalidation).""" + with self._lock: + return {uid: dict(s) for uid, s in self._stats.items()} + + +# Convenience default factory used by session.Session. +def default_tree_cache() -> TreeCache: + return TreeCache() + + +__all__ = ["TreeCache", "TreeCacheEntry", "default_tree_cache", + "DEFAULT_TTL_S", "DEFAULT_MAX_WINDOWS"] From 09889624b39acae1604584240b2f3e70dd8bd0b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:32:30 +0000 Subject: [PATCH 02/14] [P1] bounded depth + subtree scope New config key tree.default_depth (5): get_window_structure and observe_window now bound the RETURNED tree to that many levels when the caller passes no depth=; explicit depth= requests are honored up to the existing tree.max_depth hard cap. Nodes whose children were dropped are marked truncated:true with a child_count so agents know to drill in. Diff baselines (tree_tokens) keep the full capture, so since= diffs and tree hashes are unaffected by payload depth. New adapter method get_element_subtree(hwnd, element_path, max_depth): MockAdapter navigates the positional element-id path of a fresh capture; WindowsAdapter navigates raw-UIA FindAll child indices so only the requested branch is walked (falling back to full walk + extraction for non-positional ids or navigation failure). The _uia_walk internals are refactored into reusable _uia_handles/_uia_walk_element pieces with module-level UIA property-id constants. ScreenObserver. get_element_subtree() prefers a fresh cached capture (no walk), then the adapter's scoped walk, then full walk + extraction; extraction returns depth-pruned copies so cached trees are never mutated. Exposed as scope= (element id) + depth= on get_window_structure and depth= on observe_window across REST (/api/structure, /api/observe), MCP schemas, and dispatch. Scoped responses carry tree_token=null since a subtree is not a valid diff baseline. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 3 +- config.json.example | 3 +- main.py | 2 +- mcp_server.py | 16 +- observer.py | 363 ++++++++++++++++++++++++++++++----------- tests/test_tools_p3.py | 133 +++++++++++++++ tools.py | 99 +++++++++-- web_inspector.py | 10 +- 8 files changed, 515 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index c2acadd..854dc19 100644 --- a/README.md +++ b/README.md @@ -416,7 +416,7 @@ The web inspector exposes the following endpoints (all `GET` unless noted): | Endpoint | Params | Description | |----------|--------|-------------| | `GET /api/windows` | — | List all visible windows | -| `GET /api/structure` | `window_index` | Accessibility element tree (JSON) | +| `GET /api/structure` | `window_index`, `depth`, `scope` | Accessibility element tree (JSON). Returned to `tree.default_depth` levels by default; deeper branches carry `truncated: true` + `child_count` — drill in with `scope=` and/or a larger `depth` (capped at `tree.max_depth`) | | `GET /api/description` | `window_index` | Combined description (accessibility + OCR + VLM, whatever is available). `mode` query parameter is accepted but ignored — always returns combined output. | | `GET /api/sketch` | `window_index`, `grid_width`, `grid_height`, `ocr` | ASCII layout sketch | | `GET /api/screenshot` | `window_index` | Screenshot as base64 PNG | @@ -522,6 +522,7 @@ The following shows the built-in defaults (when no `config.json` is provided). T }, "tree": { "max_depth": 8, // hard cap on traversal depth + "default_depth": 5, // depth returned when no depth= is requested (drill in via scope=) "cache_ttl_s": 2.0 // per-window tree cache TTL; input actions invalidate automatically }, "logging": { diff --git a/config.json.example b/config.json.example index 1784cff..0b1196e 100644 --- a/config.json.example +++ b/config.json.example @@ -79,9 +79,10 @@ "vlm_fallback": false }, - "_tree": "Accessibility-tree walk limits. 'max_depth' caps recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval. Raise it if your target app legitimately nests deeper. 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse.", + "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse.", "tree": { "max_depth": 8, + "default_depth": 5, "cache_ttl_s": 2.0 }, diff --git a/main.py b/main.py index 6ce1d7a..cd8005f 100644 --- a/main.py +++ b/main.py @@ -63,7 +63,7 @@ "model": None, "max_tokens": 1500}, "ascii_sketch": {"grid_width": 110, "grid_height": 38, "unicode_box": True}, - "tree": {"max_depth": 8, "cache_ttl_s": 2.0}, + "tree": {"max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0}, "logging": {"level": "INFO"}, "mock": False, "platform": "auto", diff --git a/mcp_server.py b/mcp_server.py index f528e45..02bc321 100755 --- a/mcp_server.py +++ b/mcp_server.py @@ -57,15 +57,23 @@ "description": ( "Return the accessibility element tree for a window as structured JSON. " "Each node carries id, name, role, value, bounds, enabled, focused, " - "keyboard_shortcut, and a children array. Supports server-side " - "filtering: roles, exclude_roles, name_regex, visible_only, " - "max_text_len, prune_empty, max_nodes (with page_cursor)." + "keyboard_shortcut, and a children array. Trees are returned to " + "tree.default_depth levels by default; deeper branches are marked " + "truncated:true with a child_count — pass depth (up to " + "tree.max_depth) or drill into one branch with scope=. " + "Supports server-side filtering: roles, exclude_roles, name_regex, " + "visible_only, max_text_len, prune_empty, max_nodes (with " + "page_cursor)." ), "inputSchema": { "type": "object", "properties": { "window_index": {"type": "integer"}, "window_uid": {"type": "string"}, + "depth": {"type": "integer", + "description": "Tree depth to return (default tree.default_depth, capped at tree.max_depth)."}, + "scope": {"type": "string", + "description": "Element id (e.g. 'root.3.2') — return only that subtree."}, "roles": {"type": "array", "items": {"type": "string"}}, "exclude_roles": {"type": "array", "items": {"type": "string"}}, "name_regex": {"type": "string"}, @@ -456,6 +464,8 @@ "window_index": {"type": "integer"}, "since": {"type": "string"}, "format": {"type": "string", "enum": ["custom", "json-patch"]}, + "depth": {"type": "integer", + "description": "Tree depth for full-format responses (default tree.default_depth, capped at tree.max_depth)."}, }, "required": [], }, diff --git a/observer.py b/observer.py index 3b73cc4..96c0a02 100644 --- a/observer.py +++ b/observer.py @@ -19,7 +19,7 @@ import platform import time import traceback -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace as _dc_replace from typing import Any, Callable, Dict, List, Optional, Tuple logger = logging.getLogger(__name__) @@ -178,6 +178,62 @@ class WindowResolution: requested_uid: Optional[str] +# ─── Subtree helpers (P1 perf: scoped drill-in) ────────────────────────────── + +def find_element_by_path(root: Optional[UIElement], + element_path: str) -> Optional[UIElement]: + """Locate an element by its positional element-id path (e.g. 'root.3.2'). + + Prefers walking the id prefixes level by level (cheap); falls back to a + full DFS by exact id for trees whose ids are not strictly positional + (e.g. nodes injected by tree synthesis get ids like 'root.2.x1').""" + if root is None or not element_path: + return None + if root.element_id == element_path: + return root + # Fast path: navigate children whose ids extend the current prefix. + if element_path.startswith(root.element_id + "."): + node = root + prefix = root.element_id + rest = element_path[len(prefix) + 1:] + found = True + for seg in rest.split("."): + prefix = f"{prefix}.{seg}" + nxt = next((c for c in node.children if c.element_id == prefix), + None) + if nxt is None: + found = False + break + node = nxt + if found: + return node + # Fallback: exhaustive search by exact id. + stack = [root] + while stack: + e = stack.pop() + if e.element_id == element_path: + return e + stack.extend(e.children) + return None + + +def prune_tree_depth(elem: Optional[UIElement], + max_depth: Optional[int]) -> Optional[UIElement]: + """Return a copy of *elem* limited to *max_depth* levels below it. + + Nodes are shallow-copied (bounds objects are shared) so the original — + possibly cache-resident — tree is never mutated.""" + if elem is None or max_depth is None: + return elem + + def _copy(e: UIElement, depth: int) -> UIElement: + kids = ([] if depth >= max_depth + else [_copy(c, depth + 1) for c in e.children]) + return _dc_replace(e, children=kids) + + return _copy(elem, 0) + + # ───────────────────────────────────────────────────────────────────────────── # Mock Adapter (no OS dependencies; safe to run in any environment) # ───────────────────────────────────────────────────────────────────────────── @@ -226,6 +282,16 @@ def get_element_tree(self, hwnd=None) -> Optional[UIElement]: tree = mutated return tree + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path*, to *max_depth* + levels below it. The mock world is synthetic, so this navigates the + positional element-id path of a fresh capture.""" + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + def _build_tree(self, hwnd=None) -> Optional[UIElement]: if self.scenario is not None: return self.scenario.get_element_tree(hwnd) @@ -363,6 +429,23 @@ def perform_action(self, action: str, element_id: str = None, 50039: "SemanticZoom",50040: "AppBar", } +# UIA property IDs (UIAutomationClient.h) used by the raw-COM walker. +_UIA_NAME = 30005 +_UIA_CTRL_TYPE = 30003 +_UIA_ENABLED = 30010 +_UIA_FOCUSED = 30008 +_UIA_VALUE = 30045 +_UIA_ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" +_UIA_ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" +_UIA_HELP_TEXT = 30013 # tooltip / description +_UIA_AUTOMATION_ID = 30011 +_UIA_RANGE_VALUE = 30047 +_UIA_RANGE_MIN = 30049 +_UIA_RANGE_MAX = 30050 +_UIA_IS_SELECTED = 30079 +_UIA_EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf +_UIA_SCOPE_CHILDREN = 0x2 + # Windows Adapter (requires: pywinauto, pywin32, psutil) # ───────────────────────────────────────────────────────────────────────────── @@ -482,6 +565,72 @@ def get_element_tree(self, hwnd=None) -> Optional[UIElement]: return uia_tree return self._synthesize_trees(uia_tree, pw_tree) + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path* via raw UIA. + + Navigates the positional child indices of the element-id path + ('root.3.2' → child 3 → child 2) so only the requested branch is + traversed. Falls back to a full walk plus extraction when the path + contains non-positional segments (synthesized ids) or navigation + fails.""" + try: + import win32gui + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + except Exception as e: + print(f"[WindowsAdapter:get_element_subtree] win32gui: {e}") + return None + + if max_depth is None: + max_depth = self.config.get("tree", {}).get("max_depth", 8) + + indices = self._parse_positional_path(element_path) + if indices is not None: + try: + raw_uia, true_cond = self._uia_handles() + elem = raw_uia.ElementFromHandle(hwnd) + for idx in indices: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + if idx < 0 or idx >= kids.Length: + elem = None + break + elem = kids.GetElement(idx) + if elem is not None: + return self._uia_walk_element( + elem, element_path, 0, max_depth, true_cond) + except Exception as e: + logger.debug(f"[WindowsAdapter:get_element_subtree] " + f"navigation failed ({e}); falling back") + + # Fallback: full walk + extraction. + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + @staticmethod + def _parse_positional_path(element_path: str) -> Optional[List[int]]: + """'root.3.2' → [3, 2]; None when the path is not purely positional.""" + segs = (element_path or "").split(".") + if not segs or segs[0] != "root": + return None + try: + return [int(s) for s in segs[1:]] + except ValueError: + return None # synthesized ids like 'root.2.x1' + + def _uia_handles(self): + """Return (raw IUIAutomation interface, true condition). + + pywinauto already initialised comtypes/UIA at import time; retrieve + the raw COM interface from its singleton (pywinauto ≥0.6 exposes it + as .iuia; older versions expose it directly).""" + from pywinauto.uia_defines import IUIA + _iuia_obj = IUIA() + raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) + return raw_uia, raw_uia.CreateTrueCondition() + def _uia_walk(self, hwnd: int) -> Optional[UIElement]: """Walk the accessibility tree via raw IUIAutomation COM calls. @@ -491,107 +640,85 @@ def _uia_walk(self, hwnd: int) -> Optional[UIElement]: always uses UIA and correctly crosses Chromium fragment boundaries. """ try: - # pywinauto already initialised comtypes/UIA at import time. - # Retrieve the raw IUIAutomation COM interface from pywinauto's singleton. - from pywinauto.uia_defines import IUIA - _iuia_obj = IUIA() - # pywinauto ≥0.6 exposes it as .iuia; older versions expose it directly. - raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) - + raw_uia, true_cond = self._uia_handles() root = raw_uia.ElementFromHandle(hwnd) - true_cond = raw_uia.CreateTrueCondition() max_depth = self.config.get("tree", {}).get("max_depth", 8) - - # UIA property IDs (UIAutomationClient.h) - _NAME = 30005 - _CTRL_TYPE = 30003 - _ENABLED = 30010 - _FOCUSED = 30008 - _VALUE = 30045 - _ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" - _ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" - _HELP_TEXT = 30013 # tooltip / description - _AUTOMATION_ID = 30011 - _RANGE_VALUE = 30047 - _RANGE_MIN = 30049 - _RANGE_MAX = 30050 - _IS_SELECTED = 30079 - _EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf - _SCOPE_CHILDREN = 0x2 - - def _prop(elem, pid, default=None): - try: - v = elem.GetCurrentPropertyValue(pid) - return v if v is not None else default - except Exception: - return default - - def walk(elem, elem_id: str, depth: int) -> UIElement: - name = _prop(elem, _NAME, "") or "" - ctrl = _prop(elem, _CTRL_TYPE, 0) or 0 - role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") - try: - r = elem.CurrentBoundingRectangle - bounds = Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) - except Exception: - bounds = Bounds(0, 0, 0, 0) - enabled = bool(_prop(elem, _ENABLED, True)) - focused = bool(_prop(elem, _FOCUSED, False)) - value = _prop(elem, _VALUE) or None - # Keyboard shortcut: prefer access key, fall back to accelerator. - ks = _prop(elem, _ACCESS_KEY) or _prop(elem, _ACCEL_KEY) or None - desc = _prop(elem, _HELP_TEXT) or None - aid = _prop(elem, _AUTOMATION_ID) or None - # RangeValue pattern: slider / progress / scrollbar - vn = _prop(elem, _RANGE_VALUE) - vmin = _prop(elem, _RANGE_MIN) - vmax = _prop(elem, _RANGE_MAX) - # SelectionItem.IsSelected: checkbox / radio / tab / menuitem - sel_raw = _prop(elem, _IS_SELECTED) - sel = bool(sel_raw) if sel_raw is not None else None - # ExpandCollapse: combobox / treeitem / menuitem - exp_raw = _prop(elem, _EXPAND_STATE) - if exp_raw in (0, 1): - exp = bool(exp_raw) - else: - exp = None - try: - vn_f = float(vn) if vn is not None else None - except Exception: - vn_f = None - try: - vmin_f = float(vmin) if vmin is not None else None - except Exception: - vmin_f = None - try: - vmax_f = float(vmax) if vmax is not None else None - except Exception: - vmax_f = None - node = UIElement( - element_id=elem_id, name=name, role=role, value=value, - bounds=bounds, enabled=enabled, focused=focused, - keyboard_shortcut=ks or None, - description=desc or None, - selected=sel, expanded=exp, - value_now=vn_f, value_min=vmin_f, value_max=vmax_f, - identifier=str(aid) if aid else None, - ) - if depth < max_depth: - try: - kids = elem.FindAll(_SCOPE_CHILDREN, true_cond) - for i in range(kids.Length): - node.children.append( - walk(kids.GetElement(i), f"{elem_id}.{i}", depth + 1) - ) - except Exception: - pass - return node - - return walk(root, "root", 0) + return self._uia_walk_element(root, "root", 0, max_depth, true_cond) except Exception as e: logger.debug(f"[WindowsAdapter:_uia_walk] {e}") return None + @staticmethod + def _uia_prop(elem, pid, default=None): + try: + v = elem.GetCurrentPropertyValue(pid) + return v if v is not None else default + except Exception: + return default + + def _uia_walk_element(self, elem, elem_id: str, depth: int, + max_depth: int, true_cond) -> UIElement: + _prop = self._uia_prop + name = _prop(elem, _UIA_NAME, "") or "" + ctrl = _prop(elem, _UIA_CTRL_TYPE, 0) or 0 + role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") + try: + r = elem.CurrentBoundingRectangle + bounds = Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) + except Exception: + bounds = Bounds(0, 0, 0, 0) + enabled = bool(_prop(elem, _UIA_ENABLED, True)) + focused = bool(_prop(elem, _UIA_FOCUSED, False)) + value = _prop(elem, _UIA_VALUE) or None + # Keyboard shortcut: prefer access key, fall back to accelerator. + ks = _prop(elem, _UIA_ACCESS_KEY) or _prop(elem, _UIA_ACCEL_KEY) or None + desc = _prop(elem, _UIA_HELP_TEXT) or None + aid = _prop(elem, _UIA_AUTOMATION_ID) or None + # RangeValue pattern: slider / progress / scrollbar + vn = _prop(elem, _UIA_RANGE_VALUE) + vmin = _prop(elem, _UIA_RANGE_MIN) + vmax = _prop(elem, _UIA_RANGE_MAX) + # SelectionItem.IsSelected: checkbox / radio / tab / menuitem + sel_raw = _prop(elem, _UIA_IS_SELECTED) + sel = bool(sel_raw) if sel_raw is not None else None + # ExpandCollapse: combobox / treeitem / menuitem + exp_raw = _prop(elem, _UIA_EXPAND_STATE) + if exp_raw in (0, 1): + exp = bool(exp_raw) + else: + exp = None + try: + vn_f = float(vn) if vn is not None else None + except Exception: + vn_f = None + try: + vmin_f = float(vmin) if vmin is not None else None + except Exception: + vmin_f = None + try: + vmax_f = float(vmax) if vmax is not None else None + except Exception: + vmax_f = None + node = UIElement( + element_id=elem_id, name=name, role=role, value=value, + bounds=bounds, enabled=enabled, focused=focused, + keyboard_shortcut=ks or None, + description=desc or None, + selected=sel, expanded=exp, + value_now=vn_f, value_min=vmin_f, value_max=vmax_f, + identifier=str(aid) if aid else None, + ) + if depth < max_depth: + try: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + for i in range(kids.Length): + node.children.append(self._uia_walk_element( + kids.GetElement(i), f"{elem_id}.{i}", + depth + 1, max_depth, true_cond)) + except Exception: + pass + return node + def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: try: try: @@ -1263,6 +1390,49 @@ def _tree_cache(): except Exception: return None + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None, + window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return only the subtree rooted at *element_path* ('root.3.2'). + + Resolution order: + 1. a fresh cached full capture (no walk at all), + 2. the adapter's native get_element_subtree (walks just the branch), + 3. full walk + extraction. + The result is depth-limited to *max_depth* levels below the subtree + root and safe to mutate (cache extraction returns copies).""" + tree_cfg = self.config.get("tree", {}) or {} + if max_depth is None: + max_depth = int(tree_cfg.get("max_depth", 8)) + + # 1. Serve from a fresh cached capture. + cache = self._tree_cache() + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, + ttl_s=float(tree_cfg.get("cache_ttl_s", 2.0))) + if entry is not None: + sub = find_element_by_path(entry.tree, element_path) + if sub is not None: + return prune_tree_depth(sub, max_depth) + + # 2. Adapter-native scoped walk. + native = getattr(self._adapter, "get_element_subtree", None) + if native is not None: + try: + sub = native(hwnd, element_path, max_depth) + if sub is not None: + return sub + except Exception: + logger.exception("[ScreenObserver:get_element_subtree] " + "adapter subtree walk failed; falling back") + + # 3. Full walk + extraction. + tree = self.get_element_tree(hwnd, window_uid=window_uid, + use_cache=use_cache) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + def get_screenshot(self, hwnd=None) -> Optional[bytes]: return self._adapter.get_screenshot(hwnd) @@ -1408,6 +1578,7 @@ def _has(mod: str) -> bool: }, "config": { "tree_max_depth": (self.config.get("tree", {}) or {}).get("max_depth", 8), + "tree_default_depth": (self.config.get("tree", {}) or {}).get("default_depth", 5), "ascii_grid": { "width": (self.config.get("ascii_sketch", {}) or {}).get("grid_width", 110), "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), diff --git a/tests/test_tools_p3.py b/tests/test_tools_p3.py index 2ada7f8..ac9ede3 100644 --- a/tests/test_tools_p3.py +++ b/tests/test_tools_p3.py @@ -55,3 +55,136 @@ def test_description_focus_element(client): f"/api/description?window_index=0&mode=accessibility&focus_element={child_id}" ).get_json() assert r["ok"] is True + + +# ─── P1 perf: bounded depth + subtree scope ────────────────────────────────── + +from observer import Bounds, UIElement # noqa: E402 + + +def _graft_deep_chain(tree, levels=9): + """Attach a linear chain of Group nodes below the tree root.""" + node = tree + for i in range(levels): + child = UIElement(f"{node.element_id}.{len(node.children)}", + f"Deep{i}", "Group", bounds=Bounds(0, 0, 5, 5)) + node.children.append(child) + node = child + return tree + + +def _max_depth_of(node, depth=0): + kids = node.get("children") or [] + if not kids: + return depth + return max(_max_depth_of(c, depth + 1) for c in kids) + + +def _find_truncated(node): + if node.get("truncated"): + return node + for c in node.get("children") or []: + f = _find_truncated(c) + if f is not None: + return f + return None + + +def test_structure_default_depth_truncates(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/structure?window_index=0").get_json() + assert r["ok"] is True + assert r["depth_used"] == 5 # tree.default_depth + assert r["depth_truncated"] is True + assert _max_depth_of(r["tree"]) == 5 + marker = _find_truncated(r["tree"]) + assert marker is not None + assert marker["child_count"] == 1 + assert marker["children"] == [] + + +def test_structure_explicit_depth_capped(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/structure?window_index=0&depth=7").get_json() + assert r["depth_used"] == 7 + assert _max_depth_of(r["tree"]) == 7 + + r = client.get("/api/structure?window_index=0&depth=20").get_json() + assert r["depth_used"] == 8 # clamped to tree.max_depth + + +def test_structure_shallow_tree_not_marked(client): + r = client.get("/api/structure?window_index=0").get_json() + assert r["depth_truncated"] is False # mock tree is only 2 deep + assert _find_truncated(r["tree"]) is None + + +def test_structure_scope_subtree(client): + r = client.get("/api/structure?window_index=0&scope=root.4").get_json() + assert r["ok"] is True + assert r["scope"] == "root.4" + assert r["tree"]["id"] == "root.4" + assert r["tree"]["role"] == "StatusBar" + assert len(r["tree"]["children"]) == 4 + assert r["tree_token"] is None # scoped ≠ diff baseline + + +def test_structure_scope_with_depth(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + deep_root = "root.7" # chain head appended at index 7 + r = client.get( + f"/api/structure?window_index=0&scope={deep_root}&depth=2" + ).get_json() + assert r["ok"] is True + assert r["tree"]["id"] == deep_root + assert _max_depth_of(r["tree"]) <= 2 + + +def test_structure_scope_not_found(client): + r = client.get("/api/structure?window_index=0&scope=root.99").get_json() + assert r["ok"] is False + assert r["error"]["code"] == "ElementNotFound" + + +def test_observe_full_is_depth_bounded(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/observe?window_index=0").get_json() + assert r["format"] == "full" + assert r["depth_used"] == 5 + assert r["depth_truncated"] is True + assert _max_depth_of(r["tree"]) == 5 + + # Diffs still compare the FULL captures (depth only bounds the payload). + diff = client.get( + f"/api/observe?window_index=0&since={r['tree_token']}" + ).get_json() + assert diff["unchanged"] is True + + +def test_mock_adapter_subtree_walk(observer): + adapter = observer._adapter + sub = adapter.get_element_subtree(None, "root.0", max_depth=0) + assert sub.element_id == "root.0" + assert sub.role == "MenuBar" + assert sub.children == [] # depth 0 prunes children + + sub = adapter.get_element_subtree(None, "root.0", max_depth=1) + assert len(sub.children) == 5 + + +def test_observer_subtree_served_from_cache(observer): + w = observer.list_windows()[0] + observer.get_element_tree(w.handle, window_uid=w.window_uid) + assert observer._adapter.capture_count == 1 + sub = observer.get_element_subtree(w.handle, "root.4", max_depth=2, + window_uid=w.window_uid) + assert sub is not None and sub.element_id == "root.4" + assert observer._adapter.capture_count == 1 # extracted from cache + + # Extraction copies — the cached tree keeps its children intact. + shallow = observer.get_element_subtree(w.handle, "root.4", max_depth=0, + window_uid=w.window_uid) + assert shallow.children == [] + full = observer.get_element_tree(w.handle, window_uid=w.window_uid) + sb = next(c for c in full.children if c.element_id == "root.4") + assert len(sb.children) == 4 diff --git a/tools.py b/tools.py index 2a8fbfb..9726495 100644 --- a/tools.py +++ b/tools.py @@ -594,6 +594,49 @@ def scroll(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) +def _effective_depth(ctx: ToolContext, args: Dict[str, Any]) -> int: + """Depth to return: caller's depth= (clamped to tree.max_depth) or + tree.default_depth when the caller passes none.""" + tree_cfg = ctx.config.get("tree", {}) or {} + hard_cap = int(tree_cfg.get("max_depth", 8)) + requested = args.get("depth") + if requested is None: + depth = int(tree_cfg.get("default_depth", 5)) + else: + try: + depth = int(requested) + except (TypeError, ValueError): + depth = int(tree_cfg.get("default_depth", 5)) + return max(0, min(depth, hard_cap)) + + +def _truncate_depth(node: Optional[Dict[str, Any]], max_depth: int, + _depth: int = 0) -> Tuple[Optional[Dict[str, Any]], bool]: + """Copy *node* limited to *max_depth* levels below it. + + Nodes whose children were dropped are marked ``truncated: true`` with a + ``child_count`` so the caller knows to drill in (via scope=/depth=). + Returns (tree, any_node_truncated). The input dict is not mutated.""" + if node is None: + return None, False + out = dict(node) + children = node.get("children") or [] + if _depth >= max_depth and children: + out["children"] = [] + out["truncated"] = True + out["child_count"] = len(children) + return out, True + truncated_any = False + new_children: List[Dict[str, Any]] = [] + for c in children: + nc, t = _truncate_depth(c, max_depth, _depth + 1) + if nc is not None: + new_children.append(nc) + truncated_any = truncated_any or t + out["children"] = new_children + return out, truncated_any + + def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: step_id, caused_by = _new_step_id("get_window_structure") windows, res = _resolve_window(ctx, args) @@ -601,14 +644,33 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An if info is None: return error_dict(Code.WINDOW_GONE, "no windows available", step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) + + depth = _effective_depth(ctx, args) + scope = args.get("scope") + + if scope: + # Drill into one branch only (element-id path, e.g. 'root.3.2'). + tree = ctx.observer.get_element_subtree( + info.handle, scope, max_depth=depth, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.ELEMENT_NOT_FOUND, + f"no element with id {scope!r} to scope to", + step_id=step_id, scope=scope, + window_uid=info.window_uid) + token = None # scoped captures are not valid diff baselines + else: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + token = None serialized = tree.to_dict() th = tree_hash(tree) - token = get_session().tree_tokens.put(info.window_uid, serialized, th) + if not scope: + token = get_session().tree_tokens.put(info.window_uid, serialized, th) + serialized, depth_truncated = _truncate_depth(serialized, depth) # P3 filtering / paging -------------------------------------------------- roles = args.get("roles") @@ -645,7 +707,7 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An filtered, max_nodes=max_nodes, page_cursor=page_cursor, ) - return { + out = { "ok": True, "success": True, "step_id": step_id, "caused_by_step_id": caused_by, "window": info.title, @@ -657,7 +719,12 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An "tree_token": token, "truncated": truncated, "next_cursor": next_cursor, + "depth_used": depth, + "depth_truncated": depth_truncated, } + if scope: + out["scope"] = scope + return out def get_screenshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: @@ -713,6 +780,7 @@ def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: # ─── P2: observe-with-diff, snapshots, wait_for, composites ────────────────── def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, + depth: Optional[int] = None, ) -> Tuple[Optional[UIElement], Dict[str, Any]]: tree = ctx.observer.get_element_tree(info.handle, window_uid=info.window_uid) @@ -720,7 +788,12 @@ def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, return None, {"error": "no tree"} serialized = tree.to_dict() th = tree_hash(tree) + # Diff baselines keep the full capture; only the returned tree is + # depth-bounded (with truncated-node markers). token = get_session().tree_tokens.put(info.window_uid, serialized, th) + depth_truncated = False + if depth is not None: + serialized, depth_truncated = _truncate_depth(serialized, depth) return tree, { "format": "full", "window_uid": info.window_uid, @@ -729,6 +802,8 @@ def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, "tree_hash": th, "tree_token": token, "base_token": None, + "depth_used": depth, + "depth_truncated": depth_truncated, } @@ -743,9 +818,10 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: step_id=step_id) since = args.get("since") fmt = args.get("format", "custom") + depth = _effective_depth(ctx, args) if not since: - _, full = _serialize_full_observation(ctx, info) + _, full = _serialize_full_observation(ctx, info, depth=depth) full.update({"ok": True, "success": True, "step_id": step_id, "caused_by_step_id": caused_by, "format": "full"}) @@ -762,14 +838,17 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: new_token = get_session().tree_tokens.put(info.window_uid, serialized, th) if entry is None or entry.window_uid != info.window_uid: - # Token expired/wrong-window: return full tree. + # Token expired/wrong-window: return full tree (depth-bounded). + out_tree, depth_truncated = _truncate_depth(serialized, depth) return { "ok": True, "success": True, "step_id": step_id, "caused_by_step_id": caused_by, "window_uid": info.window_uid, "window": info.title, - "tree": serialized, "tree_hash": th, + "tree": out_tree, "tree_hash": th, "tree_token": new_token, "base_token": None, "format": "full", + "depth_used": depth, + "depth_truncated": depth_truncated, } if fmt == "json-patch": diff --git a/web_inspector.py b/web_inspector.py index caaa872..271e345 100755 --- a/web_inspector.py +++ b/web_inspector.py @@ -1253,7 +1253,7 @@ def api_structure(): for bool_key in ("visible_only", "prune_empty"): if bool_key in args: args[bool_key] = str(args[bool_key]).lower() in ("1", "true", "yes") - for int_key in ("max_text_len", "max_nodes"): + for int_key in ("max_text_len", "max_nodes", "depth"): if int_key in args: try: args[int_key] = int(args[int_key]) @@ -1512,7 +1512,13 @@ def api_element_select(): @app.route("/api/observe") def api_observe(): - return _tool_response("observe_window", _merge_query()) + args = _merge_query() + if "depth" in args: + try: + args["depth"] = int(args["depth"]) + except (TypeError, ValueError): + args.pop("depth", None) + return _tool_response("observe_window", args) @app.route("/api/snapshot", methods=["POST"]) def api_snapshot(): From 89ee4b93669e1cc63f526a0001ba39a5bc1b30cd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:36:55 +0000 Subject: [PATCH 03/14] [P1] uia_only strategy + CacheRequest batching (mock-verified) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New config key tree.strategy: "merged" (default, unchanged behavior: raw-UIA walk + pywinauto walk + synthesis) or "uia_only", which returns the raw-UIA tree directly and skips the second pywinauto _walk and the _synthesize_trees pass — roughly halving Windows capture time. When the UIA walk fails entirely, uia_only still falls through to the pywinauto path as a rescue. The raw-UIA walker now builds a CacheRequest over the ~15 property IDs it reads (plus BoundingRectangle) and fetches children per level with FindAllBuildCache, reading properties via GetCachedPropertyValue — one cross-process COM round trip per node's children instead of ~15 per node. Every stage is guarded: CacheRequest construction failure disables batching for the walk, a per-node FindAllBuildCache failure falls back to FindAll + GetCurrentPropertyValue for that branch, and cached bounds fall back CachedBoundingRectangle → BoundingRectangle SAFEARRAY → CurrentBoundingRectangle. get_element_subtree navigation reuses the same batched walker. Tests: tests/test_uia_walk.py drives WindowsAdapter against fake comtypes/pywinauto/pywin32 modules injected via sys.modules, covering the batched path (zero per-node GetCurrentPropertyValue round trips), both fallback layers, uia_only vs merged strategy, and scoped subtree navigation. Manual verification on real Windows required. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 3 +- config.json.example | 5 +- main.py | 3 +- observer.py | 149 ++++++++++++++++++----- tests/test_uia_walk.py | 267 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 393 insertions(+), 34 deletions(-) create mode 100644 tests/test_uia_walk.py diff --git a/README.md b/README.md index 854dc19..ef33d39 100644 --- a/README.md +++ b/README.md @@ -523,7 +523,8 @@ The following shows the built-in defaults (when no `config.json` is provided). T "tree": { "max_depth": 8, // hard cap on traversal depth "default_depth": 5, // depth returned when no depth= is requested (drill in via scope=) - "cache_ttl_s": 2.0 // per-window tree cache TTL; input actions invalidate automatically + "cache_ttl_s": 2.0, // per-window tree cache TTL; input actions invalidate automatically + "strategy": "merged" // Windows capture pipeline: "merged" (UIA + pywinauto) or "uia_only" (faster) }, "logging": { "level": "INFO" // DEBUG / INFO / WARNING / ERROR diff --git a/config.json.example b/config.json.example index 0b1196e..b993f83 100644 --- a/config.json.example +++ b/config.json.example @@ -79,11 +79,12 @@ "vlm_fallback": false }, - "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse.", + "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse. 'strategy' (Windows only) selects the capture pipeline: 'merged' (default) runs both the raw-UIA walk and a second pywinauto walk and synthesizes them — richest tree, slowest; 'uia_only' runs just the raw-UIA walk — roughly half the capture time, may miss a few native-control properties the merge would contribute.", "tree": { "max_depth": 8, "default_depth": 5, - "cache_ttl_s": 2.0 + "cache_ttl_s": 2.0, + "strategy": "merged" }, "_logging": "Python logging level for the OSScreenObserver loggers. Use DEBUG to trace adapter behavior, WARNING to silence routine info.", diff --git a/main.py b/main.py index cd8005f..29741cd 100644 --- a/main.py +++ b/main.py @@ -63,7 +63,8 @@ "model": None, "max_tokens": 1500}, "ascii_sketch": {"grid_width": 110, "grid_height": 38, "unicode_box": True}, - "tree": {"max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0}, + "tree": {"max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0, + "strategy": "merged"}, "logging": {"level": "INFO"}, "mock": False, "platform": "auto", diff --git a/observer.py b/observer.py index 96c0a02..f0b9c14 100644 --- a/observer.py +++ b/observer.py @@ -430,6 +430,7 @@ def perform_action(self, action: str, element_id: str = None, } # UIA property IDs (UIAutomationClient.h) used by the raw-COM walker. +_UIA_BOUNDING_RECT = 30001 _UIA_NAME = 30005 _UIA_CTRL_TYPE = 30003 _UIA_ENABLED = 30010 @@ -446,6 +447,15 @@ def perform_action(self, action: str, element_id: str = None, _UIA_EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf _UIA_SCOPE_CHILDREN = 0x2 +# Properties bulk-fetched via a UIA CacheRequest so each level of the walk is +# one COM round trip (FindAllBuildCache) instead of ~15 per node. +_UIA_CACHED_PROPS = ( + _UIA_BOUNDING_RECT, _UIA_NAME, _UIA_CTRL_TYPE, _UIA_ENABLED, + _UIA_FOCUSED, _UIA_VALUE, _UIA_ACCESS_KEY, _UIA_ACCEL_KEY, + _UIA_HELP_TEXT, _UIA_AUTOMATION_ID, _UIA_RANGE_VALUE, _UIA_RANGE_MIN, + _UIA_RANGE_MAX, _UIA_IS_SELECTED, _UIA_EXPAND_STATE, +) + # Windows Adapter (requires: pywinauto, pywin32, psutil) # ───────────────────────────────────────────────────────────────────────────── @@ -546,7 +556,14 @@ def get_element_tree(self, hwnd=None) -> Optional[UIElement]: # Run both walkers and synthesise: UIA crosses Chromium fragment boundaries # (gets web content); pywinauto may surface native-control properties that # UIA omits. The merged result gives the LLM everything either source sees. + # tree.strategy == "uia_only" skips the second pywinauto walk and the + # synthesis pass entirely — roughly halving capture time — at the cost + # of the extra native-control properties the merge would contribute. + strategy = str(self.config.get("tree", {}).get("strategy", + "merged")).lower() uia_tree = self._uia_walk(hwnd) + if strategy == "uia_only" and uia_tree is not None: + return uia_tree pw_tree = None try: app = self._Application(backend="uia").connect(handle=hwnd) @@ -599,7 +616,8 @@ def get_element_subtree(self, hwnd=None, element_path: str = "root", elem = kids.GetElement(idx) if elem is not None: return self._uia_walk_element( - elem, element_path, 0, max_depth, true_cond) + elem, element_path, 0, max_depth, true_cond, + cache_request=self._uia_cache_request(raw_uia)) except Exception as e: logger.debug(f"[WindowsAdapter:get_element_subtree] " f"navigation failed ({e}); falling back") @@ -638,51 +656,106 @@ def _uia_walk(self, hwnd: int) -> Optional[UIElement]: elements (e.g. Chrome_RenderWidgetHostHWND), missing all web content. Using FindAll(TreeScope_Children) directly on the IUIAutomationElement always uses UIA and correctly crosses Chromium fragment boundaries. + + When the COM API supports it, a CacheRequest bulk-fetches the walked + properties per level (FindAllBuildCache + GetCachedPropertyValue) — + one cross-process round trip per node's children instead of ~15 per + node. Any CacheRequest failure falls back to the per-property path. """ try: raw_uia, true_cond = self._uia_handles() root = raw_uia.ElementFromHandle(hwnd) max_depth = self.config.get("tree", {}).get("max_depth", 8) - return self._uia_walk_element(root, "root", 0, max_depth, true_cond) + cache_request = self._uia_cache_request(raw_uia) + return self._uia_walk_element(root, "root", 0, max_depth, + true_cond, + cache_request=cache_request) except Exception as e: logger.debug(f"[WindowsAdapter:_uia_walk] {e}") return None @staticmethod - def _uia_prop(elem, pid, default=None): + def _uia_cache_request(raw_uia): + """Build a CacheRequest covering the properties the walker reads. + Returns None (per-property fallback) when construction fails.""" + try: + cr = raw_uia.CreateCacheRequest() + for pid in _UIA_CACHED_PROPS: + cr.AddProperty(pid) + return cr + except Exception as e: + logger.debug(f"[WindowsAdapter:_uia_cache_request] CacheRequest " + f"unavailable ({e}); using per-property fetches") + return None + + @staticmethod + def _uia_prop(elem, pid, default=None, cached=False): + if cached: + try: + v = elem.GetCachedPropertyValue(pid) + return v if v is not None else default + except Exception: + pass # cache miss/failure — fall back to a live fetch try: v = elem.GetCurrentPropertyValue(pid) return v if v is not None else default except Exception: return default - def _uia_walk_element(self, elem, elem_id: str, depth: int, - max_depth: int, true_cond) -> UIElement: - _prop = self._uia_prop - name = _prop(elem, _UIA_NAME, "") or "" - ctrl = _prop(elem, _UIA_CTRL_TYPE, 0) or 0 - role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") + @staticmethod + def _uia_bounds(elem, cached=False) -> Bounds: + if cached: + try: + r = elem.CachedBoundingRectangle + return Bounds(r.left, r.top, + r.right - r.left, r.bottom - r.top) + except Exception: + try: + arr = elem.GetCachedPropertyValue(_UIA_BOUNDING_RECT) + if arr is not None and len(arr) == 4: + # VT_R8 SAFEARRAY: [left, top, width, height] + return Bounds(int(arr[0]), int(arr[1]), + int(arr[2]), int(arr[3])) + except Exception: + pass try: r = elem.CurrentBoundingRectangle - bounds = Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) + return Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) except Exception: - bounds = Bounds(0, 0, 0, 0) - enabled = bool(_prop(elem, _UIA_ENABLED, True)) - focused = bool(_prop(elem, _UIA_FOCUSED, False)) - value = _prop(elem, _UIA_VALUE) or None + return Bounds(0, 0, 0, 0) + + def _uia_walk_element(self, elem, elem_id: str, depth: int, + max_depth: int, true_cond, + cache_request=None, cached: bool = False + ) -> UIElement: + """Build a UIElement for *elem* and recurse into its children. + + *cached* means this element was fetched via FindAllBuildCache and its + properties can be read with GetCachedPropertyValue (no round trip). + """ + def _prop(pid, default=None): + return self._uia_prop(elem, pid, default, cached=cached) + + name = _prop(_UIA_NAME, "") or "" + ctrl = _prop(_UIA_CTRL_TYPE, 0) or 0 + role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") + bounds = self._uia_bounds(elem, cached=cached) + enabled = bool(_prop(_UIA_ENABLED, True)) + focused = bool(_prop(_UIA_FOCUSED, False)) + value = _prop(_UIA_VALUE) or None # Keyboard shortcut: prefer access key, fall back to accelerator. - ks = _prop(elem, _UIA_ACCESS_KEY) or _prop(elem, _UIA_ACCEL_KEY) or None - desc = _prop(elem, _UIA_HELP_TEXT) or None - aid = _prop(elem, _UIA_AUTOMATION_ID) or None + ks = _prop(_UIA_ACCESS_KEY) or _prop(_UIA_ACCEL_KEY) or None + desc = _prop(_UIA_HELP_TEXT) or None + aid = _prop(_UIA_AUTOMATION_ID) or None # RangeValue pattern: slider / progress / scrollbar - vn = _prop(elem, _UIA_RANGE_VALUE) - vmin = _prop(elem, _UIA_RANGE_MIN) - vmax = _prop(elem, _UIA_RANGE_MAX) + vn = _prop(_UIA_RANGE_VALUE) + vmin = _prop(_UIA_RANGE_MIN) + vmax = _prop(_UIA_RANGE_MAX) # SelectionItem.IsSelected: checkbox / radio / tab / menuitem - sel_raw = _prop(elem, _UIA_IS_SELECTED) + sel_raw = _prop(_UIA_IS_SELECTED) sel = bool(sel_raw) if sel_raw is not None else None # ExpandCollapse: combobox / treeitem / menuitem - exp_raw = _prop(elem, _UIA_EXPAND_STATE) + exp_raw = _prop(_UIA_EXPAND_STATE) if exp_raw in (0, 1): exp = bool(exp_raw) else: @@ -709,14 +782,30 @@ def _uia_walk_element(self, elem, elem_id: str, depth: int, identifier=str(aid) if aid else None, ) if depth < max_depth: - try: - kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) - for i in range(kids.Length): - node.children.append(self._uia_walk_element( - kids.GetElement(i), f"{elem_id}.{i}", - depth + 1, max_depth, true_cond)) - except Exception: - pass + kids = None + kids_cached = False + if cache_request is not None: + try: + kids = elem.FindAllBuildCache(_UIA_SCOPE_CHILDREN, + true_cond, cache_request) + kids_cached = True + except Exception: + kids = None # per-node fallback below + if kids is None: + try: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + except Exception: + kids = None + if kids is not None: + try: + for i in range(kids.Length): + node.children.append(self._uia_walk_element( + kids.GetElement(i), f"{elem_id}.{i}", + depth + 1, max_depth, true_cond, + cache_request=cache_request, + cached=kids_cached)) + except Exception: + pass return node def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: diff --git a/tests/test_uia_walk.py b/tests/test_uia_walk.py new file mode 100644 index 0000000..6236055 --- /dev/null +++ b/tests/test_uia_walk.py @@ -0,0 +1,267 @@ +"""Unit tests for the WindowsAdapter raw-UIA walker (P1 perf). + +Real COM/UIA is unavailable off-Windows, so these tests inject fake +comtypes/pywinauto/pywin32 modules via sys.modules and exercise both the +CacheRequest bulk-fetch path and the per-property fallback path. +Manual verification on real Windows is still required. +""" +from __future__ import annotations + +import sys +import types + +import pytest + +from observer import WindowsAdapter + + +# ─── Fake COM / UIA infrastructure ─────────────────────────────────────────── + +_NAME, _CTRL_TYPE = 30005, 30003 +_BOUNDING_RECT = 30001 + + +class FakeRect: + def __init__(self, left=0, top=0, right=10, bottom=10): + self.left, self.top, self.right, self.bottom = left, top, right, bottom + + +class FakeElementList: + def __init__(self, items): + self._items = list(items) + + @property + def Length(self): + return len(self._items) + + def GetElement(self, i): + return self._items[i] + + +class FakeUIAElement: + """IUIAutomationElement stand-in with call counters.""" + + def __init__(self, name, ctrl_type=50000, children=(), + fail_build_cache=False): + self._props = {_NAME: name, _CTRL_TYPE: ctrl_type} + self._children = list(children) + self.fail_build_cache = fail_build_cache + self.current_calls = 0 # GetCurrentPropertyValue round trips + self.cached_calls = 0 + self.find_all_calls = 0 + self.build_cache_calls = 0 + self.fetched_via_cache = False + + # Per-property (cross-process on real Windows). + def GetCurrentPropertyValue(self, pid): + self.current_calls += 1 + return self._props.get(pid) + + def GetCachedPropertyValue(self, pid): + self.cached_calls += 1 + if pid == _BOUNDING_RECT: + return [1.0, 2.0, 3.0, 4.0] + return self._props.get(pid) + + @property + def CurrentBoundingRectangle(self): + self.current_calls += 1 + return FakeRect() + + @property + def CachedBoundingRectangle(self): + raise RuntimeError("not cached — use GetCachedPropertyValue") + + def FindAll(self, scope, cond): + self.find_all_calls += 1 + return FakeElementList(self._children) + + def FindAllBuildCache(self, scope, cond, cache_request): + self.build_cache_calls += 1 + if self.fail_build_cache: + raise RuntimeError("BuildCache unsupported here") + assert cache_request is not None + for c in self._children: + c.fetched_via_cache = True + return FakeElementList(self._children) + + +class FakeCacheRequest: + def __init__(self): + self.properties = [] + + def AddProperty(self, pid): + self.properties.append(pid) + + +class FakeRawUIA: + def __init__(self, root, cache_request_fails=False): + self._root = root + self.cache_request_fails = cache_request_fails + self.cache_requests_built = 0 + + def ElementFromHandle(self, hwnd): + return self._root + + def CreateTrueCondition(self): + return object() + + def CreateCacheRequest(self): + if self.cache_request_fails: + raise RuntimeError("CacheRequest not supported") + self.cache_requests_built += 1 + return FakeCacheRequest() + + +class FakeApplication: + """pywinauto.Application stand-in that records connect() attempts.""" + connect_calls = 0 + + def __init__(self, backend=None): + pass + + def connect(self, handle=None): + FakeApplication.connect_calls += 1 + raise RuntimeError("no real UIA here") # → pw_tree stays None + + +def _make_fake_tree(): + """root → [child0(Button) → [grand0(Text)], child1(Edit)]""" + grand0 = FakeUIAElement("Grand", 50020) + child0 = FakeUIAElement("Child A", 50000, children=[grand0]) + child1 = FakeUIAElement("Child B", 50004) + root = FakeUIAElement("Root Window", 50032, children=[child0, child1]) + return root, child0, child1, grand0 + + +@pytest.fixture() +def fake_win(monkeypatch): + """Install fake win32/pywinauto modules; returns a factory that builds a + WindowsAdapter wired to a fake UIA tree.""" + win32gui = types.ModuleType("win32gui") + win32gui.GetForegroundWindow = lambda: 1 + monkeypatch.setitem(sys.modules, "win32gui", win32gui) + monkeypatch.setitem(sys.modules, "win32process", + types.ModuleType("win32process")) + monkeypatch.setitem(sys.modules, "psutil", types.ModuleType("psutil")) + + FakeApplication.connect_calls = 0 + pywinauto = types.ModuleType("pywinauto") + pywinauto.Application = FakeApplication + monkeypatch.setitem(sys.modules, "pywinauto", pywinauto) + + uia_defines = types.ModuleType("pywinauto.uia_defines") + pywinauto.uia_defines = uia_defines + monkeypatch.setitem(sys.modules, "pywinauto.uia_defines", uia_defines) + + def _build(config=None, cache_request_fails=False): + root, *rest = _make_fake_tree() + raw = FakeRawUIA(root, cache_request_fails=cache_request_fails) + + class FakeIUIA: + def __init__(self): + self.iuia = raw + + uia_defines.IUIA = FakeIUIA + adapter = WindowsAdapter(config or {"tree": {"max_depth": 8}}) + return adapter, raw, root, rest + + return _build + + +# ─── CacheRequest bulk-fetch path ──────────────────────────────────────────── + +def test_uia_walk_uses_cache_request(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win() + tree = adapter._uia_walk(1) + + assert tree is not None + assert tree.name == "Root Window" + assert [c.name for c in tree.children] == ["Child A", "Child B"] + assert tree.children[0].children[0].name == "Grand" + assert tree.children[0].children[0].element_id == "root.0.0" + + assert raw.cache_requests_built == 1 + # Children were fetched with FindAllBuildCache, not FindAll. + assert root.build_cache_calls == 1 + assert root.find_all_calls == 0 + # Cached elements were read via GetCachedPropertyValue — zero per-node + # GetCurrentPropertyValue round trips. + for el in (child0, child1, grand0): + assert el.fetched_via_cache is True + assert el.current_calls == 0 + assert el.cached_calls > 0 + # Cached bounds came from the BoundingRectangle SAFEARRAY fallback. + assert tree.children[0].bounds.to_dict() == { + "x": 1, "y": 2, "width": 3, "height": 4} + + +def test_cache_request_covers_walked_properties(fake_win): + adapter, raw, _, _ = fake_win() + cr = adapter._uia_cache_request(raw) + assert _NAME in cr.properties and _CTRL_TYPE in cr.properties + assert _BOUNDING_RECT in cr.properties + assert len(cr.properties) >= 12 + + +# ─── Fallback paths ────────────────────────────────────────────────────────── + +def test_uia_walk_falls_back_when_cache_request_fails(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win( + cache_request_fails=True) + tree = adapter._uia_walk(1) + + assert tree is not None + assert [c.name for c in tree.children] == ["Child A", "Child B"] + # No BuildCache attempted; classic FindAll + per-property fetches. + assert root.build_cache_calls == 0 + assert root.find_all_calls == 1 + for el in (child0, child1): + assert el.fetched_via_cache is False + assert el.current_calls > 0 + + +def test_uia_walk_falls_back_per_node_when_build_cache_raises(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win() + child0.fail_build_cache = True # BuildCache breaks below child0 + tree = adapter._uia_walk(1) + + assert tree is not None + # child0's children still walked, via the FindAll fallback… + assert child0.find_all_calls == 1 + assert tree.children[0].children[0].name == "Grand" + # …and grand0 was therefore read per-property, not from a cache. + assert grand0.fetched_via_cache is False + assert grand0.current_calls > 0 + + +# ─── tree.strategy = uia_only ──────────────────────────────────────────────── + +def test_uia_only_skips_pywinauto_walk(fake_win): + adapter, _, _, _ = fake_win(config={ + "tree": {"max_depth": 8, "strategy": "uia_only"}}) + tree = adapter.get_element_tree(1) + assert tree is not None + assert tree.name == "Root Window" + assert FakeApplication.connect_calls == 0 + + +def test_merged_strategy_still_attempts_pywinauto(fake_win): + adapter, _, _, _ = fake_win(config={ + "tree": {"max_depth": 8, "strategy": "merged"}}) + tree = adapter.get_element_tree(1) + assert tree is not None # uia tree survives pw failure + assert FakeApplication.connect_calls == 1 + + +# ─── Subtree navigation (commit 2 path, exercised against fake COM) ───────── + +def test_get_element_subtree_navigates_indices(fake_win): + adapter, _, root, (child0, child1, grand0) = fake_win() + sub = adapter.get_element_subtree(1, "root.0", max_depth=8) + assert sub is not None + assert sub.element_id == "root.0" + assert sub.name == "Child A" + assert sub.children[0].name == "Grand" + # Only the requested branch was walked — child1 untouched. + assert child1.current_calls == 0 and child1.cached_calls == 0 From 62737e60d3cf2a8f8fd0ea2d255060df6df11606 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:40:24 +0000 Subject: [PATCH 04/14] [P1] changed_only diff + perf telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit observe_window gains changed_only=true (REST /api/observe, MCP schema, dispatch): the fresh capture's tree_hash is compared against the window's last cached capture — unchanged windows return a tiny {unchanged: true, tree_hash} response; changed windows return diff.diff_custom(old, new) plus the fresh hash instead of the full tree; a window with no baseline returns the depth-bounded full tree. The baseline is looked up with TreeCache.peek() (TTL-independent) before the fresh bypass capture refreshes the cache, and since= keeps its existing token-based semantics (it takes precedence when both are passed). Every get_window_structure / observe_window result now carries perf: {capture_ms, node_count, cache: hit|miss|bypass, depth_used}, sourced from ScreenObserver.get_element_tree_with_meta (scoped captures report timing around the subtree resolution and whether a fresh cache entry backed it). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 1 + mcp_server.py | 8 +++- tests/test_tools_p2.py | 72 ++++++++++++++++++++++++++++++++ tools.py | 93 ++++++++++++++++++++++++++++++++++++++---- web_inspector.py | 3 ++ 5 files changed, 168 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ef33d39..594ce95 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,7 @@ The web inspector exposes the following endpoints (all `GET` unless noted): |----------|--------|-------------| | `GET /api/windows` | — | List all visible windows | | `GET /api/structure` | `window_index`, `depth`, `scope` | Accessibility element tree (JSON). Returned to `tree.default_depth` levels by default; deeper branches carry `truncated: true` + `child_count` — drill in with `scope=` and/or a larger `depth` (capped at `tree.max_depth`) | +| `GET /api/observe` | `window_index`/`window_uid`, `since`, `changed_only`, `depth` | Tree observation. `since=` returns a diff against that baseline; `changed_only=1` compares against the window's last capture and returns a tiny `{unchanged: true}` or a diff instead of the full tree. Responses include `perf` (`capture_ms`, `node_count`, `cache` hit/miss/bypass, `depth_used`) | | `GET /api/description` | `window_index` | Combined description (accessibility + OCR + VLM, whatever is available). `mode` query parameter is accepted but ignored — always returns combined output. | | `GET /api/sketch` | `window_index`, `grid_width`, `grid_height`, `ocr` | ASCII layout sketch | | `GET /api/screenshot` | `window_index` | Screenshot as base64 PNG | diff --git a/mcp_server.py b/mcp_server.py index 02bc321..a9c911e 100755 --- a/mcp_server.py +++ b/mcp_server.py @@ -455,7 +455,11 @@ "tree_token from a previous observation as 'since' to get only " "what changed (custom diff format by default; pass " "format='json-patch' for RFC 6902). An expired token returns the " - "full tree with base_token=null." + "full tree with base_token=null. Alternatively pass " + "changed_only=true (no since needed) to compare against the " + "window's last capture: unchanged windows return a tiny " + "{unchanged: true, tree_hash} response, changed windows return a " + "diff instead of the full tree." ), "inputSchema": { "type": "object", @@ -466,6 +470,8 @@ "format": {"type": "string", "enum": ["custom", "json-patch"]}, "depth": {"type": "integer", "description": "Tree depth for full-format responses (default tree.default_depth, capped at tree.max_depth)."}, + "changed_only": {"type": "boolean", + "description": "Compare against the window's last capture; returns {unchanged:true} or a diff instead of the full tree."}, }, "required": [], }, diff --git a/tests/test_tools_p2.py b/tests/test_tools_p2.py index 0210e0e..73505ef 100644 --- a/tests/test_tools_p2.py +++ b/tests/test_tools_p2.py @@ -85,3 +85,75 @@ def test_click_and_observe(client): }).get_json() assert r["ok"] is True assert "observation" in r + + +# ─── P1 perf: changed_only + perf telemetry ────────────────────────────────── + +def _mutate_editor(tree): + for c in tree.children: + if c.role == "Document": + c.value = "changed content" + return tree + + +def test_observe_changed_only_unchanged(client): + client.get("/api/observe?window_index=0") # warm the baseline + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["changed_only"] is True + assert r["unchanged"] is True + assert r["tree_hash"].startswith("sha1:") + assert "tree" not in r and "changes" not in r # tiny response + + +def test_observe_changed_only_returns_diff(client, observer): + base = client.get("/api/observe?window_index=0").get_json() + observer._adapter.tree_mutator = _mutate_editor + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["unchanged"] is False + assert r["format"] == "custom" + assert r["changes"] # non-empty diff + assert r["tree_hash"] != base["tree_hash"] + assert "tree" not in r # diff, not full tree + + +def test_observe_changed_only_without_baseline_returns_full(client): + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["format"] == "full" + assert "tree" in r + assert r["perf"]["cache"] == "bypass" + + +def test_observe_perf_telemetry(client): + r = client.get("/api/observe?window_index=0").get_json() + perf = r["perf"] + assert set(perf) == {"capture_ms", "node_count", "cache", "depth_used"} + assert perf["cache"] == "miss" + assert perf["node_count"] > 1 + assert perf["depth_used"] == 5 + + +def test_structure_perf_telemetry_cache_hit(client): + r1 = client.get("/api/structure?window_index=0").get_json() + assert r1["perf"]["cache"] == "miss" + r2 = client.get("/api/structure?window_index=0").get_json() + assert r2["perf"]["cache"] == "hit" + assert r2["perf"]["node_count"] == r1["perf"]["node_count"] + + +def test_structure_scoped_perf(client): + client.get("/api/structure?window_index=0") # warm cache + r = client.get("/api/structure?window_index=0&scope=root.4").get_json() + assert r["perf"]["cache"] == "hit" + assert r["perf"]["depth_used"] == 5 + + +def test_observe_since_perf(client): + full = client.get("/api/observe?window_index=0").get_json() + diff = client.get( + f"/api/observe?window_index=0&since={full['tree_token']}" + ).get_json() + assert "perf" in diff + assert diff["perf"]["cache"] in ("hit", "miss") diff --git a/tools.py b/tools.py index 9726495..d7e0516 100644 --- a/tools.py +++ b/tools.py @@ -647,25 +647,38 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An depth = _effective_depth(ctx, args) scope = args.get("scope") + token = None if scope: # Drill into one branch only (element-id path, e.g. 'root.3.2'). + ttl = float((ctx.config.get("tree", {}) or {}).get("cache_ttl_s", 2.0)) + had_fresh_entry = (get_session().tree_cache.get( + info.window_uid, ttl_s=ttl) is not None) + started = time.time() tree = ctx.observer.get_element_subtree( info.handle, scope, max_depth=depth, window_uid=info.window_uid) + capture_ms = int((time.time() - started) * 1000) if tree is None: return error_dict(Code.ELEMENT_NOT_FOUND, f"no element with id {scope!r} to scope to", step_id=step_id, scope=scope, window_uid=info.window_uid) - token = None # scoped captures are not valid diff baselines + perf = {"capture_ms": capture_ms, + "node_count": len(tree.flat_list()), + "cache": "hit" if had_fresh_entry else "miss", + "depth_used": depth} + # scoped captures are not valid diff baselines → no tree_token else: - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) - token = None + perf = {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} serialized = tree.to_dict() th = tree_hash(tree) if not scope: @@ -721,6 +734,7 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An "next_cursor": next_cursor, "depth_used": depth, "depth_truncated": depth_truncated, + "perf": perf, } if scope: out["scope"] = scope @@ -779,11 +793,18 @@ def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: # ─── P2: observe-with-diff, snapshots, wait_for, composites ────────────────── +def _perf_dict(meta: Dict[str, Any], depth: Optional[int]) -> Dict[str, Any]: + return {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} + + def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, depth: Optional[int] = None, ) -> Tuple[Optional[UIElement], Dict[str, Any]]: - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) if tree is None: return None, {"error": "no tree"} serialized = tree.to_dict() @@ -804,6 +825,7 @@ def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, "base_token": None, "depth_used": depth, "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), } @@ -819,6 +841,11 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: since = args.get("since") fmt = args.get("format", "custom") depth = _effective_depth(ctx, args) + changed_only = bool(args.get("changed_only")) + + if not since and changed_only: + return _observe_changed_only(ctx, info, depth=depth, + step_id=step_id, caused_by=caused_by) if not since: _, full = _serialize_full_observation(ctx, info, depth=depth) @@ -828,8 +855,8 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: return full entry = get_session().tree_tokens.get(since) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) if tree is None: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id, window_uid=info.window_uid) @@ -849,6 +876,7 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: "format": "full", "depth_used": depth, "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), } if fmt == "json-patch": @@ -867,9 +895,58 @@ def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: "changes": changes, "unchanged": len(changes) == 0, "tree_hash": th, + "perf": _perf_dict(meta, depth), } +def _observe_changed_only(ctx: ToolContext, info: WindowInfo, *, depth: int, + step_id: int, caused_by: Optional[int] + ) -> Dict[str, Any]: + """observe_window changed_only=true: compare a fresh capture against the + last cached capture of the window. Unchanged → a tiny + {unchanged: true, tree_hash} response; changed → a custom diff instead + of the full tree; no baseline → full tree.""" + from diff import diff_custom + sess = get_session() + # Baseline: the most recent capture, regardless of cache TTL. + baseline = sess.tree_cache.peek(info.window_uid) + + # Fresh capture (bypass the cache — the whole point is to detect drift). + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid, use_cache=False) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + serialized = tree.to_dict() + th = tree_hash(tree) + new_token = sess.tree_tokens.put(info.window_uid, serialized, th) + perf = _perf_dict(meta, depth) + + base = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree_hash": th, "tree_token": new_token, + "changed_only": True, + "perf": perf, + } + + if baseline is None: + # Nothing to compare against — return the (depth-bounded) full tree. + out_tree, depth_truncated = _truncate_depth(serialized, depth) + base.update({"format": "full", "tree": out_tree, "base_token": None, + "depth_used": depth, "depth_truncated": depth_truncated}) + return base + + if baseline.tree_hash == th: + base["unchanged"] = True + return base + + changes = diff_custom(baseline.serialized, serialized) + base.update({"format": "custom", "changes": changes, "unchanged": False}) + return base + + def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: step_id, caused_by = _new_step_id("snapshot") windows = ctx.observer.list_windows() diff --git a/web_inspector.py b/web_inspector.py index 271e345..ed3ef9b 100755 --- a/web_inspector.py +++ b/web_inspector.py @@ -1518,6 +1518,9 @@ def api_observe(): args["depth"] = int(args["depth"]) except (TypeError, ValueError): args.pop("depth", None) + if "changed_only" in args: + args["changed_only"] = str(args["changed_only"]).lower() in ( + "1", "true", "yes") return _tool_response("observe_window", args) @app.route("/api/snapshot", methods=["POST"]) From eed718e2a45288b9152b74925697cece4c6dad04 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 18:42:58 +0000 Subject: [PATCH 05/14] [P1] degradation signal get_window_structure now attaches degraded: {reason: "sparse_accessibility_tree", named_node_count, threshold, suggested_fallbacks: [get_ocr, get_screen_description]} when the captured tree has fewer named non-root nodes than tree.sparse_threshold (default 5), so agents pivot to pixel-based perception instead of acting on a near-empty tree from an accessibility-dark window. Scoped (subtree) requests are exempt since a leaf branch being small is expected. get_capabilities gains tree_stats: per-window last-capture statistics (node_count, named_node_count, capture_ms, captured_at) sourced from the TreeCache stats ledger, which survives input-driven cache invalidation. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 17 ++++++++++--- config.json.example | 5 ++-- main.py | 2 +- observer.py | 11 ++++++++ tests/test_tree_cache.py | 55 ++++++++++++++++++++++++++++++++++++++++ tools.py | 14 ++++++++++ 6 files changed, 97 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 594ce95..86005ec 100644 --- a/README.md +++ b/README.md @@ -525,7 +525,8 @@ The following shows the built-in defaults (when no `config.json` is provided). T "max_depth": 8, // hard cap on traversal depth "default_depth": 5, // depth returned when no depth= is requested (drill in via scope=) "cache_ttl_s": 2.0, // per-window tree cache TTL; input actions invalidate automatically - "strategy": "merged" // Windows capture pipeline: "merged" (UIA + pywinauto) or "uia_only" (faster) + "strategy": "merged", // Windows capture pipeline: "merged" (UIA + pywinauto) or "uia_only" (faster) + "sparse_threshold": 5 // fewer named nodes than this → degraded (sparse tree) hint in results }, "logging": { "level": "INFO" // DEBUG / INFO / WARNING / ERROR @@ -661,15 +662,23 @@ Default CI lane selects `not user` so the new tier is opt-in. 1. **Accessibility-dark applications** — Games, Electron apps with custom renderers, and applications that do not instrument UIA will produce sparse trees. The OCR - and VLM modalities degrade more gracefully in these cases. + and VLM modalities degrade more gracefully in these cases; `get_window_structure` + flags such captures with `degraded: {reason: "sparse_accessibility_tree", …}` + (threshold: `tree.sparse_threshold`) and `get_capabilities` reports per-window + `tree_stats` so agents can switch modality automatically. 2. **Prompt injection risk** — Screen content is included verbatim in tool results. Malicious content on-screen could attempt to influence the AI's behavior. Apply appropriate trust boundaries when using this server in production contexts. 3. **Performance** — Full tree traversal on a complex window (e.g., a browser with - many DOM-mapped UIA nodes) can take several seconds. The `tree.max_depth` - config key limits traversal depth. + many DOM-mapped UIA nodes) can take several seconds. Mitigations: captures are + cached per window for `tree.cache_ttl_s` (input actions invalidate), results + are depth-bounded to `tree.default_depth` with `scope=`/`depth=` drill-in, + `observe_window` supports `changed_only=1`, and on Windows `tree.strategy: + "uia_only"` plus UIA CacheRequest batching cut per-walk COM round trips. + Every structure/observe result reports `perf.capture_ms` so slow windows are + visible. `tree.max_depth` still hard-caps traversal depth. 4. **Action safety** — The `click_at`, `type_text`, and `press_key` tools execute real input events. Apply appropriate authorization controls before exposing diff --git a/config.json.example b/config.json.example index b993f83..4879f67 100644 --- a/config.json.example +++ b/config.json.example @@ -79,12 +79,13 @@ "vlm_fallback": false }, - "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse. 'strategy' (Windows only) selects the capture pipeline: 'merged' (default) runs both the raw-UIA walk and a second pywinauto walk and synthesizes them — richest tree, slowest; 'uia_only' runs just the raw-UIA walk — roughly half the capture time, may miss a few native-control properties the merge would contribute.", + "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse. 'strategy' (Windows only) selects the capture pipeline: 'merged' (default) runs both the raw-UIA walk and a second pywinauto walk and synthesizes them — richest tree, slowest; 'uia_only' runs just the raw-UIA walk — roughly half the capture time, may miss a few native-control properties the merge would contribute. 'sparse_threshold': when a capture has fewer than this many named non-root nodes, get_window_structure attaches a degraded:{reason:'sparse_accessibility_tree'} hint suggesting OCR/VLM fallbacks.", "tree": { "max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0, - "strategy": "merged" + "strategy": "merged", + "sparse_threshold": 5 }, "_logging": "Python logging level for the OSScreenObserver loggers. Use DEBUG to trace adapter behavior, WARNING to silence routine info.", diff --git a/main.py b/main.py index 29741cd..ecd20c6 100644 --- a/main.py +++ b/main.py @@ -64,7 +64,7 @@ "max_tokens": 1500}, "ascii_sketch": {"grid_width": 110, "grid_height": 38, "unicode_box": True}, "tree": {"max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0, - "strategy": "merged"}, + "strategy": "merged", "sparse_threshold": 5}, "logging": {"level": "INFO"}, "mock": False, "platform": "auto", diff --git a/observer.py b/observer.py index f0b9c14..a61ab9b 100644 --- a/observer.py +++ b/observer.py @@ -1673,8 +1673,19 @@ def _has(mod: str) -> bool: "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), }, }, + # Per-window last-capture statistics (node_count, + # named_node_count, capture_ms, captured_at) so agents can spot + # accessibility-dark windows without another walk. + "tree_stats": self._last_capture_stats(), } + def _last_capture_stats(self) -> Dict[str, Dict[str, Any]]: + cache = self._tree_cache() + try: + return cache.stats() if cache is not None else {} + except Exception: + return {} + # ── Element occlusion (design doc D14) ──────────────────────────────────── def is_element_occluded(self, element_bounds: Bounds, target_hwnd: Any, diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py index d153b8c..85f814d 100644 --- a/tests/test_tree_cache.py +++ b/tests/test_tree_cache.py @@ -205,3 +205,58 @@ def test_mock_adapter_mutation_hook(observer): tree = observer.get_element_tree(None) assert tree.name == "Replaced" assert adapter.capture_count == 1 + + +# ─── P1: degradation signal (sparse accessibility trees) ───────────────────── + +def _sparse_tree(tree): + """Simulate an accessibility-dark window: one named child only.""" + return UIElement( + "root", "Game Window", "Window", bounds=Bounds(0, 0, 800, 600), + children=[ + UIElement("root.0", "Canvas", "Pane", bounds=Bounds(0, 0, 800, 600)), + UIElement("root.1", "", "Pane", bounds=Bounds(0, 0, 10, 10)), + ]) + + +def test_sparse_tree_attaches_degraded(ctx, observer): + observer._adapter.tree_mutator = _sparse_tree + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["ok"] is True + d = r["degraded"] + assert d["reason"] == "sparse_accessibility_tree" + assert d["named_node_count"] == 1 # unnamed nodes don't count + assert d["suggested_fallbacks"] == ["get_ocr", "get_screen_description"] + + +def test_rich_tree_has_no_degraded(ctx): + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["ok"] is True + assert "degraded" not in r + + +def test_sparse_threshold_configurable(ctx, observer, config): + config.setdefault("tree", {})["sparse_threshold"] = 100 + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["degraded"]["threshold"] == 100 # even the rich mock tree trips + + +def test_capabilities_reports_tree_stats(ctx, observer): + _, uid = _first_window(observer) + caps0 = _tools.dispatch(ctx, "get_capabilities", {}) + assert caps0["tree_stats"] == {} # nothing captured yet + + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + caps = _tools.dispatch(ctx, "get_capabilities", {}) + stats = caps["tree_stats"][uid] + assert stats["node_count"] > 1 + assert stats["named_node_count"] >= 5 + assert "capture_ms" in stats and "captured_at" in stats + + +def test_capabilities_stats_survive_input_invalidation(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + _tools.dispatch(ctx, "type_text", {"text": "x"}) # invalidates cache + caps = _tools.dispatch(ctx, "get_capabilities", {}) + assert uid in caps["tree_stats"] diff --git a/tools.py b/tools.py index d7e0516..2c90164 100644 --- a/tools.py +++ b/tools.py @@ -738,6 +738,20 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An } if scope: out["scope"] = scope + else: + # Degradation signal: accessibility-dark windows (games, custom + # renderers) produce sparse trees — steer the agent to pixel-based + # fallbacks instead of letting it act on a near-empty tree. + named = sum(1 for e in tree.flat_list()[1:] if (e.name or "").strip()) + threshold = int((ctx.config.get("tree", {}) or {}) + .get("sparse_threshold", 5)) + if named < threshold: + out["degraded"] = { + "reason": "sparse_accessibility_tree", + "named_node_count": named, + "threshold": threshold, + "suggested_fallbacks": ["get_ocr", "get_screen_description"], + } return out From e954e5b101161897b128c1912cdccd7a9a298b5d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:14:52 +0000 Subject: [PATCH 06/14] [P2] secure defaults: loopback bind + opt-in remote, CORS origins config - web_ui.host built-in default flips 0.0.0.0 -> 127.0.0.1; the existing --host CLI flag / config web_ui.host stay the explicit opt-in path to 0.0.0.0 for Docker deployments and testing (no authentication added, per direction: bind policy is the control). - Startup prints a prominent banner + logger.warning whenever the bind is non-loopback ("unauthenticated action API exposed to the network"). - CORS(app) blanket permissive CORS removed: web_ui.cors_origins config (default [] = no CORS headers, same-origin only; the inspector UI is served same-origin so it keeps working). ["*"] or explicit origins re-enable CORS for Docker/testing dashboards. /api/action never gets permissive CORS by default. - README "Security & Network Bind Defaults" rewritten for the new reality (loopback default, opt-in flag, Docker note, CORS config); config reference + config.json.example updated. Note: this repo ships no Dockerfile/compose of its own (the Docker harness lives in the AutoGUI repo), so the container opt-in is documented rather than patched into an entrypoint. - Tests: bind-default config resolution, non-loopback warning banner, and CORS behavior via the Flask test client (no ACAO by default incl. /api/action + preflight; present for ["*"] and matching explicit origins; absent for non-matching origins). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 37 +++++---- config.json.example | 5 +- main.py | 42 +++++++++- tests/test_security_defaults.py | 140 ++++++++++++++++++++++++++++++++ web_inspector.py | 16 +++- 5 files changed, 222 insertions(+), 18 deletions(-) create mode 100644 tests/test_security_defaults.py diff --git a/README.md b/README.md index 86005ec..ace662f 100644 --- a/README.md +++ b/README.md @@ -17,29 +17,37 @@ OSScreenObserver exposes a full REST API on port `5001` (configurable). Most `/a > ### Security & Network Bind Defaults > -> **The default bind host is now `0.0.0.0` (all network interfaces) on port `5001`.** -> This default is intended for **testing inside an isolated sandbox or container** (e.g., a disposable VM/dev container) where exposing the API to the container network is convenient and safe. +> **The default bind host is `127.0.0.1` (loopback-only) on port `5001`.** +> The REST API has **no authentication** — any client that can reach the port can call every endpoint, including `/api/action`, which can click, type, and otherwise control the host desktop. Binding to loopback by default means only processes on the same machine can reach it. > -> **The REST API has NO authentication.** Any client that can reach the port can call every endpoint, including `/api/action`, which can click, type, and otherwise control the host desktop. -> -> **For local-only use on a workstation, override the bind to loopback:** +> **Opt-in network exposure (Docker, isolated test VMs):** pass the bind host explicitly on the command line or in config — this is the deliberate opt-in path, and startup prints a prominent warning whenever the bind is non-loopback ("unauthenticated action API exposed to the network"): > > ```bash -> # Command-line override (recommended for local dev) -> python main.py --mode both --host 127.0.0.1 +> # Explicit opt-in — e.g. inside a container whose port is published +> python main.py --mode both --host 0.0.0.0 > ``` > -> Or edit `config.json`: +> Or in `config.json`: > > ```json > { -> "web_ui": { "host": "127.0.0.1", "port": 5001 } +> "web_ui": { "host": "0.0.0.0", "port": 5001 } > } > ``` > -> Do **not** expose the default `0.0.0.0` bind on a workstation connected to an untrusted network (home Wi-Fi, café, corporate LAN, public cloud VM) without a firewall, reverse proxy with authentication, or VPN in front of it. +> **Docker note:** this repo ships no Dockerfile of its own; containerized runs (e.g. the shared AutoGUI Docker harness — see "Docker harness" below) must pass `--host 0.0.0.0` (or set `web_ui.host`) explicitly so the server stays reachable through the container's published port. Nothing else changes — the opt-in flag is the only difference from a local run. +> +> Do **not** use a non-loopback bind on a workstation connected to an untrusted network (home Wi-Fi, café, corporate LAN, public cloud VM) without a firewall, reverse proxy with authentication, or VPN in front of it. +> +> **CORS:** disabled by default. The server sends no `Access-Control-Allow-Origin` headers unless `web_ui.cors_origins` is configured (default `[]`), so browsers enforce the same-origin policy — websites open in the user's browser cannot script cross-origin calls to the API, and the bundled web inspector keeps working because it is served same-origin at `/`. To allow cross-origin callers (an external dashboard, or Docker/testing scenarios), list explicit origins — or `["*"]` for anything-goes test rigs: +> +> ```json +> { +> "web_ui": { "host": "127.0.0.1", "port": 5001, "cors_origins": ["http://localhost:3000"] } +> } +> ``` > -> **CORS warning:** The Flask server enables permissive CORS for all routes by default (`CORS(app)`). Any website running in the user's browser can send cross-origin requests to the API — including destructive `/api/action` calls. Restrict CORS origins or add an authentication/proxy layer before exposing the server to a multi-user environment. +> Permissive CORS is never sent on `/api/action` (or any other route) by default; enabling `["*"]` re-opens the cross-origin attack surface described above, so combine it with a non-loopback bind only inside fully isolated environments. ### Startup modes @@ -491,14 +499,15 @@ contains `"success": false` with an explanatory error message — the click is ## Configuration Reference (`config.json`) -The following shows the built-in defaults (when no `config.json` is provided). The shipped `config.json` overrides `web_ui.host` to `127.0.0.1` for loopback-only access. +The following shows the built-in defaults (when no `config.json` is provided). ```jsonc { "web_ui": { - "host": "0.0.0.0", // bind address; use "127.0.0.1" for loopback-only + "host": "127.0.0.1", // bind address; loopback-only by default — set "0.0.0.0" (or use --host) to opt in to network exposure (unauthenticated!) "port": 5001, // HTTP port - "debug": false + "debug": false, + "cors_origins": [] // CORS allowlist; [] = no CORS headers (same-origin only), ["*"] = permissive (isolated test rigs only) }, "mcp": { "server_name": "os-screen-observer", diff --git a/config.json.example b/config.json.example index 4879f67..d57213e 100644 --- a/config.json.example +++ b/config.json.example @@ -1,11 +1,12 @@ { "_about": "Copy this file to config.json and edit for your machine. config.json is in .gitignore. All sections are optional; OSScreenObserver falls back to reasonable defaults when a key is missing. Keys starting with an underscore are documentation-only and ignored at load time.", - "_web_ui": "HTTP inspector server. Bind to 127.0.0.1 unless you are deliberately exposing it to your LAN. Disable debug in shared environments.", + "_web_ui": "HTTP inspector server. The API is unauthenticated, so 'host' defaults to loopback (127.0.0.1); set '0.0.0.0' (or pass --host 0.0.0.0) only to deliberately expose it — e.g. inside a Docker container with a published port — and expect a startup warning. 'cors_origins' is the CORS allowlist: [] (default) sends no CORS headers so browsers enforce same-origin (the bundled inspector UI is same-origin and unaffected); list explicit origins, or [\"*\"] for isolated Docker/testing rigs. Disable debug in shared environments.", "web_ui": { "host": "127.0.0.1", "port": 5001, - "debug": false + "debug": false, + "cors_origins": [] }, "_mcp": "Identifiers reported over the MCP framing channel. Bump 'version' when you publish a breaking change to a custom fork.", diff --git a/main.py b/main.py index ecd20c6..b51abad 100644 --- a/main.py +++ b/main.py @@ -54,7 +54,11 @@ # ───────────────────────────────────────────────────────────────────────────── _DEFAULT_CONFIG = { - "web_ui": {"host": "0.0.0.0", "port": 5001, "debug": False}, + # Loopback by default: the HTTP API is unauthenticated, so it must not + # be reachable from the network unless the operator explicitly opts in + # via --host / web_ui.host (e.g. 0.0.0.0 for Docker deployments). + "web_ui": {"host": "127.0.0.1", "port": 5001, "debug": False, + "cors_origins": []}, "mcp": {"server_name": "os-screen-observer", "version": "0.1.0"}, "ocr": {"enabled": True, "tesseract_cmd": None, "min_confidence": 30}, "vlm": {"enabled": False, @@ -156,6 +160,32 @@ def config_load_status() -> dict: "config_error": _CONFIG_LOAD_ERROR or None} +_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def bind_warning(host: str) -> "str | None": + """Return a prominent warning banner when *host* exposes the HTTP API + beyond loopback, or None for loopback binds. + + The web/REST surface is unauthenticated by design (see README + "Security & Network Bind Defaults"); a non-loopback bind is an explicit + operator opt-in (Docker, isolated test VMs) and deserves a loud notice. + """ + if (host or "").strip().lower() in _LOOPBACK_HOSTS: + return None + return ( + "╔══════════════════════════════════════════════════════════════════════╗\n" + "║ SECURITY WARNING: ║\n" + "║ unauthenticated action API exposed to the network ║\n" + f"║ web_ui.host = {host!r:<54s} ║\n" + "║ Anyone who can reach this port can observe the screen and inject ║\n" + "║ clicks/keystrokes via /api/action. Only use a non-loopback bind ║\n" + "║ inside an isolated container/VM or behind a firewall or ║\n" + "║ authenticating reverse proxy. ║\n" + "╚══════════════════════════════════════════════════════════════════════╝" + ) + + def setup_logging(config: dict) -> None: level_name = config.get("logging", {}).get("level", "INFO").upper() level = getattr(logging, level_name, logging.INFO) @@ -366,6 +396,16 @@ def main() -> None: host = config["web_ui"]["host"] port = config["web_ui"]["port"] + + warning = bind_warning(host) + if warning: + print(warning, file=sys.stderr) + logger.warning( + "[main] unauthenticated action API exposed to the network " + f"(web_ui.host={host!r}); this is an explicit opt-in — see " + "README 'Security & Network Bind Defaults'" + ) + app = create_web_app(observer, renderer, describer, config) def _run_flask(): diff --git a/tests/test_security_defaults.py b/tests/test_security_defaults.py new file mode 100644 index 0000000..15dc863 --- /dev/null +++ b/tests/test_security_defaults.py @@ -0,0 +1,140 @@ +"""[P2] Secure-defaults tests: loopback bind default, opt-in remote bind, +and config-driven CORS behavior. + +The HTTP API is unauthenticated by design (no auth is added — binding +policy is the security control): + - built-in default bind must be loopback (127.0.0.1); + - --host / web_ui.host remain the explicit opt-in path to 0.0.0.0; + - non-loopback binds must produce a prominent startup warning; + - no CORS headers are sent unless web_ui.cors_origins is configured. +""" +from __future__ import annotations + +import copy +import json + +import main as _main +from main import _DEFAULT_CONFIG, bind_warning, load_config + + +# ── Bind default resolution ────────────────────────────────────────────────── + + +def test_builtin_default_bind_is_loopback(): + assert _DEFAULT_CONFIG["web_ui"]["host"] == "127.0.0.1" + + +def test_builtin_default_cors_origins_empty(): + assert _DEFAULT_CONFIG["web_ui"]["cors_origins"] == [] + + +def test_load_config_without_host_key_resolves_loopback(tmp_path): + p = tmp_path / "config.json" + p.write_text(json.dumps({"web_ui": {"port": 6001}})) + cfg = load_config(str(p)) + assert cfg["web_ui"]["host"] == "127.0.0.1" + assert cfg["web_ui"]["port"] == 6001 + + +def test_load_config_explicit_opt_in_preserved(tmp_path): + p = tmp_path / "config.json" + p.write_text(json.dumps({"web_ui": {"host": "0.0.0.0"}})) + cfg = load_config(str(p)) + assert cfg["web_ui"]["host"] == "0.0.0.0" + + +def test_example_config_binds_loopback(tmp_path): + # config.json.example is the bootstrap source of truth for first runs. + import os + example = os.path.join(os.path.dirname(_main.__file__), + "config.json.example") + with open(example, encoding="utf-8") as f: + cfg = json.load(f) + assert cfg["web_ui"]["host"] == "127.0.0.1" + assert cfg["web_ui"]["cors_origins"] == [] + + +# ── Non-loopback bind warning ──────────────────────────────────────────────── + + +def test_bind_warning_none_for_loopback(): + assert bind_warning("127.0.0.1") is None + assert bind_warning("localhost") is None + assert bind_warning("::1") is None + + +def test_bind_warning_for_all_interfaces(): + w = bind_warning("0.0.0.0") + assert w is not None + assert "unauthenticated action API exposed to the network" in w + + +def test_bind_warning_for_lan_address(): + w = bind_warning("192.168.1.20") + assert w is not None + assert "192.168.1.20" in w + + +# ── CORS behavior via Flask test client ────────────────────────────────────── + + +def _make_client(cors_origins=None): + from ascii_renderer import ASCIIRenderer + from description import DescriptionGenerator + from observer import ScreenObserver + from web_inspector import create_web_app + + cfg = copy.deepcopy(_DEFAULT_CONFIG) + cfg["mock"] = True + if cors_origins is not None: + cfg["web_ui"]["cors_origins"] = cors_origins + app = create_web_app(ScreenObserver(cfg), ASCIIRenderer(cfg), + DescriptionGenerator(cfg), cfg) + return app.test_client() + + +def test_no_cors_headers_by_default(fresh_session): + client = _make_client() + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + assert r.status_code == 200 + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_no_cors_on_action_by_default(fresh_session): + client = _make_client() + r = client.post("/api/action", + json={"action": "click_at", "x": 10, "y": 10}, + headers={"Origin": "http://evil.example"}) + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_no_cors_preflight_grant_on_action_by_default(fresh_session): + client = _make_client() + r = client.options( + "/api/action", + headers={"Origin": "http://evil.example", + "Access-Control-Request-Method": "POST"}) + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_cors_wildcard_opt_in(fresh_session): + client = _make_client(cors_origins=["*"]) + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + # flask-cors echoes the requesting origin for a "*" allowlist; either + # form grants cross-origin access. + assert r.headers.get("Access-Control-Allow-Origin") in ( + "*", "http://evil.example") + + +def test_cors_explicit_origin_allowed(fresh_session): + client = _make_client(cors_origins=["http://localhost:3000"]) + r = client.get("/api/windows", + headers={"Origin": "http://localhost:3000"}) + assert (r.headers.get("Access-Control-Allow-Origin") + == "http://localhost:3000") + + +def test_cors_explicit_origin_rejects_others(fresh_session): + client = _make_client(cors_origins=["http://localhost:3000"]) + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + assert "Access-Control-Allow-Origin" not in r.headers diff --git a/web_inspector.py b/web_inspector.py index ed3ef9b..fda5244 100755 --- a/web_inspector.py +++ b/web_inspector.py @@ -13,6 +13,9 @@ The HTML/CSS/JS is inlined as a template string so the entire server is a single importable Python module with no external static files. + +CORS policy: no CORS headers are emitted unless web_ui.cors_origins is set +in config (default [] = same-origin only). See create_web_app(). """ import base64 @@ -1187,7 +1190,18 @@ def create_web_app( shared observer/renderer/describer instances. """ app = Flask(__name__) - CORS(app) + + # CORS is opt-in (web_ui.cors_origins, default []). With the default no + # Access-Control-Allow-Origin header is ever sent, so browsers enforce + # same-origin — the bundled inspector UI at "/" is served same-origin and + # keeps working. Operators can list explicit origins, or ["*"] for + # Docker/testing scenarios where cross-origin dashboards need access. + # Never enable "*" together with a non-loopback bind outside an isolated + # environment: /api/action is unauthenticated and destructive. + cors_origins = list((config.get("web_ui") or {}).get("cors_origins") or []) + if cors_origins: + CORS(app, origins=cors_origins) + logger.warning(f"CORS enabled for origins: {cors_origins}") ctx = _tools.ToolContext(observer=observer, renderer=renderer, describer=describer, config=config) From b50ccd6d680607cc234c929f0508e54ec3a7eb02 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:22:46 +0000 Subject: [PATCH 07/14] [P2] untrusted-content marking + confirmation-token audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - redaction.py: new trust-boundary layer. Results of the nine screen-text-carrying tools (list_windows, find_element, get_window_structure, observe_window, wait_for, snapshot_get, snapshot_diff, get_ocr, get_screen_description) are flagged `untrusted: true` and all extracted strings are stripped of ANSI escape sequences (CSI/OSC/2-byte) and non-whitespace control characters. Opaque fields (base64 data, tokens) are skipped. Wired unconditionally into tools.dispatch (independent of the opt-in redactor). README gains a "Trust Boundary: Screen Content Is Untrusted" section documenting the prompt-injection contract for MCP clients. - §21 confirmation audit: every element-targeted destructive verb (click/focus/set_value/invoke/select/hover/right_click/double_click/ key_into/clear_text/click_element_and_observe) already routed through _check_confirmation via _do_element_action. ONE unprotected verb found: `drag` resolved elements from selector/element_id endpoints but never consulted the gate. Fixed — a confirmation_required rule matching either endpoint now demands a confirm_token (propose_action(action="drag", args={selector}) issues it). Coordinate/global verbs (click_at, type_text, press_key, scroll, hover_at, bring_to_foreground + aliases/composites) carry no element identity so §21 role/name rules cannot address them — documented exclusion, with a completeness test asserting every input tool is in exactly one group. - REST<->MCP<->dispatch parity test (design doc §27): asserts every REGISTRY tool has an MCP schema (documented exclusions: right_click_at/double_click_at aliases), every MCP schema is dispatchable (legacy-only: get_screen_sketch/get_full_screenshot), every tool has a dedicated REST route registered in the Flask url_map, /api/tools enumerates the registry exactly, and each tool answers a structured envelope through the generic console. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 30 +++ redaction.py | 77 +++++++- tests/test_surface_parity.py | 176 +++++++++++++++++ tests/test_untrusted_and_confirmation.py | 231 +++++++++++++++++++++++ tools.py | 52 ++++- 5 files changed, 559 insertions(+), 7 deletions(-) create mode 100644 tests/test_surface_parity.py create mode 100644 tests/test_untrusted_and_confirmation.py diff --git a/README.md b/README.md index ace662f..5ab011c 100644 --- a/README.md +++ b/README.md @@ -417,6 +417,36 @@ in the tools menu. You can then ask Claude to: --- +## Trust Boundary: Screen Content Is Untrusted + +Everything OSScreenObserver reads off the screen is **attacker-influenced +input**: window titles, accessibility-tree element names and values, OCR +words, and VLM descriptions all come from whatever happens to be displayed +— a web page, email, or document on screen can deliberately contain +prompt-injection text ("ignore your previous instructions and …"). + +The server marks and sanitizes this data so clients can handle it safely: + +- Results of screen-text-carrying tools (`list_windows`, `find_element`, + `get_window_structure`, `observe_window`, `wait_for`, `snapshot_get`, + `snapshot_diff`, `get_ocr`, `get_screen_description`) carry a top-level + **`untrusted: true`** flag. +- ANSI escape sequences and non-whitespace control characters are stripped + from extracted text, so screen content cannot smuggle terminal escapes + into logs or client UIs. + +**Guidance for MCP clients / agents:** treat every `untrusted: true` field +as *data, never instructions*. Text found on screen must not change the +agent's goals, unlock destructive actions, or be executed/evaluated. Pair +this with the confirmation-token flow (`propose_action` + +`confirmation_required` rules) so that even a successfully injected "click +Delete" suggestion still requires an explicit out-of-band token before the +destructive verb executes. Note that screenshots (base64 PNGs) are equally +attacker-influenced; they are not flagged per-pixel, so the same rule +applies to anything a VLM extracts from them. + +--- + ## REST API Reference The web inspector exposes the following endpoints (all `GET` unless noted): diff --git a/redaction.py b/redaction.py index bc72774..3976c8a 100644 --- a/redaction.py +++ b/redaction.py @@ -1,8 +1,17 @@ """ -redaction.py — Sensitive-region redaction (design doc §18, D7). +redaction.py — Sensitive-region redaction (design doc §18, D7) and +untrusted-screen-content marking (P2 trust boundary). Default: tree node names/values + OCR text + VLM preamble are scrubbed. Opt-in: redaction.blur_screenshots paints matched bboxes solid black. + +Trust boundary: everything read off the screen (window titles, element +names/values, OCR words, VLM descriptions) is attacker-influenced data — +a web page or document on screen can contain prompt-injection text. +``mark_untrusted`` flags the results of screen-text-carrying tools with +``untrusted: true`` and strips ANSI escape sequences / control characters +so extracted text cannot smuggle terminal escapes to downstream consumers. +MCP clients must treat these fields as data, never as instructions. """ from __future__ import annotations @@ -17,6 +26,72 @@ DEFAULT_REPLACEMENT = "[REDACTED]" +# ─── Untrusted screen content (P2 trust boundary) ──────────────────────────── + +# CSI (ESC [ … cmd), OSC (ESC ] … BEL/ST), and single-char escapes. +_ANSI_ESCAPE_RE = re.compile( + r"\x1b(?:\[[0-9;?]*[ -/]*[@-~]" # CSI sequences + r"|\][^\x07\x1b]*(?:\x07|\x1b\\)?" # OSC sequences + r"|[@-Z\\-_])" # 2-byte escapes (incl. lone ESC-x) +) +# C0/C1 control characters except \t \n \r (layout-preserving whitespace). +_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + +# Tool results that carry text extracted from the screen (window titles, +# element names/values, OCR words, descriptions). Their responses are +# flagged ``untrusted: true`` and sanitized. Action receipts also embed +# screen-derived selectors, but flagging every action result would drown +# the signal; perception results are where injection text actually lands. +UNTRUSTED_RESULT_TOOLS = frozenset({ + "list_windows", + "find_element", + "get_window_structure", + "observe_window", + "wait_for", + "snapshot_get", + "snapshot_diff", + "get_ocr", + "get_screen_description", +}) + +# Keys whose values are machine artifacts (base64 blobs, opaque tokens) — +# skipped during sanitization for performance and fidelity. +_SANITIZE_SKIP_KEYS = frozenset({ + "data", "screenshot_b64", "tree_token", "base_token", + "snapshot_id", "confirm_token", +}) + + +def sanitize_screen_text(text: str) -> str: + """Strip ANSI escape sequences and non-whitespace control characters + from screen-extracted text. Idempotent; returns non-str input as-is.""" + if not isinstance(text, str) or not text: + return text + text = _ANSI_ESCAPE_RE.sub("", text) + return _CONTROL_CHARS_RE.sub("", text) + + +def _sanitize_value(value: Any, key: str = "") -> Any: + if isinstance(value, str): + return sanitize_screen_text(value) + if isinstance(value, dict): + return {k: (v if k in _SANITIZE_SKIP_KEYS else _sanitize_value(v, k)) + for k, v in value.items()} + if isinstance(value, list): + return [_sanitize_value(v) for v in value] + return value + + +def mark_untrusted(tool: str, result: Any) -> Any: + """Flag screen-text-carrying tool results as untrusted and sanitize + their string payloads. No-op for tools that return no screen text.""" + if tool not in UNTRUSTED_RESULT_TOOLS or not isinstance(result, dict): + return result + out = _sanitize_value(result) + out["untrusted"] = True + return out + + class Redactor: def __init__(self, config: Dict[str, Any]) -> None: cfg = (config.get("redaction") or {}) diff --git a/tests/test_surface_parity.py b/tests/test_surface_parity.py new file mode 100644 index 0000000..1b462c1 --- /dev/null +++ b/tests/test_surface_parity.py @@ -0,0 +1,176 @@ +"""[P2] REST ↔ MCP ↔ dispatch parity (design doc §27). + +Every tool in tools.REGISTRY must be reachable with the same parameter +surface from both public surfaces: + + - REST: the generic tool console (POST /api/tool/) forwards the + JSON body verbatim to tools.dispatch, and GET /api/tools enumerates + the registry. Most tools additionally have a dedicated route. + - MCP: mcp_server advertises a schema in _TOOLS and forwards + `arguments` verbatim to tools.dispatch for every REGISTRY tool. + +Exceptions are explicit, documented allowlists below — any new tool that +is added to one surface but not the others fails these tests. +""" +from __future__ import annotations + +import pytest + +import mcp_server as _mcp +import tools as _tools + +# REGISTRY tools with no dedicated MCP schema: pure aliases of click_at — +# MCP callers pass button="right" / double=true to click_at instead. +MCP_SCHEMA_EXCLUSIONS = { + "right_click_at", + "double_click_at", +} + +# MCP tools handled by legacy composite handlers inside mcp_server._dispatch +# (not part of tools.REGISTRY); their REST equivalents are /api/sketch and +# /api/full_screenshot. +MCP_LEGACY_ONLY = { + "get_screen_sketch", + "get_full_screenshot", +} + +# REGISTRY tools with no dedicated REST route (reachable via the generic +# console only): pure aliases of click_at — REST callers POST /api/action +# with {"action": "click_at", "button": "right"} / {"double": true}. +REST_DEDICATED_ROUTE_EXCLUSIONS = { + "right_click_at", + "double_click_at", +} + +# tool name -> dedicated REST route (path prefix as registered in Flask). +REST_ROUTES = { + "list_windows": "/api/windows", + "get_capabilities": "/api/capabilities", + "get_monitors": "/api/monitors", + "find_element": "/api/find_element", + "get_window_structure": "/api/structure", + "get_screenshot": "/api/screenshot", + "get_visible_areas": "/api/visible_areas", + "click_element": "/api/element/click", + "focus_element": "/api/element/focus", + "set_value": "/api/element/set_value", + "invoke_element": "/api/element/invoke", + "select_option": "/api/element/select", + "click_at": "/api/action", + "type_text": "/api/action", + "press_key": "/api/action", + "scroll": "/api/action", + "bring_to_foreground": "/api/bring_to_foreground", + "observe_window": "/api/observe", + "snapshot": "/api/snapshot", + "snapshot_get": "/api/snapshot/", + "snapshot_diff": "/api/snapshot/diff", + "snapshot_drop": "/api/snapshot/", + "wait_for": "/api/wait_for", + "wait_idle": "/api/wait_idle", + "click_element_and_observe": "/api/element/click_and_observe", + "type_and_observe": "/api/type_and_observe", + "press_key_and_observe": "/api/key_and_observe", + "get_screenshot_cropped": "/api/screenshot/cropped", + "get_ocr": "/api/ocr", + "get_screen_description": "/api/description", + "trace_start": "/api/trace/start", + "trace_stop": "/api/trace/stop", + "trace_status": "/api/trace/status", + "replay_start": "/api/replay/start", + "replay_step": "/api/replay/step", + "replay_status": "/api/replay/status", + "replay_stop": "/api/replay/stop", + "load_scenario": "/api/scenario/load", + "assert_state": "/api/assert_state", + "get_budget_status": "/api/budget_status", + "get_redaction_status": "/api/redaction_status", + "propose_action": "/api/propose_action", + "hover_at": "/api/hover", + "hover_element": "/api/hover", + "right_click_element": "/api/element/right_click", + "double_click_element": "/api/element/double_click", + "drag": "/api/drag", + "key_into_element": "/api/element/key", + "clear_text": "/api/element/clear_text", +} + +# Tools not invoked in the smoke loop: session-stateful (an active trace +# would then record every subsequent call in this process's global session). +SMOKE_SKIP = {"trace_start", "trace_stop"} + +# Minimal args so no tool blocks (waits) or errors on missing input in a +# way unrelated to reachability. +SMOKE_ARGS = { + "wait_for": {"timeout_ms": 20, "predicate": {"kind": "window_exists", + "title_regex": "."}}, + "wait_idle": {"timeout_ms": 20, "quiet_ms": 5}, + "hover_at": {"hover_ms": 1}, + "hover_element": {"hover_ms": 1, + "selector": 'MenuItem[name="File"]'}, +} + + +def test_mcp_advertises_every_registry_tool(): + mcp_names = {t["name"] for t in _mcp._TOOLS} + missing = set(_tools.REGISTRY) - mcp_names - MCP_SCHEMA_EXCLUSIONS + assert not missing, f"REGISTRY tools missing an MCP schema: {sorted(missing)}" + + +def test_mcp_schema_exclusions_are_not_advertised_stale(): + # Keep the exclusion list honest: everything on it must exist in the + # REGISTRY and must genuinely lack an MCP schema. + mcp_names = {t["name"] for t in _mcp._TOOLS} + for name in MCP_SCHEMA_EXCLUSIONS: + assert name in _tools.REGISTRY + assert name not in mcp_names + + +def test_every_mcp_tool_is_dispatchable(): + mcp_names = {t["name"] for t in _mcp._TOOLS} + unknown = mcp_names - set(_tools.REGISTRY) - MCP_LEGACY_ONLY + assert not unknown, f"MCP schemas with no dispatch target: {sorted(unknown)}" + + +def test_mcp_schemas_have_object_parameter_surface(): + # MCP forwards `arguments` verbatim to tools.dispatch, so every schema + # must declare an object inputSchema (same dict-shaped surface REST + # forwards from the JSON body). + for t in _mcp._TOOLS: + schema = t.get("inputSchema") or {} + assert schema.get("type") == "object", t["name"] + assert isinstance(schema.get("properties", {}), dict), t["name"] + + +def test_rest_console_enumerates_registry(client): + data = client.get("/api/tools").get_json() + assert data["ok"] is True + assert data["tools"] == sorted(_tools.REGISTRY) + + +def test_every_registry_tool_has_dedicated_rest_route(app): + rules = {r.rule for r in app.url_map.iter_rules()} + for name in _tools.REGISTRY: + if name in REST_DEDICATED_ROUTE_EXCLUSIONS: + assert name not in REST_ROUTES + continue + assert name in REST_ROUTES, f"no documented REST route for {name}" + assert REST_ROUTES[name] in rules, ( + f"{name}: documented route {REST_ROUTES[name]} not registered") + + +@pytest.mark.parametrize("name", sorted(_tools.REGISTRY)) +def test_rest_console_reaches_tool(client, name): + """POST /api/tool/ must reach dispatch for every REGISTRY tool + (a structured envelope comes back — never Flask's 404/405 or the + dispatcher's 'unknown tool' error).""" + if name in SMOKE_SKIP: + pytest.skip("session-stateful; reachability covered by url_map test") + r = client.post(f"/api/tool/{name}", json=SMOKE_ARGS.get(name, {})) + assert r.status_code != 404 or r.get_json() is not None + data = r.get_json() + assert isinstance(data, dict), name + assert "ok" in data, name + if data["ok"] is False: + msg = (data.get("error") or {}).get("message", "") + assert not msg.startswith("unknown tool"), name diff --git a/tests/test_untrusted_and_confirmation.py b/tests/test_untrusted_and_confirmation.py new file mode 100644 index 0000000..76cfc2a --- /dev/null +++ b/tests/test_untrusted_and_confirmation.py @@ -0,0 +1,231 @@ +"""[P2] Trust-boundary tests: untrusted-content marking on screen-derived +results, ANSI/control-character sanitization, and the design-doc §21 +confirmation-token audit (every destructive element-targeted verb must +route through the confirmation gate when confirmation mode is enabled). +""" +from __future__ import annotations + +import pytest + +import tools as _tools +from redaction import ( + UNTRUSTED_RESULT_TOOLS, mark_untrusted, sanitize_screen_text, +) + + +# ── sanitize_screen_text ───────────────────────────────────────────────────── + + +def test_sanitize_strips_csi_sequences(): + assert sanitize_screen_text("a\x1b[31mred\x1b[0mb") == "aredb" + + +def test_sanitize_strips_osc_sequences(): + assert sanitize_screen_text("x\x1b]0;evil title\x07y") == "xy" + + +def test_sanitize_strips_control_chars(): + assert sanitize_screen_text("a\x00b\x08c\x7fd") == "abcd" + + +def test_sanitize_keeps_layout_whitespace(): + assert sanitize_screen_text("line1\nline2\tcol\r") == "line1\nline2\tcol\r" + + +def test_sanitize_non_string_passthrough(): + assert sanitize_screen_text(None) is None + assert sanitize_screen_text(42) == 42 + assert sanitize_screen_text("") == "" + + +# ── mark_untrusted ─────────────────────────────────────────────────────────── + + +def test_mark_untrusted_flags_and_sanitizes_nested_text(): + result = { + "ok": True, + "tree": {"name": "evil\x1b[2Jname", "value": "v\x00v", + "children": [{"name": "child\x1b[31m"}]}, + "windows": [{"title": "T\x07itle"}], + } + out = mark_untrusted("get_window_structure", result) + assert out["untrusted"] is True + assert out["tree"]["name"] == "evilname" + assert out["tree"]["value"] == "vv" + assert out["tree"]["children"][0]["name"] == "child" + assert out["windows"][0]["title"] == "Title" + + +def test_mark_untrusted_skips_opaque_fields(): + result = {"ok": True, "data": "base64\x1b[31mstuff", + "tree_token": "tt:ab\x00cd"} + out = mark_untrusted("get_ocr", result) + assert out["data"] == "base64\x1b[31mstuff" # untouched + assert out["tree_token"] == "tt:ab\x00cd" # untouched + assert out["untrusted"] is True + + +def test_mark_untrusted_noop_for_action_tools(): + result = {"ok": True, "action": "click_element"} + out = mark_untrusted("click_element", result) + assert "untrusted" not in out + + +def test_untrusted_tool_list_is_read_tools_only(): + # No destructive verb should be in the untrusted-marking list; + # perception tools carry the screen text. + for name in UNTRUSTED_RESULT_TOOLS: + assert not _tools._is_input_tool(name) + + +# ── Wire-through: REST results carry the flag ──────────────────────────────── + + +def test_windows_result_is_untrusted(client): + data = client.get("/api/windows").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_structure_result_is_untrusted(client): + data = client.get("/api/structure").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_observe_result_is_untrusted(client): + data = client.get("/api/observe").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_action_receipt_not_flagged(client): + ws = client.get("/api/windows").get_json() + uid = ws["windows"][0]["window_uid"] + r = client.post("/api/element/click", json={ + "window_uid": uid, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }).get_json() + assert r["ok"] is True + assert "untrusted" not in r + + +# ── §21 confirmation audit ─────────────────────────────────────────────────── +# +# Every destructive verb that resolves a concrete element target must route +# through _check_confirmation when confirmation_required rules are set. +# +# Coordinate/global verbs carry no element identity, so the role/name-based +# rules of §21 cannot apply to them by construction. They are the documented +# exclusion list below; the completeness test asserts every input tool is +# accounted for in exactly one of the two groups. + +ELEMENT_VERBS = [ + ("click_element", {}), + ("focus_element", {}), + ("set_value", {"value": "x"}), + ("invoke_element", {}), + ("select_option", {"option_name": "x"}), + ("hover_element", {"hover_ms": 1}), + ("right_click_element", {}), + ("double_click_element", {}), + ("key_into_element", {"keys": "ctrl+a"}), + ("clear_text", {}), + ("click_element_and_observe", {}), +] + +# §21 scope limit: rules match element role/name; these verbs act on raw +# coordinates / global input focus and never resolve an element, so no +# rule can name them a destructive target. (click_element_and_observe is +# gated via click_element; type/press_key composites wrap the excluded +# global verbs.) `drag` IS gated when either endpoint is element-addressed. +COORDINATE_VERB_EXCLUSIONS = { + "click_at", "right_click_at", "double_click_at", + "type_text", "press_key", "scroll", + "hover_at", "bring_to_foreground", + "type_and_observe", "press_key_and_observe", +} + + +@pytest.fixture() +def confirm_ctx(config, observer, renderer, describer): + config["confirmation_required"] = [{"name_regex": "(?i)file"}] + return _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + +def _file_menu_args(ctx): + win = ctx.observer.list_windows()[0] + return {"window_uid": win.window_uid, + "selector": 'Window/MenuBar/MenuItem[name="File"]'} + + +@pytest.mark.parametrize("verb,extra", ELEMENT_VERBS) +def test_element_verb_requires_confirmation(confirm_ctx, verb, extra): + args = {**_file_menu_args(confirm_ctx), **extra} + r = _tools.dispatch(confirm_ctx, verb, args) + assert r["ok"] is False, verb + assert r["error"]["code"] == "ConfirmationRequired", verb + + +def test_drag_requires_confirmation_for_element_endpoint(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationRequired" + + +def test_drag_requires_confirmation_for_destination_endpoint(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"x": 100, "y": 100}, + "to": {"selector": base["selector"]}, + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationRequired" + + +def test_drag_with_valid_token_passes_gate(confirm_ctx): + base = _file_menu_args(confirm_ctx) + prop = _tools.dispatch(confirm_ctx, "propose_action", { + "action": "drag", + "args": base, + }) + assert prop["ok"] is True + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + "confirm_token": prop["confirm_token"], + }) + # The gate is satisfied; the drag itself may still fail on headless CI + # (pyautogui unavailable) — but never with a confirmation error. + if r["ok"] is False: + assert r["error"]["code"] not in ("ConfirmationRequired", + "ConfirmationInvalid") + + +def test_drag_with_bogus_token_rejected(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + "confirm_token": "ct:doesnotexist", + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationInvalid" + + +def test_confirmation_audit_covers_every_input_tool(): + gated = {v for v, _ in ELEMENT_VERBS} | {"drag"} + accounted = gated | COORDINATE_VERB_EXCLUSIONS + input_tools = {n for n in _tools.REGISTRY if _tools._is_input_tool(n)} + unaccounted = input_tools - accounted + assert not unaccounted, ( + f"input tools missing from the §21 audit: {sorted(unaccounted)}") diff --git a/tools.py b/tools.py index 2c90164..0327b1d 100644 --- a/tools.py +++ b/tools.py @@ -27,6 +27,7 @@ from observer import ( ScreenObserver, UIElement, WindowInfo, WindowResolution, ) +from redaction import mark_untrusted from session import Session, get_session logger = logging.getLogger(__name__) @@ -371,13 +372,13 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An ) -def _check_confirmation(ctx: ToolContext, action_name: str, - args: Dict[str, Any], - target: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Returns an error dict to short-circuit when confirmation is required.""" +def _confirmation_rules_match(ctx: ToolContext, + target: Dict[str, Any]) -> bool: + """True when any configured confirmation_required rule matches the + target's role/name (derived from the selector tail).""" confirm = ctx.config.get("confirmation_required") or [] if not confirm: - return None + return False name_to_test = "" role_to_test = "" # Best-effort: derive name/role from the selector tail. @@ -398,7 +399,14 @@ def _matches_rule(rule: Dict[str, Any]) -> bool: return False return bool(rname or rrole) - if not any(_matches_rule(r) for r in confirm): + return any(_matches_rule(r) for r in confirm) + + +def _check_confirmation(ctx: ToolContext, action_name: str, + args: Dict[str, Any], + target: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Returns an error dict to short-circuit when confirmation is required.""" + if not _confirmation_rules_match(ctx, target): return None token = args.get("confirm_token") @@ -2080,6 +2088,30 @@ def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: return error_dict(Code.BAD_REQUEST, "drag requires from/to as {x,y} or {selector|element_id}", step_id=step_id) + + # §21 confirmation gate: drag endpoints addressed by selector/element_id + # resolve to concrete elements, so a confirmation_required rule matching + # either endpoint (e.g. a "Trash" drop target) must demand a token, just + # like the other element-targeted verbs. The first matching endpoint is + # validated (propose_action(action="drag", args={"selector": …}) issues + # the token for that endpoint's selector). + if tree is not None and info is not None: + for spec in (src, dst): + if not (spec.get("selector") or spec.get("element_id")): + continue + elem, selector_str, err = _resolve_element(tree, spec) + if err or elem is None: + continue + tgt = {"window_uid": info.window_uid, + "element_id": elem.element_id, + "selector": selector_str, + "bounds": elem.bounds.to_dict()} + if _confirmation_rules_match(ctx, tgt): + confirm_check = _check_confirmation(ctx, "drag", args, tgt) + if confirm_check is not None: + return {**confirm_check, "step_id": step_id} + break # token validated for the matching endpoint + duration = float(args.get("duration_s", 0.5)) path = [p1, ((p1[0] + p2[0]) // 2, (p1[1] + p2[1]) // 2), p2] try: @@ -2282,6 +2314,14 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any except Exception: logger.exception("redaction failed") + # Untrusted-content marking (always on): screen-derived text is + # attacker-influenced data (prompt injection), never instructions. + # Flag it and strip ANSI/control characters. + try: + result = mark_untrusted(name, result) + except Exception: + logger.exception("untrusted-content marking failed") + # Budget accounting. if sess.budgets is not None: try: From 44812792129f660115419031704b808e6ebcc511 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:26:10 +0000 Subject: [PATCH 08/14] [P2] healthz/metrics telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tree_cache.py: session-cumulative counters — cache hits/misses, entry count, and a capture-latency summary (count/total/max/mean ms) fed by every put(); exposed via TreeCache.counters(). - /api/metrics (Prometheus text format kept): new oso_tree_cache_hits_total / oso_tree_cache_misses_total counters, oso_tree_cache_entries gauge, oso_tree_capture_ms summary (_sum/_count) + oso_tree_capture_ms_max gauge, alongside the existing step/uptime/budget/trace series. - /api/healthz (verified real + made cheap): now reports the same tree_cache counter block; the OCR diagnostic (which spawns a `tesseract --version` subprocess) is computed once per process and cached so the endpoint is safe to poll; swallowed diagnostics failures now logger.debug instead of silent pass. - README endpoint quick-reference updated for both endpoints. - Tests: healthz counter block + hit/miss movement + step-count increment + diagnose-once cheapness; metrics content-type, all new series present with HELP/TYPE lines, and counters move after a cold+warm structure read. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- README.md | 4 +- tests/test_telemetry.py | 97 +++++++++++++++++++++++++++++++++++++++++ tree_cache.py | 30 +++++++++++++ web_inspector.py | 44 ++++++++++++++++--- 4 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 tests/test_telemetry.py diff --git a/README.md b/README.md index 5ab011c..144e207 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ curl http://127.0.0.1:5001/api/healthz | `GET` | `/api/screenshot` | Base64-encoded PNG screenshot | | `POST` | `/api/action` | Execute click, type, key, or scroll action | | `GET` | `/api/capabilities` | Server capabilities and platform info | -| `GET` | `/api/healthz` | Health and uptime | -| `GET` | `/api/metrics` | Prometheus-format metrics (`text/plain`) | +| `GET` | `/api/healthz` | Health, uptime, step count, tree-cache hit/miss + capture-latency telemetry, config/OCR diagnostics (cheap to poll — the OCR probe is computed once per process) | +| `GET` | `/api/metrics` | Prometheus-format metrics (`text/plain`): `oso_step_count`, `oso_uptime_seconds`, `oso_tree_cache_hits_total` / `oso_tree_cache_misses_total` / `oso_tree_cache_entries`, `oso_tree_capture_ms` summary (+`_max`), budget counters when configured, `oso_active_trace` | ### Example workflow diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 0000000..1f8dc41 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,97 @@ +"""[P2] Telemetry tests: /api/healthz and /api/metrics expose step counts, +tree-cache hit/miss counters, and a capture-latency summary.""" +from __future__ import annotations + +import re + + +def _metric(body: str, name: str) -> float: + m = re.search(rf"^{re.escape(name)} ([0-9.]+)$", body, re.M) + assert m, f"metric {name} not found in:\n{body}" + return float(m.group(1)) + + +# ── /api/healthz ───────────────────────────────────────────────────────────── + + +def test_healthz_includes_tree_cache_counters(client): + data = client.get("/api/healthz").get_json() + assert data["ok"] is True + tc = data["tree_cache"] + for key in ("hits", "misses", "entries", "capture_count", + "capture_ms_total", "capture_ms_max", "capture_ms_mean"): + assert key in tc, key + + +def test_healthz_tree_cache_counts_hits_and_misses(client): + client.get("/api/structure") # cold: miss + capture + client.get("/api/structure") # warm (TTL 2s): hit + tc = client.get("/api/healthz").get_json()["tree_cache"] + assert tc["misses"] >= 1 + assert tc["hits"] >= 1 + assert tc["capture_count"] >= 1 + + +def test_healthz_step_count_increments(client): + before = client.get("/api/healthz").get_json()["step_count"] + client.get("/api/windows") + after = client.get("/api/healthz").get_json()["step_count"] + assert after == before + 1 + + +def test_healthz_ocr_diagnostic_computed_once(client, monkeypatch): + """healthz must stay cheap: the tesseract probe (subprocess spawn) runs + at most once per process, not per poll.""" + import ocr_util + calls = {"n": 0} + real = ocr_util.diagnose + + def counting_diagnose(cfg): + calls["n"] += 1 + return real(cfg) + + monkeypatch.setattr(ocr_util, "diagnose", counting_diagnose) + r1 = client.get("/api/healthz") + r2 = client.get("/api/healthz") + assert r1.status_code == 200 and r2.status_code == 200 + assert calls["n"] <= 1 + + +# ── /api/metrics ───────────────────────────────────────────────────────────── + + +def test_metrics_prometheus_content_type(client): + r = client.get("/api/metrics") + assert r.status_code == 200 + assert r.content_type.startswith("text/plain") + + +def test_metrics_exposes_tree_cache_and_latency(client): + body = client.get("/api/metrics").get_data(as_text=True) + for name in ("oso_step_count", + "oso_uptime_seconds", + "oso_tree_cache_hits_total", + "oso_tree_cache_misses_total", + "oso_tree_cache_entries", + "oso_tree_capture_ms_sum", + "oso_tree_capture_ms_count", + "oso_tree_capture_ms_max"): + _metric(body, name) + + +def test_metrics_cache_counters_move(client): + client.get("/api/structure") # miss + capture + client.get("/api/structure") # hit + body = client.get("/api/metrics").get_data(as_text=True) + assert _metric(body, "oso_tree_cache_misses_total") >= 1 + assert _metric(body, "oso_tree_cache_hits_total") >= 1 + assert _metric(body, "oso_tree_capture_ms_count") >= 1 + assert _metric(body, "oso_tree_cache_entries") >= 1 + + +def test_metrics_help_and_type_lines(client): + body = client.get("/api/metrics").get_data(as_text=True) + assert "# TYPE oso_tree_cache_hits_total counter" in body + assert "# TYPE oso_tree_cache_misses_total counter" in body + assert "# TYPE oso_tree_cache_entries gauge" in body + assert "# TYPE oso_tree_capture_ms summary" in body diff --git a/tree_cache.py b/tree_cache.py index c9d4a4a..eafa0fb 100644 --- a/tree_cache.py +++ b/tree_cache.py @@ -62,6 +62,12 @@ def __init__(self, ttl_s: float = DEFAULT_TTL_S, # Last-capture stats per window; kept across invalidation so # capability reporting works even when the cache is cold. self._stats: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + # Telemetry counters (P2): cache effectiveness + capture latency. + self._hits = 0 + self._misses = 0 + self._capture_count = 0 + self._capture_ms_total = 0 + self._capture_ms_max = 0 # ── Read ───────────────────────────────────────────────────────────────── @@ -73,11 +79,14 @@ def get(self, window_uid: str, with self._lock: entry = self._entries.get(window_uid) if entry is None: + self._misses += 1 return None if entry.age_s() > limit: self._entries.pop(window_uid, None) + self._misses += 1 return None self._entries.move_to_end(window_uid) + self._hits += 1 return entry def peek(self, window_uid: str) -> Optional[TreeCacheEntry]: @@ -99,6 +108,10 @@ def put(self, window_uid: str, *, tree: Any, serialized: Dict[str, Any], named_node_count=int(named_node_count), ) with self._lock: + self._capture_count += 1 + self._capture_ms_total += entry.capture_ms + if entry.capture_ms > self._capture_ms_max: + self._capture_ms_max = entry.capture_ms self._entries[window_uid] = entry self._entries.move_to_end(window_uid) while len(self._entries) > self.max_windows: @@ -140,6 +153,23 @@ def stats(self) -> Dict[str, Dict[str, Any]]: with self._lock: return {uid: dict(s) for uid, s in self._stats.items()} + def counters(self) -> Dict[str, Any]: + """Session-cumulative telemetry: cache hit/miss counts and a + capture-latency summary (count / total / max / mean ms). Surfaced + by /api/healthz and /api/metrics.""" + with self._lock: + mean = (self._capture_ms_total / self._capture_count + if self._capture_count else 0.0) + return { + "hits": self._hits, + "misses": self._misses, + "entries": len(self._entries), + "capture_count": self._capture_count, + "capture_ms_total": self._capture_ms_total, + "capture_ms_max": self._capture_ms_max, + "capture_ms_mean": round(mean, 2), + } + # Convenience default factory used by session.Session. def default_tree_cache() -> TreeCache: diff --git a/web_inspector.py b/web_inspector.py index fda5244..2385288 100755 --- a/web_inspector.py +++ b/web_inspector.py @@ -1671,6 +1671,7 @@ def api_clear_text(): def api_metrics(): from session import get_session s = get_session() + tc = s.tree_cache.counters() lines = [ "# HELP oso_step_count Total tool calls processed", "# TYPE oso_step_count counter", @@ -1678,6 +1679,28 @@ def api_metrics(): "# HELP oso_uptime_seconds Process uptime", "# TYPE oso_uptime_seconds gauge", f"oso_uptime_seconds {int(s.steps.uptime_s)}", + # ── Tree-cache effectiveness (P2) ───────────────────────────── + "# HELP oso_tree_cache_hits_total Tree-cache lookups served " + "from a fresh cached capture", + "# TYPE oso_tree_cache_hits_total counter", + f"oso_tree_cache_hits_total {tc['hits']}", + "# HELP oso_tree_cache_misses_total Tree-cache lookups that " + "required a fresh accessibility-tree walk", + "# TYPE oso_tree_cache_misses_total counter", + f"oso_tree_cache_misses_total {tc['misses']}", + "# HELP oso_tree_cache_entries Windows currently cached", + "# TYPE oso_tree_cache_entries gauge", + f"oso_tree_cache_entries {tc['entries']}", + # ── Capture-latency summary (P2) ────────────────────────────── + "# HELP oso_tree_capture_ms Accessibility-tree capture latency " + "summary (milliseconds)", + "# TYPE oso_tree_capture_ms summary", + f"oso_tree_capture_ms_sum {tc['capture_ms_total']}", + f"oso_tree_capture_ms_count {tc['capture_count']}", + "# HELP oso_tree_capture_ms_max Slowest tree capture this " + "session (milliseconds)", + "# TYPE oso_tree_capture_ms_max gauge", + f"oso_tree_capture_ms_max {tc['capture_ms_max']}", ] if s.budgets is not None: st = s.budgets.status() @@ -1709,6 +1732,11 @@ def api_tool_run(name: str): args = _merge_query() return _tool_response(name, args) + # /api/healthz must stay cheap enough to poll: the OCR diagnostic + # spawns a `tesseract --version` subprocess, so it is computed once + # per process and reused (binary availability doesn't change mid-run). + _healthz_ocr_cache: dict = {} + @app.route("/api/healthz") def api_healthz(): from session import get_session @@ -1719,18 +1747,22 @@ def api_healthz(): "step_count": s.steps.count, "adapter": type(observer._adapter).__name__, "version": (config.get("mcp", {}) or {}).get("version", "0.2.0"), + # P2 telemetry: cache effectiveness + capture-latency summary. + "tree_cache": s.tree_cache.counters(), } # Surface common misconfigurations. try: from main import config_load_status out.update(config_load_status()) - except Exception: - pass + except Exception as e: + logger.debug(f"healthz: config_load_status unavailable: {e}") try: - from ocr_util import diagnose as _ocr_diag - out["ocr"] = _ocr_diag(config) - except Exception: - pass + if "ocr" not in _healthz_ocr_cache: + from ocr_util import diagnose as _ocr_diag + _healthz_ocr_cache["ocr"] = _ocr_diag(config) + out["ocr"] = _healthz_ocr_cache["ocr"] + except Exception as e: + logger.debug(f"healthz: OCR diagnose failed: {e}") return jsonify(out) return app From 19d0dfbf804e4121a85664b60695b5b3e41a4a36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:39:58 +0000 Subject: [PATCH 09/14] [P2] mypy + except audit + coverage floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject.toml (new): mypy lenient global (ignore_missing_imports, follow_imports=silent) with strict-ish overrides for tree_cache.py, errors.py, session.py (disallow_untyped_defs/incomplete_defs, check_untyped_defs, warn_return_any, no_implicit_optional); ruff lint select pinned to the default E4/E7/E9/F so E722 (bare except) is an explicit, guaranteed gate; coverage config with source=["."] and omit=tests/* for a stable floor basis. - mypy: fixed all 43 pre-existing errors across 12 files (Optional narrowing in budgets/session/ocr_util/ollama_setup, implicit-Optional defaults in observer.perform_action, _resolve_element None contract in tools.py element actions + propose_action, Optional tree threading in get_window_structure/observe paths, PIL Image typing, duplicated `params`/`prefix` bindings in window_agent/ollama_setup, Playwright :has-text double-match in element_selectors, ctypes.windll inline ignores on the two Windows-only guard sites). `mypy .` is clean. - except audit: 26 silent `except …: pass` blocks triaged down to 7. 19 now log context via logger.debug (OCR configure/preprocess guards, UIA/AT-SPI partial-walk truncation, trace pre/post hash, screenshot crop fallbacks, selector-as-element_id fallback) and budget accounting failures escalate to logger.exception. The 7 kept are precise-typed cleanup guards (OSError tmp unlink, ValueError ring.remove, FileNotFoundError wmctrl->xdotool fallback, int-parse) or commented UIA cached-property fallbacks that legitimately degrade. - CI: new `mypy .` step; regression lane now runs with `--cov --cov-fail-under=53` (measured 56.25% source coverage on the lane; floor set 3 points below). mypy added to requirements-dev.txt; coverage/mypy artifacts gitignored. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- .github/workflows/ci.yml | 10 ++++-- .gitignore | 6 ++++ ascii_renderer.py | 16 ++++----- budgets.py | 7 ++-- description.py | 22 ++++++------ element_selectors.py | 4 +-- linux_adapter.py | 13 +++---- main.py | 7 ++-- mcp_server.py | 10 +++--- observer.py | 23 +++++++------ ocr_util.py | 2 +- ollama_setup.py | 11 +++--- pyproject.toml | 41 ++++++++++++++++++++++ requirements-dev.txt | 1 + session.py | 1 + tools.py | 73 +++++++++++++++++++++++----------------- web_inspector.py | 8 +++-- window_agent.py | 6 ++-- 18 files changed, 170 insertions(+), 91 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9b9c35..c6f0260 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,14 @@ jobs: pip install -r requirements.txt -r requirements-dev.txt - name: Lint with ruff run: ruff check . --exclude tests - - name: Run regression tests - run: pytest tests/ -q -m "not user" --ignore=tests/user + - name: Type-check with mypy + run: mypy . + - name: Run regression tests (with coverage floor) + # Coverage floor: measured 56% source coverage on this lane + # (tests/ omitted via pyproject [tool.coverage.run]); floor set + # 3 points below so incidental drift doesn't break CI while real + # regressions in test breadth do. + run: pytest tests/ -q -m "not user" --ignore=tests/user --cov --cov-fail-under=53 - name: Run user tests (when display + tesseract present) run: pytest tests/user/ -q -m "user and not slow_llm and not slow_vlm and not needs_display" diff --git a/.gitignore b/.gitignore index fe3c460..b6724cc 100755 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,9 @@ config.json # Test harness outputs test-results/ .pytest_cache/ + +# Coverage artifacts (pytest-cov) +.coverage +.coverage.* +htmlcov/ +.mypy_cache/ diff --git a/ascii_renderer.py b/ascii_renderer.py index e8489c1..e4cb5bb 100755 --- a/ascii_renderer.py +++ b/ascii_renderer.py @@ -323,8 +323,8 @@ def _preprocess_image(img: "Image.Image", upscale: int) -> Tuple["Image.Image", img = img.convert("L") try: img = ImageOps.autocontrast(img, cutoff=2) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr preprocess] autocontrast skipped: {e}") return img, float(upscale or 1) @@ -358,8 +358,8 @@ def _ocr_lines(screenshot_bytes: bytes, try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr] tesseract_cmd configure failed: {e}") ocr_cfg = (config or {}).get("ocr") or {} min_conf = int(ocr_cfg.get("min_confidence", 30)) @@ -442,8 +442,8 @@ def _ocr_roi_text(crop: "Image.Image", psm: int, config: Optional[dict]) -> str: resample=__import__("PIL").Image.LANCZOS, ).convert("L") crop = ImageOps.autocontrast(crop, cutoff=2) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr roi] crop preprocess skipped: {e}") txt = pytesseract.image_to_string(crop, config=f"--psm {psm} --oem 3") return " ".join(txt.split()).strip() except Exception: @@ -535,8 +535,8 @@ def __init__(self, config: dict): try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[init] tesseract_cmd configure failed: {e}") # ── public string-only API (back-compat) ───────────────────────────────── diff --git a/budgets.py b/budgets.py index 9da714b..ebbecd2 100644 --- a/budgets.py +++ b/budgets.py @@ -48,12 +48,12 @@ def is_set(self) -> bool: return self.limit is not None def remaining(self) -> Optional[int]: - if not self.is_set(): + if self.limit is None: return None return max(0, self.limit - self.used) def trip(self) -> bool: - return self.is_set() and self.used >= self.limit + return self.limit is not None and self.used >= self.limit class BudgetStore: @@ -93,7 +93,8 @@ def gate(self, tool: str) -> Optional[Dict[str, Any]]: with self.lock: now = time.time() elapsed = now - self.started_at - if self.session_secs.is_set() and elapsed >= self.session_secs.limit: + session_limit = self.session_secs.limit + if session_limit is not None and elapsed >= session_limit: return error_dict(Code.BUDGET_EXCEEDED, "max_session_seconds reached", elapsed=int(elapsed)) diff --git a/description.py b/description.py index b581cb3..e6903b8 100755 --- a/description.py +++ b/description.py @@ -243,8 +243,8 @@ def __init__(self, config: dict): try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[init] tesseract_cmd configure failed: {e}") # ── Accessibility tree → prose ──────────────────────────────────────────── @@ -537,7 +537,7 @@ def _prepare_image(screenshot_bytes: bytes, max_dim: int) -> bytes: return screenshot_bytes scale = max_dim / long_edge new_size = (max(1, int(w * scale)), max(1, int(h * scale))) - resized = img.resize(new_size, Image.LANCZOS) + resized = img.resize(new_size, Image.Resampling.LANCZOS) buf = io.BytesIO() resized.save(buf, format="PNG", optimize=True) return buf.getvalue() @@ -718,11 +718,11 @@ def from_vlm_multipass( env["_passes"]["scene_ms"] = int((time.time() - t0) * 1000) scene_obj: Dict[str, Any] = {} if raw1: - scene_obj, err = _tolerant_json_loads(raw1) - if scene_obj is None: - scene_obj = {} + parsed1, err = _tolerant_json_loads(raw1) + if parsed1 is None: env["_passes"]["scene_error"] = err else: + scene_obj = parsed1 for k in ("app", "screen_type", "primary_task"): if scene_obj.get(k) is not None: env[k] = scene_obj[k] @@ -736,11 +736,11 @@ def from_vlm_multipass( env["_passes"]["controls_ms"] = int((time.time() - t0) * 1000) controls_obj: Dict[str, Any] = {} if raw2: - controls_obj, err = _tolerant_json_loads(raw2) - if controls_obj is None: - controls_obj = {} + parsed2, err = _tolerant_json_loads(raw2) + if parsed2 is None: env["_passes"]["controls_error"] = err else: + controls_obj = parsed2 for k in ("focused", "modal_open", "controls", "sensitive_regions"): if controls_obj.get(k) is not None: @@ -770,8 +770,8 @@ def from_vlm_multipass( tree_text = "" try: tree_text = self.from_tree(root, window) - except Exception: - pass + except Exception as e: + logger.debug(f"[vlm verify] tree grounding skipped: {e}") ctxv = ( f"\n" f"{json.dumps(env.get('controls') or [], ensure_ascii=False)}\n" diff --git a/element_selectors.py b/element_selectors.py index 13dc13f..6b197e4 100644 --- a/element_selectors.py +++ b/element_selectors.py @@ -361,8 +361,8 @@ def _parse_role_and_preds(token: str) -> Tuple[str, List[Predicate]]: pos = end + 1 continue # Playwright :has-text("…") / :text("…") → name*="…" - if rest[pos] == ":" and _HASTEXT_RE.match(rest, pos): - ht = _HASTEXT_RE.match(rest, pos) + ht = _HASTEXT_RE.match(rest, pos) if rest[pos] == ":" else None + if ht is not None: text_val = ht.group("dq") if ht.group("dq") is not None else ht.group("sq") preds.append(Predicate(key="name", op="*=", value=text_val)) pos = ht.end() diff --git a/linux_adapter.py b/linux_adapter.py index ac61abe..aed6b90 100644 --- a/linux_adapter.py +++ b/linux_adapter.py @@ -70,8 +70,8 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": states = node.getState().getStates() enabled = pyatspi.STATE_ENABLED in states focused = pyatspi.STATE_FOCUSED in states - selected = (pyatspi.STATE_SELECTED in states - or pyatspi.STATE_CHECKED in states) + selected: 'bool | None' = (pyatspi.STATE_SELECTED in states + or pyatspi.STATE_CHECKED in states) expanded = pyatspi.STATE_EXPANDED in states if ( pyatspi.STATE_EXPANDABLE in states) else None # STATE_SELECTED on non-selectable widgets is meaningless; only @@ -92,8 +92,9 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": value_now = float(iv.currentValue) value_min = float(iv.minimumValue) value_max = float(iv.maximumValue) - except Exception: - pass + except Exception as e: + logger.debug(f"[atspi] value interface unavailable for " + f"{eid}: {e}") ui = UIElement( element_id=eid, name=name, role=role, value=value, @@ -108,8 +109,8 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": for i in range(node.childCount): ui.children.append(_walk(node[i], f"{eid}.{i}", depth + 1, max_depth)) - except Exception: - pass + except Exception as e: + logger.debug(f"[atspi] child walk truncated at {eid}: {e}") return ui def get_element_tree(hwnd=None) -> Optional[UIElement]: diff --git a/main.py b/main.py index b51abad..1c6de29 100644 --- a/main.py +++ b/main.py @@ -135,10 +135,11 @@ def load_config(path: str) -> dict: print(f"\n[main:load_config] {msg}\n", file=sys.stderr) return dict(_DEFAULT_CONFIG) # Deep-merge with defaults so missing keys are always present - merged = dict(_DEFAULT_CONFIG) + merged: dict = dict(_DEFAULT_CONFIG) for k, v in cfg.items(): - if isinstance(v, dict) and isinstance(merged.get(k), dict): - merged[k] = {**merged[k], **v} + base = merged.get(k) + if isinstance(v, dict) and isinstance(base, dict): + merged[k] = {**base, **v} else: merged[k] = v return merged diff --git a/mcp_server.py b/mcp_server.py index a9c911e..c56343b 100755 --- a/mcp_server.py +++ b/mcp_server.py @@ -1156,8 +1156,10 @@ def _t_full_screenshot(self, hwnd, info, args) -> Dict: buf2 = _io2.BytesIO() full_img.crop(crop_box).save(buf2, format="PNG") ocr_bytes = buf2.getvalue() - except Exception: - pass + except Exception as e: + logger.debug( + f"[get_full_screenshot] window crop for OCR overlay " + f"failed; using full image: {e}") sketch = self.renderer.render( root = tree, screen_bounds = ref, @@ -1172,8 +1174,8 @@ def _t_full_screenshot(self, hwnd, info, args) -> Dict: from PIL import Image as _Image _img = _Image.open(_io.BytesIO(shot)) img_w, img_h = _img.size - except Exception: - pass + except Exception as e: + logger.debug(f"[get_full_screenshot] size probe failed: {e}") return { "window": info.title if info else "(full screen)", diff --git a/observer.py b/observer.py index a61ab9b..ff890fa 100644 --- a/observer.py +++ b/observer.py @@ -394,7 +394,7 @@ def get_screenshot(self, hwnd=None) -> Optional[bytes]: traceback.print_exc() return None - def perform_action(self, action: str, element_id: str = None, + def perform_action(self, action: str, element_id: Optional[str] = None, value: Any = None, hwnd=None) -> Dict: if self.scenario is not None: handled = self.scenario.handle_action(action=action, @@ -533,8 +533,9 @@ def get_windows_above_bounds(self, hwnd) -> List[Bounds]: hh = rect[3] - rect[1] if w > 0 and hh > 0: above.append(Bounds(rect[0], rect[1], w, hh)) - except Exception: - pass + except Exception as e: + logger.debug(f"[occlusion] window rect probe failed " + f"for hwnd {h}: {e}") try: h = win32gui.GetWindow(h, GW_HWNDNEXT) except Exception: @@ -804,8 +805,9 @@ def _prop(pid, default=None): depth + 1, max_depth, true_cond, cache_request=cache_request, cached=kids_cached)) - except Exception: - pass + except Exception as e: + logger.debug(f"[uia] child walk truncated at " + f"{elem_id}: {e}") return node def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: @@ -956,7 +958,8 @@ def get_screenshot(self, hwnd=None) -> Optional[bytes]: save_dc.SelectObject(save_bmp) # PW_RENDERFULLCONTENT (0x2) renders hardware-accelerated content too - ok = ctypes.windll.user32.PrintWindow(hwnd, save_dc.GetSafeHdc(), 2) + ok = ctypes.windll.user32.PrintWindow( # type: ignore[attr-defined] + hwnd, save_dc.GetSafeHdc(), 2) bmpinfo = save_bmp.GetInfo() bmpstr = save_bmp.GetBitmapBits(True) @@ -1004,7 +1007,7 @@ def get_screenshot(self, hwnd=None) -> Optional[bytes]: traceback.print_exc() return None - def perform_action(self, action: str, element_id: str = None, + def perform_action(self, action: str, element_id: Optional[str] = None, value: Any = None, hwnd=None) -> Dict: try: import pyautogui @@ -1540,7 +1543,7 @@ def get_full_display_screenshot(self) -> Optional[bytes]: logger.warning(f"[ScreenObserver:get_full_display_screenshot] {e}; falling back") return self._adapter.get_screenshot() - def perform_action(self, action: str, element_id: str = None, + def perform_action(self, action: str, element_id: Optional[str] = None, value: Any = None, hwnd=None) -> Dict: return self._adapter.perform_action(action, element_id, value, hwnd) @@ -1816,8 +1819,8 @@ def _activate_windows(self, hwnd: int) -> tuple: try: import ctypes import ctypes.wintypes - user32 = ctypes.windll.user32 - kernel32 = ctypes.windll.kernel32 + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] # Restore if minimised (IsIconic) or hidden. if user32.IsIconic(hwnd): diff --git a/ocr_util.py b/ocr_util.py index debc855..8b5a048 100644 --- a/ocr_util.py +++ b/ocr_util.py @@ -92,7 +92,7 @@ def diagnose(config: Optional[Dict[str, Any]]) -> Dict[str, Any]: info["error"] = (f"tesseract_cmd={cmd!r} does not exist on disk and " f"is not on PATH. {INSTALL_HINT}") return info - info["configured_path_exists"] = bool(cmd) and ( + info["configured_path_exists"] = cmd is not None and ( os.path.exists(cmd) or shutil.which(cmd) is not None ) try: diff --git a/ollama_setup.py b/ollama_setup.py index 3f7b461..952be08 100644 --- a/ollama_setup.py +++ b/ollama_setup.py @@ -89,7 +89,8 @@ def _ask_runner() -> Optional[List[str]]: file=sys.stderr, ) - options: List[Tuple[str, List[str]]] = [] + # Each option's prefix is a token list; None marks the "custom" choice. + options: List[Tuple[str, Optional[List[str]]]] = [] # Option 1: native ollama on PATH if _test_runner(["ollama"]): @@ -118,8 +119,8 @@ def _ask_runner() -> Optional[List[str]]: if raw.isdigit(): idx = int(raw) - 1 if 0 <= idx < len(options): - _, prefix = options[idx] - if prefix is None: + _, chosen = options[idx] + if chosen is None: # Custom entry try: custom = input( @@ -145,9 +146,9 @@ def _ask_runner() -> Optional[List[str]]: if ok != "y": continue return tokens - if prefix == []: # skip + if chosen == []: # skip return [] - return prefix + return chosen print(f" Please enter a number 1–{len(options)}.", file=sys.stderr) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0081e3f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +# Tooling configuration (quality gates, P2). +# Runtime packaging still goes through requirements*.txt. + +# ─── ruff ───────────────────────────────────────────────────────────────────── +# Mirrors ruff's default rule set explicitly so E722 (bare `except:`) is a +# documented, guaranteed gate — silent bare excepts must not come back. +[tool.ruff] +extend-exclude = ["tests"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] # E7 includes E722 (bare except) + +# ─── mypy ───────────────────────────────────────────────────────────────────── +# Lenient globally (large legacy surface, optional platform deps), strict-ish +# for the small, fully-typed core modules that everything else builds on. +[tool.mypy] +python_version = "3.11" +ignore_missing_imports = true +follow_imports = "silent" +warn_unused_configs = true +# Platform-specific adapters poke optional OS libraries; exclude the ones +# that cannot even be imported on CI's platform matrix. +exclude = ["tests/"] + +[[tool.mypy.overrides]] +module = ["tree_cache", "errors", "session"] +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +warn_return_any = true +no_implicit_optional = true + +# ─── coverage ──────────────────────────────────────────────────────────────── +[tool.coverage.run] +# Measure all source modules (even ones a given platform never imports, +# e.g. window_agent / mac_adapter) so the CI floor has a stable basis. +source = ["."] +omit = ["tests/*"] + +[tool.coverage.report] +precision = 1 diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f02ad0..924a257 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ pytest>=8.0 pytest-cov>=4.0 ruff>=0.4 +mypy>=1.8 diff --git a/session.py b/session.py index 26ee0ca..b813048 100644 --- a/session.py +++ b/session.py @@ -215,6 +215,7 @@ def next_id(self, *, is_input: bool) -> Tuple[int, Optional[int]]: with self._lock: sid = self._next self._next += 1 + caused_by: Optional[int] if is_input: caused_by = sid self._last_input_step = sid diff --git a/tools.py b/tools.py index 0327b1d..a6ac486 100644 --- a/tools.py +++ b/tools.py @@ -124,8 +124,9 @@ def _resolve_element(tree: UIElement, args: Dict[str, Any] result = sel.resolve(tree, parsed) if result.matches: return result.matches[0], parsed.canonical(), None - except (sel.SelectorParseError, Exception): - pass + except Exception as e: + logger.debug(f"element_id {element_id!r} not parseable as a " + f"selector either: {e}") # Both failed — prefer selector error if we have one (more informative). if selector_err: return None, None, selector_err @@ -300,8 +301,10 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An step_id=step_id, window_uid=info.window_uid) elem, selector_str, err = _resolve_element(tree, args) - if err: - return {**err, "step_id": step_id} + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} if not elem.enabled: return error_dict(Code.ELEMENT_DISABLED, @@ -327,6 +330,7 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An return {**confirm_check, "step_id": step_id} started = time.time() + executor_result: Dict[str, Any] if dry_run: executor_result = {"success": True, "dry_run": True} else: @@ -347,7 +351,7 @@ def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, An if info_after else None) ok = bool(executor_result.get("success", True)) - err_obj = None + err_obj: Optional[Dict[str, Any]] = None if not ok: err_obj = { "code": executor_result.get("error_code", Code.INTERNAL), @@ -687,11 +691,12 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An "node_count": meta["node_count"], "cache": meta["cache"], "depth_used": depth} - serialized = tree.to_dict() + full_serialized = tree.to_dict() th = tree_hash(tree) if not scope: - token = get_session().tree_tokens.put(info.window_uid, serialized, th) - serialized, depth_truncated = _truncate_depth(serialized, depth) + token = get_session().tree_tokens.put(info.window_uid, + full_serialized, th) + serialized, depth_truncated = _truncate_depth(full_serialized, depth) # P3 filtering / paging -------------------------------------------------- roles = args.get("roles") @@ -711,7 +716,7 @@ def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, An visible_regions = [] filtered = _filter_tree( - serialized, + serialized or {}, roles=set(roles) if roles else None, exclude_roles=set(exclude_roles) if exclude_roles else None, visible_regions=visible_regions, @@ -834,14 +839,15 @@ def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, # Diff baselines keep the full capture; only the returned tree is # depth-bounded (with truncated-node markers). token = get_session().tree_tokens.put(info.window_uid, serialized, th) + out_tree: 'Optional[Dict[str, Any]]' = serialized depth_truncated = False if depth is not None: - serialized, depth_truncated = _truncate_depth(serialized, depth) + out_tree, depth_truncated = _truncate_depth(serialized, depth) return tree, { "format": "full", "window_uid": info.window_uid, "window": info.title, - "tree": serialized, + "tree": out_tree, "tree_hash": th, "tree_token": token, "base_token": None, @@ -944,7 +950,7 @@ def _observe_changed_only(ctx: ToolContext, info: WindowInfo, *, depth: int, new_token = sess.tree_tokens.put(info.window_uid, serialized, th) perf = _perf_dict(meta, depth) - base = { + base: Dict[str, Any] = { "ok": True, "success": True, "step_id": step_id, "caused_by_step_id": caused_by, "window_uid": info.window_uid, "window": info.title, @@ -1474,7 +1480,7 @@ def _apply_crop(png_bytes: bytes, bbox: Optional[Dict[str, int]], from PIL import Image except Exception: return png_bytes, None - img = Image.open(_io.BytesIO(png_bytes)) + img: "Image.Image" = Image.open(_io.BytesIO(png_bytes)) source_bbox: Optional[Dict[str, int]] = None if bbox is not None: x = max(0, int(bbox.get("x", 0)) - padding) @@ -1532,7 +1538,7 @@ def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: "height": elem.bounds.height, } - img = Image.open(_io.BytesIO(shot)) + img: "Image.Image" = Image.open(_io.BytesIO(shot)) if bbox: x = max(0, int(bbox.get("x", 0))) y = max(0, int(bbox.get("y", 0))) @@ -1815,7 +1821,7 @@ def replay_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: def replay_step(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: import replay as _replay step_id, caused_by = _new_step_id("replay_step") - rid = args.get("replay_id") + rid = args.get("replay_id") or "" rep = _REPLAYS.get(rid) if rep is None: return error_dict(Code.BAD_REQUEST, "unknown replay_id", @@ -1833,7 +1839,7 @@ def _disp(name: str, a: Dict[str, Any]) -> Dict[str, Any]: def replay_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: step_id, caused_by = _new_step_id("replay_status") - rid = args.get("replay_id") + rid = args.get("replay_id") or "" rep = _REPLAYS.get(rid) if rep is None: return error_dict(Code.BAD_REQUEST, "unknown replay_id", @@ -1961,22 +1967,28 @@ def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: return error_dict(Code.INTERNAL, "could not retrieve element tree", step_id=step_id) elem, selector_str, err = _resolve_element(tree, inner_args) - if err: - return {**err, "step_id": step_id} + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} sess = get_session() bbox = elem.bounds.to_dict() ct = sess.confirms.issue( action=action, window_uid=info.window_uid, - selector=selector_str, bbox=bbox, args=inner_args, + selector=selector_str or "", bbox=bbox, args=inner_args, ) # Optional preview crop. + preview_b64 = None try: shot = ctx.observer.get_screenshot(info.handle) - crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, max_width=400) - preview_b64 = base64.b64encode(crop_bytes).decode() if crop_bytes else None - except Exception: - preview_b64 = None + if shot is not None: + crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, + max_width=400) + if crop_bytes: + preview_b64 = base64.b64encode(crop_bytes).decode() + except Exception as e: + logger.debug(f"propose_action: preview crop failed: {e}") return { "ok": True, "success": True, @@ -2099,8 +2111,8 @@ def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: for spec in (src, dst): if not (spec.get("selector") or spec.get("element_id")): continue - elem, selector_str, err = _resolve_element(tree, spec) - if err or elem is None: + elem, selector_str, res_err = _resolve_element(tree, spec) + if res_err or elem is None: continue tgt = {"window_uid": info.window_uid, "element_id": elem.element_id, @@ -2236,8 +2248,8 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any focused0.handle, window_uid=focused0.window_uid) if t: tree_before = tree_hash(t) - except Exception: - pass + except Exception as e: + logger.debug(f"trace: pre-call tree hash unavailable: {e}") try: result = fn(ctx, args or {}) @@ -2292,8 +2304,9 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any use_cache=False) if t: tree_after = tree_hash(t) - except Exception: - pass + except Exception as e: + logger.debug(f"trace: post-call tree hash " + f"unavailable: {e}") _tracing.record( sess.active_trace, tool=name, caller=args.get("_caller", "unknown"), @@ -2327,7 +2340,7 @@ def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any try: sess.budgets.note(name, result) except Exception: - pass + logger.exception("budget accounting failed") # Audit log. if sess.auditor is not None: diff --git a/web_inspector.py b/web_inspector.py index 2385288..131bedd 100755 --- a/web_inspector.py +++ b/web_inspector.py @@ -1214,7 +1214,7 @@ def _tool_response(name: str, args: dict): return jsonify(result) def _merge_query(extra: Optional[dict] = None) -> dict: - out = {k: v for k, v in request.args.items()} + out: dict = {k: v for k, v in request.args.items()} if "window_index" in out: try: out["window_index"] = int(out["window_index"]) @@ -1396,8 +1396,10 @@ def api_full_screenshot(): buf2 = _io2.BytesIO() full_img.crop(crop_box).save(buf2, format="PNG") ocr_bytes = buf2.getvalue() - except Exception: - pass + except Exception as e: + logger.debug( + f"[/api/full_screenshot] window crop for OCR " + f"overlay failed; using full image: {e}") sketch = renderer.render( root = tree, screen_bounds = ref, diff --git a/window_agent.py b/window_agent.py index b67a331..edcf2fd 100644 --- a/window_agent.py +++ b/window_agent.py @@ -329,7 +329,7 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, return _post(rest, "/api/wait_idle", body) elif tool_name == "observe_window_diff": - params: Dict[str, Any] = {} + params = {} if "window_uid" in args: params["window_uid"] = args["window_uid"] elif wi is not None: @@ -414,7 +414,7 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, # ── P3 extras ─────────────────────────────────────────────────────────── elif tool_name == "get_screenshot_cropped": # Pixel data is huge — same omission policy as get_screenshot. - params: Dict[str, Any] = {} + params = {} if "window_uid" in args: params["window_uid"] = args["window_uid"] elif wi is not None: @@ -429,7 +429,7 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, return result elif tool_name == "get_ocr": - params: Dict[str, Any] = {} + params = {} if "window_uid" in args: params["window_uid"] = args["window_uid"] elif wi is not None: From e3ea697bdc5bec86ad443518ab4c8db9ff53c042 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 19:59:56 +0000 Subject: [PATCH 10/14] =?UTF-8?q?[P3]=20decompose=20tools.py=20=E2=86=92?= =?UTF-8?q?=20tools/=20package=20(shim=20preserves=20imports)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical split of the 2,375-line tools.py into a package: tools/context.py ToolContext + window/element resolution helpers tools/receipts.py receipt building + confirmation gating tools/actions.py element-targeted, coordinate and composite verbs tools/observe.py tree observation, diffs, filter/page helpers tools/vision.py screenshots, OCR, screen descriptions tools/snapshots.py snapshot tools + wait_for / wait_idle tools/trace_replay.py tracing, replay, scenarios, oracles tools/meta.py windows/capabilities/status/propose_action tools/dispatch.py REGISTRY, allowlist gate, cross-cutting hooks tools/__init__.py re-exports the entire pre-split surface (including underscore helpers) so every existing import path keeps working; the only behavioral-code edit is a local dispatch import in replay_step to break the trace_replay ↔ dispatch registry cycle. No behavior changes; all tests pass unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- tools.py | 2375 ----------------------------------------- tools/__init__.py | 160 +++ tools/actions.py | 515 +++++++++ tools/context.py | 142 +++ tools/dispatch.py | 268 +++++ tools/meta.py | 133 +++ tools/observe.py | 543 ++++++++++ tools/receipts.py | 117 ++ tools/snapshots.py | 278 +++++ tools/trace_replay.py | 183 ++++ tools/vision.py | 301 ++++++ 11 files changed, 2640 insertions(+), 2375 deletions(-) delete mode 100644 tools.py create mode 100644 tools/__init__.py create mode 100644 tools/actions.py create mode 100644 tools/context.py create mode 100644 tools/dispatch.py create mode 100644 tools/meta.py create mode 100644 tools/observe.py create mode 100644 tools/receipts.py create mode 100644 tools/snapshots.py create mode 100644 tools/trace_replay.py create mode 100644 tools/vision.py diff --git a/tools.py b/tools.py deleted file mode 100644 index a6ac486..0000000 --- a/tools.py +++ /dev/null @@ -1,2375 +0,0 @@ -""" -tools.py — Central tool implementations. - -Both mcp_server.py and web_inspector.py dispatch into this module; the -MCP and REST surfaces are thin wrappers. Every tool returns a dict in -one of two shapes: - - {ok: true, step_id: …, …tool-specific fields…} - {ok: false, success: false, step_id: …, error: {code, message, …}} - -For backwards compatibility (design doc D5) success-shaped legacy fields -are preserved alongside the new `ok` / `error` object on existing tools. -""" - -from __future__ import annotations - -import base64 -import logging -import re -import time -from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple - -import element_selectors as sel -from errors import Code, error_dict, annotate_legacy_result -from hashing import focused_selector, tree_hash -from observer import ( - ScreenObserver, UIElement, WindowInfo, WindowResolution, -) -from redaction import mark_untrusted -from session import Session, get_session - -logger = logging.getLogger(__name__) - - -# ─── Context ────────────────────────────────────────────────────────────────── - -@dataclass -class ToolContext: - observer: ScreenObserver - renderer: Any - describer: Any - config: Dict[str, Any] - - @property - def session(self) -> Session: - return get_session() - - -# ─── Helpers ────────────────────────────────────────────────────────────────── - -def _is_input_tool(name: str) -> bool: - return name in { - "click_at", "type_text", "press_key", "scroll", "bring_to_foreground", - "click_element", "focus_element", "set_value", "invoke_element", - "select_option", "hover_at", "hover_element", - "right_click_at", "right_click_element", - "double_click_at", "double_click_element", - "drag", "key_into_element", "clear_text", - "click_element_and_observe", "type_and_observe", "press_key_and_observe", - } - - -def _new_step_id(name: str) -> Tuple[int, Optional[int]]: - return get_session().steps.next_id(is_input=_is_input_tool(name)) - - -def _resolve_window(ctx: ToolContext, args: Dict[str, Any] - ) -> Tuple[List[WindowInfo], WindowResolution]: - windows = ctx.observer.list_windows() - res = ctx.observer.resolve_window( - windows, - window_uid=args.get("window_uid"), - window_index=args.get("window_index"), - window_title=args.get("window_title"), - ) - return windows, res - - -def _focused_window(windows: List[WindowInfo]) -> Optional[WindowInfo]: - for w in windows: - if w.is_focused: - return w - return windows[0] if windows else None - - -def _resolve_element(tree: UIElement, args: Dict[str, Any] - ) -> Tuple[Optional[UIElement], Optional[str], Optional[Dict]]: - """ - Returns (element, selector_string, error_dict). Either *element* is set - or *error_dict* is. The selector string is resolved or derived. - """ - selector = args.get("selector") - element_id = args.get("element_id") - - # Try selector first, then element_id. When both are provided, element_id - # acts as a fallback so a bad/unmatched selector doesn't block the call. - selector_err = None - if selector: - try: - parsed = sel.parse(selector) - result = sel.resolve(tree, parsed) - if result.matches: - return result.matches[0], parsed.canonical(), None - selector_err = error_dict( - Code.ELEMENT_NOT_FOUND, - f"no element matches selector {selector!r}", - selector=selector, - ) - except sel.SelectorParseError as e: - selector_err = error_dict(Code.BAD_REQUEST, - f"selector parse error: {e}", - selector=selector) - - if element_id: - elem = _find_by_id(tree, element_id) - if elem is not None: - derived = sel.selector_for(tree, element_id) or "" - return elem, derived, None - # element_id didn't match any internal ID — try parsing it as a selector - # (LLMs often pass the display-format string e.g. 'TabItem "name"' as an id). - try: - parsed = sel.parse(element_id) - result = sel.resolve(tree, parsed) - if result.matches: - return result.matches[0], parsed.canonical(), None - except Exception as e: - logger.debug(f"element_id {element_id!r} not parseable as a " - f"selector either: {e}") - # Both failed — prefer selector error if we have one (more informative). - if selector_err: - return None, None, selector_err - return None, None, error_dict( - Code.ELEMENT_NOT_FOUND, - f"no element with id {element_id!r}", - element_id=element_id, - ) - - if selector_err: - return None, None, selector_err - - return None, None, error_dict(Code.BAD_REQUEST, - "either element_id or selector is required") - - -def _find_by_id(elem: UIElement, target: str) -> Optional[UIElement]: - if elem.element_id == target: - return elem - for c in elem.children: - r = _find_by_id(c, target) - if r is not None: - return r - return None - - -def _new_dialogs(before: List[WindowInfo], after: List[WindowInfo]) -> List[Dict]: - before_uids = {w.window_uid for w in before} - return [ - {"window_uid": w.window_uid, "title": w.title} - for w in after if w.window_uid and w.window_uid not in before_uids - ] - - -# ─── Read-only tools ────────────────────────────────────────────────────────── - -def list_windows(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("list_windows") - try: - windows = ctx.observer.list_windows() - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "is_mock": ctx.observer.is_mock, - "count": len(windows), - "windows": [{"index": i, **w.to_dict()} for i, w in enumerate(windows)], - } - except Exception as e: - logger.exception("list_windows failed") - return error_dict(Code.INTERNAL, str(e), step_id=step_id) - - -def get_capabilities(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_capabilities") - out = ctx.observer.get_capabilities() - out.update({"step_id": step_id, "caused_by_step_id": caused_by, "success": True}) - return out - - -def get_monitors(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_monitors") - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "monitors": ctx.observer.get_monitors(), - } - - -def find_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("find_element") - selector_text = args.get("selector") - if not selector_text: - return error_dict(Code.BAD_REQUEST, "selector is required", - step_id=step_id) - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - try: - parsed = sel.parse(selector_text) - except sel.SelectorParseError as e: - return error_dict(Code.BAD_REQUEST, f"selector parse error: {e}", - step_id=step_id) - result = sel.resolve(tree, parsed) - if not result.matches: - return error_dict(Code.ELEMENT_NOT_FOUND, - f"no element matches {selector_text!r}", - step_id=step_id, selector=selector_text, - window_uid=info.window_uid) - first = result.matches[0] - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, - "element_id": first.element_id, - "selector": parsed.canonical(), - "bounds": first.bounds.to_dict(), - "ambiguous_matches": len(result.matches), - "all_matches": [ - {"element_id": m.element_id, "bounds": m.bounds.to_dict(), - "name": m.name, "role": m.role} - for m in result.matches - ], - } - - -# ─── Receipts and action wrappers ──────────────────────────────────────────── - -def _build_receipt(*, step_id: int, action: str, target: Dict[str, Any], - before_tree: Optional[UIElement], - before_windows: List[WindowInfo], - after_tree: Optional[UIElement], - after_windows: List[WindowInfo], - duration_ms: int, dry_run: bool, ok: bool, - extra: Optional[Dict[str, Any]] = None, - error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - before_hash = tree_hash(before_tree) if before_tree else "" - after_hash = tree_hash(after_tree) if after_tree else "" - receipt: Dict[str, Any] = { - "ok": ok, "success": ok, - "step_id": step_id, "caused_by_step_id": step_id, - "action": action, - "dry_run": dry_run, - "target": target, - "before": { - "tree_hash": before_hash, - "focused_selector": focused_selector(before_tree) if before_tree else "", - }, - "after": { - "tree_hash": after_hash, - "focused_selector": focused_selector(after_tree) if after_tree else "", - }, - "changed": (before_hash != after_hash) and not dry_run, - "new_dialogs": _new_dialogs(before_windows, after_windows), - "duration_ms": duration_ms, - } - if extra: - receipt.update(extra) - if error: - receipt["error"] = error - return receipt - - -def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, Any], - executor: Callable[[UIElement, WindowInfo, Dict[str, Any]], - Dict[str, Any]]) -> Dict[str, Any]: - step_id, _ = _new_step_id(action_name) - dry_run = bool(args.get("dry_run")) - - # Budget gate (no-op until P5 plumbs budgets in). - sess = get_session() - if sess.budgets is not None: - gate = sess.budgets.gate(action_name) - if gate is not None: - return {**gate, "step_id": step_id} - - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - - elem, selector_str, err = _resolve_element(tree, args) - if err or elem is None: - return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, - "element resolution failed")), - "step_id": step_id} - - if not elem.enabled: - return error_dict(Code.ELEMENT_DISABLED, - f"element is disabled: {selector_str}", - step_id=step_id, selector=selector_str) - - occluded = ctx.observer.is_element_occluded(elem.bounds, info.handle, windows) - if occluded: - return error_dict(Code.ELEMENT_OCCLUDED, - f"element is occluded: {selector_str}", - step_id=step_id, selector=selector_str) - - target = { - "window_uid": info.window_uid, - "element_id": elem.element_id, - "selector": selector_str, - "bounds": elem.bounds.to_dict(), - } - - # Confirmation gate (no-op until P5 plumbs confirms in). - confirm_check = _check_confirmation(ctx, action_name, args, target) - if confirm_check is not None: - return {**confirm_check, "step_id": step_id} - - started = time.time() - executor_result: Dict[str, Any] - if dry_run: - executor_result = {"success": True, "dry_run": True} - else: - try: - executor_result = executor(elem, info, args) - except Exception as e: - logger.exception(f"{action_name} executor failed") - executor_result = {"success": False, "error": str(e)} - - duration_ms = int((time.time() - started) * 1000) - after_windows = ctx.observer.list_windows() - info_after = ctx.observer.window_by_uid(after_windows, info.window_uid) - # Post-action re-read must bypass the tree cache so the receipt's - # after-state reflects reality (the fresh capture refreshes the cache). - after_tree = (ctx.observer.get_element_tree(info_after.handle, - window_uid=info_after.window_uid, - use_cache=False) - if info_after else None) - - ok = bool(executor_result.get("success", True)) - err_obj: Optional[Dict[str, Any]] = None - if not ok: - err_obj = { - "code": executor_result.get("error_code", Code.INTERNAL), - "message": str(executor_result.get("error", "action failed")), - "recoverable": False, - "suggested_next_tool": None, - "context": {}, - } - - extra: Dict[str, Any] = {} - if "warning" in (res.warning or ""): - extra["warning"] = res.warning - if res.warning: - extra["warning"] = res.warning - - return _build_receipt( - step_id=step_id, action=action_name, target=target, - before_tree=tree, before_windows=windows, - after_tree=after_tree, after_windows=after_windows, - duration_ms=duration_ms, dry_run=dry_run, ok=ok, - extra=extra, error=err_obj, - ) - - -def _confirmation_rules_match(ctx: ToolContext, - target: Dict[str, Any]) -> bool: - """True when any configured confirmation_required rule matches the - target's role/name (derived from the selector tail).""" - confirm = ctx.config.get("confirmation_required") or [] - if not confirm: - return False - name_to_test = "" - role_to_test = "" - # Best-effort: derive name/role from the selector tail. - sel_tail = (target.get("selector") or "").split("/")[-1] - m = re.match(r"([A-Za-z_*]\w*)", sel_tail) - if m: - role_to_test = m.group(1) - nm = re.search(r'name="([^"]*)"', sel_tail) - if nm: - name_to_test = nm.group(1) - - def _matches_rule(rule: Dict[str, Any]) -> bool: - rname = rule.get("name_regex") - rrole = rule.get("role") - if rrole and role_to_test != rrole: - return False - if rname and not re.search(rname, name_to_test or ""): - return False - return bool(rname or rrole) - - return any(_matches_rule(r) for r in confirm) - - -def _check_confirmation(ctx: ToolContext, action_name: str, - args: Dict[str, Any], - target: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Returns an error dict to short-circuit when confirmation is required.""" - if not _confirmation_rules_match(ctx, target): - return None - - token = args.get("confirm_token") - if not token: - return error_dict( - Code.CONFIRMATION_REQUIRED, - f"action {action_name} requires a confirm_token from propose_action", - action=action_name, target=target, - ) - sess = get_session() - ct = sess.confirms.consume(token) - if not ct: - return error_dict(Code.CONFIRMATION_INVALID, - "confirm_token expired, unknown, or already used", - token=token) - if ct.action != action_name or ct.window_uid != target["window_uid"] \ - or ct.selector != target["selector"]: - return error_dict(Code.CONFIRMATION_INVALID, - "confirm_token does not match the proposed action") - tol = (ctx.config.get("confirmation", {}) or {}).get("bbox_tolerance_px", 20) - bb = target["bounds"] - if (abs(bb["x"] - ct.bbox.get("x", 0)) > tol or - abs(bb["y"] - ct.bbox.get("y", 0)) > tol or - abs(bb["width"] - ct.bbox.get("width", 0)) > tol or - abs(bb["height"] - ct.bbox.get("height", 0)) > tol): - return error_dict(Code.CONFIRMATION_INVALID, - "element bounds drifted beyond confirmation tolerance") - return None - - -# ─── Element-targeted actions ───────────────────────────────────────────────── - -def click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - button = args.get("button", "left") - count = int(args.get("count", 1)) - - def _exec(elem: UIElement, info: WindowInfo, _args: Dict[str, Any] - ) -> Dict[str, Any]: - cx, cy = elem.bounds.center_x, elem.bounds.center_y - result = ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": cx, "y": cy, "button": button, - "double": (count >= 2)}, - hwnd=info.handle, - ) - return result - - return _do_element_action(ctx, action_name="click_element", - args=args, executor=_exec) - - -def focus_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - # Focus via center click (universal fallback). Adapter-specific - # SetFocus paths live in the platform adapters; today the universal - # click is sufficient on all three platforms. - return ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - return _do_element_action(ctx, action_name="focus_element", - args=args, executor=_exec) - - -def set_value(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - if "value" not in args: - step_id, _ = _new_step_id("set_value") - return error_dict(Code.BAD_REQUEST, "value is required", - step_id=step_id) - value = args["value"] - clear_first = bool(args.get("clear_first", True)) - - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - if clear_first: - ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) - ctx.observer.perform_action("key", value="delete", hwnd=info.handle) - return ctx.observer.perform_action("type", value=str(value), - hwnd=info.handle) - return _do_element_action(ctx, action_name="set_value", - args=args, executor=_exec) - - -def invoke_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - # No platform-specific InvokePattern surface yet; behaves like click. - return _do_element_action( - ctx, action_name="invoke_element", args=args, - executor=lambda elem, info, _a: ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ), - ) - - -def select_option(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Click the element to open the menu, then click the matching child.""" - option_name = args.get("option_name") - option_index = args.get("option_index") - if option_name is None and option_index is None: - step_id, _ = _new_step_id("select_option") - return error_dict(Code.BAD_REQUEST, - "option_name or option_index is required", - step_id=step_id) - - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - # Re-walk to see the now-visible option list (bypass the cache — - # the click above just changed the UI). - new_tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid, - use_cache=False) - target: Optional[UIElement] = None - if new_tree is not None: - descendants = [] - stack = [new_tree] - while stack: - e = stack.pop() - descendants.append(e) - stack.extend(e.children) - if option_name is not None: - target = next((d for d in descendants if d.name == option_name), None) - elif option_index is not None: - idx = int(option_index) - if 0 <= idx < len(elem.children): - target = elem.children[idx] - if target is None: - return {"success": False, - "error": "option not found after opening selector"} - return ctx.observer.perform_action( - "click_at", - value={"x": target.bounds.center_x, "y": target.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - - return _do_element_action(ctx, action_name="select_option", - args=args, executor=_exec) - - -# ─── Legacy actions (now also returning ActionReceipts) ────────────────────── - -def click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("click_at") - before_windows = ctx.observer.list_windows() - started = time.time() - result = ctx.observer.perform_action("click_at", value={ - "x": args.get("x", 0), "y": args.get("y", 0), - "button": args.get("button", "left"), - "double": args.get("double", False), - }) - duration_ms = int((time.time() - started) * 1000) - after_windows = ctx.observer.list_windows() - out = annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - out.setdefault("action", "click_at") - out["duration_ms"] = duration_ms - out["new_dialogs"] = _new_dialogs(before_windows, after_windows) - return out - - -def type_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("type_text") - result = ctx.observer.perform_action("type", value=args.get("text", "")) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def press_key(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("press_key") - result = ctx.observer.perform_action("key", value=args.get("keys", "")) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def scroll(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("scroll") - result = ctx.observer.perform_action("scroll", value=args) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def _effective_depth(ctx: ToolContext, args: Dict[str, Any]) -> int: - """Depth to return: caller's depth= (clamped to tree.max_depth) or - tree.default_depth when the caller passes none.""" - tree_cfg = ctx.config.get("tree", {}) or {} - hard_cap = int(tree_cfg.get("max_depth", 8)) - requested = args.get("depth") - if requested is None: - depth = int(tree_cfg.get("default_depth", 5)) - else: - try: - depth = int(requested) - except (TypeError, ValueError): - depth = int(tree_cfg.get("default_depth", 5)) - return max(0, min(depth, hard_cap)) - - -def _truncate_depth(node: Optional[Dict[str, Any]], max_depth: int, - _depth: int = 0) -> Tuple[Optional[Dict[str, Any]], bool]: - """Copy *node* limited to *max_depth* levels below it. - - Nodes whose children were dropped are marked ``truncated: true`` with a - ``child_count`` so the caller knows to drill in (via scope=/depth=). - Returns (tree, any_node_truncated). The input dict is not mutated.""" - if node is None: - return None, False - out = dict(node) - children = node.get("children") or [] - if _depth >= max_depth and children: - out["children"] = [] - out["truncated"] = True - out["child_count"] = len(children) - return out, True - truncated_any = False - new_children: List[Dict[str, Any]] = [] - for c in children: - nc, t = _truncate_depth(c, max_depth, _depth + 1) - if nc is not None: - new_children.append(nc) - truncated_any = truncated_any or t - out["children"] = new_children - return out, truncated_any - - -def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_window_structure") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - - depth = _effective_depth(ctx, args) - scope = args.get("scope") - token = None - - if scope: - # Drill into one branch only (element-id path, e.g. 'root.3.2'). - ttl = float((ctx.config.get("tree", {}) or {}).get("cache_ttl_s", 2.0)) - had_fresh_entry = (get_session().tree_cache.get( - info.window_uid, ttl_s=ttl) is not None) - started = time.time() - tree = ctx.observer.get_element_subtree( - info.handle, scope, max_depth=depth, - window_uid=info.window_uid) - capture_ms = int((time.time() - started) * 1000) - if tree is None: - return error_dict(Code.ELEMENT_NOT_FOUND, - f"no element with id {scope!r} to scope to", - step_id=step_id, scope=scope, - window_uid=info.window_uid) - perf = {"capture_ms": capture_ms, - "node_count": len(tree.flat_list()), - "cache": "hit" if had_fresh_entry else "miss", - "depth_used": depth} - # scoped captures are not valid diff baselines → no tree_token - else: - tree, meta = ctx.observer.get_element_tree_with_meta( - info.handle, window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - perf = {"capture_ms": meta["capture_ms"], - "node_count": meta["node_count"], - "cache": meta["cache"], - "depth_used": depth} - full_serialized = tree.to_dict() - th = tree_hash(tree) - if not scope: - token = get_session().tree_tokens.put(info.window_uid, - full_serialized, th) - serialized, depth_truncated = _truncate_depth(full_serialized, depth) - - # P3 filtering / paging -------------------------------------------------- - roles = args.get("roles") - exclude_roles = args.get("exclude_roles") - visible_only = bool(args.get("visible_only")) - name_regex = args.get("name_regex") - max_text_len = args.get("max_text_len") - prune_empty = bool(args.get("prune_empty")) - max_nodes = args.get("max_nodes") - page_cursor = args.get("page_cursor") - - visible_regions: Optional[List[Dict[str, int]]] = None - if visible_only: - try: - visible_regions = ctx.observer.get_visible_areas(info.handle, windows) - except Exception: - visible_regions = [] - - filtered = _filter_tree( - serialized or {}, - roles=set(roles) if roles else None, - exclude_roles=set(exclude_roles) if exclude_roles else None, - visible_regions=visible_regions, - name_regex=name_regex, - max_text_len=max_text_len, - prune_empty=prune_empty, - ) - - truncated = False - next_cursor: Optional[str] = None - node_count = _count_nodes(filtered) if filtered else 0 - if max_nodes is not None or page_cursor is not None: - filtered, truncated, next_cursor, node_count = _page_tree( - filtered, max_nodes=max_nodes, page_cursor=page_cursor, - ) - - out = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, - "window_uid": info.window_uid, - "element_count": len(tree.flat_list()), - "node_count": node_count, - "tree": filtered, - "tree_hash": th, - "tree_token": token, - "truncated": truncated, - "next_cursor": next_cursor, - "depth_used": depth, - "depth_truncated": depth_truncated, - "perf": perf, - } - if scope: - out["scope"] = scope - else: - # Degradation signal: accessibility-dark windows (games, custom - # renderers) produce sparse trees — steer the agent to pixel-based - # fallbacks instead of letting it act on a near-empty tree. - named = sum(1 for e in tree.flat_list()[1:] if (e.name or "").strip()) - threshold = int((ctx.config.get("tree", {}) or {}) - .get("sparse_threshold", 5)) - if named < threshold: - out["degraded"] = { - "reason": "sparse_accessibility_tree", - "named_node_count": named, - "threshold": threshold, - "suggested_fallbacks": ["get_ocr", "get_screen_description"], - } - return out - - -def get_screenshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_screenshot") - windows, res = _resolve_window(ctx, args) - info = res.info - hwnd = info.handle if info else None - shot = ctx.observer.get_screenshot(hwnd) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title if info else "(full screen)", - "format": "png", "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - - -def bring_to_foreground(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("bring_to_foreground") - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - result = ctx.observer.bring_to_foreground(info.handle, windows) - result["window"] = info.title - result["window_uid"] = info.window_uid - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_visible_areas") - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - areas = ctx.observer.get_visible_areas(info.handle, windows) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, - "window_uid": info.window_uid, - "visible_regions": areas, - } - - -# ─── P2: observe-with-diff, snapshots, wait_for, composites ────────────────── - -def _perf_dict(meta: Dict[str, Any], depth: Optional[int]) -> Dict[str, Any]: - return {"capture_ms": meta["capture_ms"], - "node_count": meta["node_count"], - "cache": meta["cache"], - "depth_used": depth} - - -def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, - depth: Optional[int] = None, - ) -> Tuple[Optional[UIElement], Dict[str, Any]]: - tree, meta = ctx.observer.get_element_tree_with_meta( - info.handle, window_uid=info.window_uid) - if tree is None: - return None, {"error": "no tree"} - serialized = tree.to_dict() - th = tree_hash(tree) - # Diff baselines keep the full capture; only the returned tree is - # depth-bounded (with truncated-node markers). - token = get_session().tree_tokens.put(info.window_uid, serialized, th) - out_tree: 'Optional[Dict[str, Any]]' = serialized - depth_truncated = False - if depth is not None: - out_tree, depth_truncated = _truncate_depth(serialized, depth) - return tree, { - "format": "full", - "window_uid": info.window_uid, - "window": info.title, - "tree": out_tree, - "tree_hash": th, - "tree_token": token, - "base_token": None, - "depth_used": depth, - "depth_truncated": depth_truncated, - "perf": _perf_dict(meta, depth), - } - - -def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Return the current tree, optionally as a diff against a tree_token.""" - from diff import diff_custom, diff_json_patch - step_id, caused_by = _new_step_id("observe_window") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - since = args.get("since") - fmt = args.get("format", "custom") - depth = _effective_depth(ctx, args) - changed_only = bool(args.get("changed_only")) - - if not since and changed_only: - return _observe_changed_only(ctx, info, depth=depth, - step_id=step_id, caused_by=caused_by) - - if not since: - _, full = _serialize_full_observation(ctx, info, depth=depth) - full.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "format": "full"}) - return full - - entry = get_session().tree_tokens.get(since) - tree, meta = ctx.observer.get_element_tree_with_meta( - info.handle, window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - serialized = tree.to_dict() - th = tree_hash(tree) - new_token = get_session().tree_tokens.put(info.window_uid, serialized, th) - - if entry is None or entry.window_uid != info.window_uid: - # Token expired/wrong-window: return full tree (depth-bounded). - out_tree, depth_truncated = _truncate_depth(serialized, depth) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, "window": info.title, - "tree": out_tree, "tree_hash": th, - "tree_token": new_token, "base_token": None, - "format": "full", - "depth_used": depth, - "depth_truncated": depth_truncated, - "perf": _perf_dict(meta, depth), - } - - if fmt == "json-patch": - changes = diff_json_patch(entry.serialized, serialized) - out_format = "json-patch" - else: - changes = diff_custom(entry.serialized, serialized) - out_format = "custom" - - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, "window": info.title, - "tree_token": new_token, "base_token": since, - "format": out_format, - "changes": changes, - "unchanged": len(changes) == 0, - "tree_hash": th, - "perf": _perf_dict(meta, depth), - } - - -def _observe_changed_only(ctx: ToolContext, info: WindowInfo, *, depth: int, - step_id: int, caused_by: Optional[int] - ) -> Dict[str, Any]: - """observe_window changed_only=true: compare a fresh capture against the - last cached capture of the window. Unchanged → a tiny - {unchanged: true, tree_hash} response; changed → a custom diff instead - of the full tree; no baseline → full tree.""" - from diff import diff_custom - sess = get_session() - # Baseline: the most recent capture, regardless of cache TTL. - baseline = sess.tree_cache.peek(info.window_uid) - - # Fresh capture (bypass the cache — the whole point is to detect drift). - tree, meta = ctx.observer.get_element_tree_with_meta( - info.handle, window_uid=info.window_uid, use_cache=False) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - serialized = tree.to_dict() - th = tree_hash(tree) - new_token = sess.tree_tokens.put(info.window_uid, serialized, th) - perf = _perf_dict(meta, depth) - - base: Dict[str, Any] = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, "window": info.title, - "tree_hash": th, "tree_token": new_token, - "changed_only": True, - "perf": perf, - } - - if baseline is None: - # Nothing to compare against — return the (depth-bounded) full tree. - out_tree, depth_truncated = _truncate_depth(serialized, depth) - base.update({"format": "full", "tree": out_tree, "base_token": None, - "depth_used": depth, "depth_truncated": depth_truncated}) - return base - - if baseline.tree_hash == th: - base["unchanged"] = True - return base - - changes = diff_custom(baseline.serialized, serialized) - base.update({"format": "custom", "changes": changes, "unchanged": False}) - return base - - -def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot") - windows = ctx.observer.list_windows() - trees: Dict[str, Dict[str, Any]] = {} - hashes: Dict[str, str] = {} - for w in windows: - try: - t = ctx.observer.get_element_tree(w.handle, - window_uid=w.window_uid) - if t is not None and w.window_uid: - trees[w.window_uid] = t.to_dict() - hashes[w.window_uid] = tree_hash(t) - except Exception: - continue - snap = get_session().snapshots.put( - windows=[w.to_dict() for w in windows], - trees=trees, tree_hashes=hashes, - ) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "snapshot_id": snap.snapshot_id, - "ts": snap.ts, - "summary": {"windows": len(snap.windows), "trees": len(trees)}, - } - - -def snapshot_get(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot_get") - sid = args.get("snapshot_id") - if not sid: - return error_dict(Code.BAD_REQUEST, "snapshot_id is required", - step_id=step_id) - snap = get_session().snapshots.get(sid) - if snap is None: - return error_dict(Code.SNAPSHOT_EXPIRED, - "snapshot expired or not found", - step_id=step_id, snapshot_id=sid) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "snapshot_id": snap.snapshot_id, "ts": snap.ts, - "windows": snap.windows, - "trees": snap.trees, - "tree_hashes": snap.tree_hashes, - } - - -def snapshot_diff(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - from diff import diff_custom, diff_json_patch - step_id, caused_by = _new_step_id("snapshot_diff") - a_id = args.get("a") - b_id = args.get("b") - if not a_id or not b_id: - return error_dict(Code.BAD_REQUEST, "a and b are required", - step_id=step_id) - sess = get_session() - a = sess.snapshots.get(a_id) - b = sess.snapshots.get(b_id) - if a is None or b is None: - return error_dict(Code.SNAPSHOT_EXPIRED, - "one or both snapshots are missing", - step_id=step_id) - fmt = args.get("format", "custom") - - a_uids = {w["window_uid"] for w in a.windows} - b_uids = {w["window_uid"] for w in b.windows} - windows_added = sorted(b_uids - a_uids) - windows_removed = sorted(a_uids - b_uids) - common = sorted(a_uids & b_uids) - - per_window: Dict[str, Any] = {} - for uid in common: - if uid in a.trees and uid in b.trees: - if fmt == "json-patch": - per_window[uid] = {"format": "json-patch", - "changes": diff_json_patch(a.trees[uid], b.trees[uid])} - else: - per_window[uid] = {"format": "custom", - "changes": diff_custom(a.trees[uid], b.trees[uid])} - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "windows_added": windows_added, - "windows_removed": windows_removed, - "per_window_changes": per_window, - } - - -def snapshot_drop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot_drop") - sid = args.get("snapshot_id") - dropped = get_session().snapshots.drop(sid) if sid else False - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "dropped": dropped, - } - - -# ─── wait_for / wait_idle ───────────────────────────────────────────────────── - -def _check_condition(ctx: ToolContext, cond: Dict[str, Any], - window_uid_hint: Optional[str]) -> Tuple[bool, Dict[str, Any]]: - kind = cond.get("type") - sess = get_session() - windows = ctx.observer.list_windows() - info = ctx.observer.window_by_uid(windows, window_uid_hint) or _focused_window(windows) - - if kind == "window_appears": - rx = cond.get("title_regex", "") - for w in windows: - if re.search(rx, w.title): - return True, {"window_uid": w.window_uid, "title": w.title} - return False, {} - if kind == "window_disappears": - target = cond.get("window_uid") - for w in windows: - if w.window_uid == target: - return False, {} - return True, {"window_uid": target} - if kind == "focused_changes": - focus = next((w for w in windows if w.is_focused), None) - return (focus is not None), ({"focused_uid": focus.window_uid} if focus else {}) - if kind == "tree_changes": - token = cond.get("since") - entry = sess.tree_tokens.get(token) if token else None - if entry is None or info is None: - return False, {} - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid, - use_cache=False) - return tree is not None and tree_hash(tree) != entry.tree_hash, {} - - if info is None: - return False, {} - # Polling must observe fresh state — bypass the tree cache. - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid, - use_cache=False) - if tree is None: - return False, {} - - if kind == "element_appears": - sel_text = cond.get("selector") - if not sel_text: - return False, {} - try: - res = sel.resolve(tree, sel.parse(sel_text)) - except sel.SelectorParseError: - return False, {} - if res.matches: - m = res.matches[0] - return True, {"element_id": m.element_id, "bounds": m.bounds.to_dict()} - return False, {} - if kind == "element_disappears": - sel_text = cond.get("selector") - eid = cond.get("element_id") - if sel_text: - try: - res = sel.resolve(tree, sel.parse(sel_text)) - return not res.matches, {} - except sel.SelectorParseError: - return False, {} - if eid: - return _find_by_id(tree, eid) is None, {} - return False, {} - if kind == "text_visible": - rx = cond.get("regex", "") - # Walk tree names/values. - for elem in tree.flat_list(): - joined = (elem.name or "") + " " + (elem.value or "") - if re.search(rx, joined): - return True, {"element_id": elem.element_id} - return False, {} - return False, {} - - -def wait_for(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("wait_for") - timeout_ms = int(args.get("timeout_ms", 5000)) - cap = int((ctx.config.get("wait_for", {}) or {}).get("max_timeout_ms", 60000)) - timeout_ms = min(timeout_ms, cap) - poll_ms = max(50, int(args.get("poll_ms", 200))) - conditions = args.get("any_of", []) - if not conditions: - return error_dict(Code.BAD_REQUEST, "any_of is required", - step_id=step_id) - window_uid = args.get("window_uid") - - started = time.time() - polls = 0 - while True: - polls += 1 - for i, cond in enumerate(conditions): - try: - ok, detail = _check_condition(ctx, cond, window_uid) - except Exception: - ok, detail = False, {} - if ok: - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "matched_index": i, "matched_detail": detail, - "elapsed_ms": int((time.time() - started) * 1000), - "polls": polls, - } - elapsed = (time.time() - started) * 1000 - if elapsed >= timeout_ms: - err = error_dict( - Code.TIMEOUT, f"wait_for timed out after {int(elapsed)}ms", - step_id=step_id, - ) - err.update({ - "elapsed_ms": int(elapsed), "polls": polls, - "matched_index": None, - }) - return err - time.sleep(poll_ms / 1000.0) - - -def wait_idle(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("wait_idle") - timeout_ms = int(args.get("timeout_ms", 5000)) - quiet_ms = int(args.get("quiet_ms", 750)) - poll_ms = max(50, int(args.get("poll_ms", 100))) - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - - started = time.time() - last_hash = None - last_change_at = time.time() - while (time.time() - started) * 1000 < timeout_ms: - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid, - use_cache=False) - if tree is None: - time.sleep(poll_ms / 1000.0) - continue - h = tree_hash(tree) - if h != last_hash: - last_hash = h - last_change_at = time.time() - elif (time.time() - last_change_at) * 1000 >= quiet_ms: - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "elapsed_ms": int((time.time() - started) * 1000), - "tree_hash": h, - } - time.sleep(poll_ms / 1000.0) - - err = error_dict(Code.TIMEOUT, "wait_idle timed out", step_id=step_id) - err["elapsed_ms"] = int((time.time() - started) * 1000) - return err - - -# ─── Composite action+observe ───────────────────────────────────────────────── - -def _compose_observe(ctx: ToolContext, base_result: Dict[str, Any], - wait_after_ms: int, since_token: Optional[str]) -> None: - if wait_after_ms > 0: - time.sleep(wait_after_ms / 1000.0) - args: Dict[str, Any] = {} - target = base_result.get("target") or {} - if target.get("window_uid"): - args["window_uid"] = target["window_uid"] - if since_token: - args["since"] = since_token - obs = observe_window(ctx, args) - base_result["observation"] = obs - - -def click_element_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = click_element(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -def type_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = type_text(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -def press_key_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = press_key(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -# ─── P3: tree filtering / paging helpers ───────────────────────────────────── - -def _filter_tree(node: Dict[str, Any], *, roles: Optional[set], - exclude_roles: Optional[set], - visible_regions: Optional[List[Dict[str, int]]], - name_regex: Optional[str], - max_text_len: Optional[int], - prune_empty: bool) -> Optional[Dict[str, Any]]: - if node is None: - return None - role = node.get("role") - name = node.get("name") or "" - bounds = node.get("bounds") or {} - - # Role filter - role_keep = True - if roles is not None and role not in roles: - role_keep = False - if exclude_roles is not None and role in exclude_roles: - role_keep = False - - # Name regex - name_keep = True - if name_regex: - try: - name_keep = bool(re.search(name_regex, name)) - except re.error: - name_keep = True - - # Visibility - visible_keep = True - if visible_regions is not None: - visible_keep = _intersects_any(bounds, visible_regions) - - self_keep = role_keep and name_keep and visible_keep - - # Recurse children regardless (so we can keep ancestors if descendants match) - new_children: List[Dict[str, Any]] = [] - for c in node.get("children", []) or []: - fc = _filter_tree( - c, roles=roles, exclude_roles=exclude_roles, - visible_regions=visible_regions, name_regex=name_regex, - max_text_len=max_text_len, prune_empty=prune_empty, - ) - if fc is not None: - new_children.append(fc) - - if prune_empty and not self_keep and not new_children: - return None - - # Truncate text fields if requested. - truncated_node = dict(node) - if max_text_len is not None: - n = int(max_text_len) - if isinstance(truncated_node.get("name"), str) and len(truncated_node["name"]) > n: - truncated_node["name"] = truncated_node["name"][:n] + "…" - v = truncated_node.get("value") - if isinstance(v, str) and len(v) > n: - truncated_node["value"] = v[:n] + "…" - truncated_node["children"] = new_children - return truncated_node - - -def _intersects_any(b: Dict[str, int], regions: List[Dict[str, int]]) -> bool: - if not b: - return False - bx, by = b.get("x", 0), b.get("y", 0) - bw, bh = b.get("width", 0), b.get("height", 0) - if bw <= 0 or bh <= 0: - return False - bx2, by2 = bx + bw, by + bh - for r in regions: - rx, ry = r.get("x", 0), r.get("y", 0) - rx2, ry2 = rx + r.get("width", 0), ry + r.get("height", 0) - if bx < rx2 and bx2 > rx and by < ry2 and by2 > ry: - return True - return False - - -def _count_nodes(node: Optional[Dict[str, Any]]) -> int: - if node is None: - return 0 - return 1 + sum(_count_nodes(c) for c in (node.get("children") or [])) - - -def _page_tree(node: Optional[Dict[str, Any]], *, - max_nodes: Optional[int], - page_cursor: Optional[str] - ) -> Tuple[Optional[Dict[str, Any]], bool, Optional[str], int]: - """ - Paginated DFS walk. Returns (subtree-shaped result containing only the - page slice, truncated flag, next_cursor, node_count_in_page). - - Cursors are post-order element_ids; resuming starts from the next sibling - in the original walk. This is a best-effort pager — if the tree changed, - callers will get SnapshotExpired-shaped semantics by virtue of an unknown - cursor returning truncated:false and node_count:0. - """ - if node is None: - return None, False, None, 0 - flat: List[Dict[str, Any]] = [] - _flatten(node, flat) - - if page_cursor is not None: - for i, n in enumerate(flat): - if n.get("id") == page_cursor: - flat = flat[i + 1:] - break - else: - return None, False, None, 0 - - if max_nodes is None or max_nodes >= len(flat): - # Return full (possibly trimmed) tree starting from cursor. - if page_cursor is None: - return node, False, None, len(flat) - return _flat_to_tree(flat), False, None, len(flat) - - page = flat[:max_nodes] - truncated = True - next_cursor = page[-1].get("id") if page else None - return _flat_to_tree(page), truncated, next_cursor, len(page) - - -def _flatten(node: Dict[str, Any], out: List[Dict[str, Any]]) -> None: - out.append(node) - for c in node.get("children") or []: - _flatten(c, out) - - -def _flat_to_tree(flat: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: - """Wrap a list of nodes as children of a synthetic Window root.""" - if not flat: - return None - return { - "id": "page-root", - "name": "[paged]", - "role": "Group", - "value": None, - "bounds": {"x": 0, "y": 0, "width": 0, "height": 0}, - "enabled": True, "focused": False, - "keyboard_shortcut": None, "description": None, - "children": [dict(n, children=[]) for n in flat], - } - - -# ─── P3: cropped screenshots, region OCR, budgeted description ──────────────── - -def get_screenshot_cropped(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """get_screenshot with optional bbox / element_id / max_width / padding.""" - step_id, caused_by = _new_step_id("get_screenshot_cropped") - windows, res = _resolve_window(ctx, args) - info = res.info - hwnd = info.handle if info else None - shot = ctx.observer.get_screenshot(hwnd) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - - bbox: Optional[Dict[str, int]] = args.get("bbox") - element_id: Optional[str] = args.get("element_id") - padding = int(args.get("padding_px", 0)) - max_width: Optional[int] = args.get("max_width") - - if bbox is None and element_id and info is not None: - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is not None: - elem = _find_by_id(tree, element_id) - if elem is not None: - # Convert to window-relative coordinates. - bbox = { - "x": max(0, elem.bounds.x - info.bounds.x), - "y": max(0, elem.bounds.y - info.bounds.y), - "width": elem.bounds.width, - "height": elem.bounds.height, - } - - if bbox or max_width: - shot, source_bbox = _apply_crop(shot, bbox, padding, max_width) - else: - source_bbox = None - - out: Dict[str, Any] = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title if info else "(full screen)", - "format": "png", "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - if source_bbox: - out["source_bbox"] = source_bbox - return out - - -def _apply_crop(png_bytes: bytes, bbox: Optional[Dict[str, int]], - padding: int, max_width: Optional[int] - ) -> Tuple[bytes, Optional[Dict[str, int]]]: - try: - import io as _io - from PIL import Image - except Exception: - return png_bytes, None - img: "Image.Image" = Image.open(_io.BytesIO(png_bytes)) - source_bbox: Optional[Dict[str, int]] = None - if bbox is not None: - x = max(0, int(bbox.get("x", 0)) - padding) - y = max(0, int(bbox.get("y", 0)) - padding) - x2 = min(img.width, int(bbox.get("x", 0)) + int(bbox.get("width", 0)) + padding) - y2 = min(img.height, int(bbox.get("y", 0)) + int(bbox.get("height", 0)) + padding) - if x2 > x and y2 > y: - img = img.crop((x, y, x2, y2)) - source_bbox = {"x": x, "y": y, "width": x2 - x, "height": y2 - y} - if max_width and img.width > int(max_width): - ratio = int(max_width) / float(img.width) - new_size = (int(max_width), max(1, int(img.height * ratio))) - img = img.resize(new_size) - buf = __import__("io").BytesIO() - img.save(buf, "PNG") - return buf.getvalue(), source_bbox - - -def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Region-scoped OCR; returns [{text, confidence, bbox}].""" - step_id, caused_by = _new_step_id("get_ocr") - try: - import io as _io - from PIL import Image - import pytesseract - from ocr_util import configure as _ocr_configure - _ocr_configure(ctx.config) - except Exception: - from ocr_util import INSTALL_HINT - return error_dict(Code.PLATFORM_UNSUPPORTED, - f"pytesseract / Pillow not installed. {INSTALL_HINT}", - step_id=step_id, hint=INSTALL_HINT) - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - shot = ctx.observer.get_screenshot(info.handle) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - bbox = args.get("bbox") - element_id = args.get("element_id") - if element_id and not bbox: - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is not None: - elem = _find_by_id(tree, element_id) - if elem is not None: - bbox = { - "x": max(0, elem.bounds.x - info.bounds.x), - "y": max(0, elem.bounds.y - info.bounds.y), - "width": elem.bounds.width, - "height": elem.bounds.height, - } - - img: "Image.Image" = Image.open(_io.BytesIO(shot)) - if bbox: - x = max(0, int(bbox.get("x", 0))) - y = max(0, int(bbox.get("y", 0))) - x2 = min(img.width, x + int(bbox.get("width", 0))) - y2 = min(img.height, y + int(bbox.get("height", 0))) - if x2 > x and y2 > y: - img = img.crop((x, y, x2, y2)) - - min_conf = (ctx.config.get("ocr", {}) or {}).get("min_confidence", 30) - try: - data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT) - except pytesseract.TesseractNotFoundError: - from ocr_util import diagnose as _ocr_diag, INSTALL_HINT - return error_dict( - Code.PLATFORM_UNSUPPORTED, - ("tesseract binary not found — check ocr.tesseract_cmd in " - f"config.json. {INSTALL_HINT}"), - step_id=step_id, **_ocr_diag(ctx.config), - ) - except Exception as e: - return error_dict(Code.INTERNAL, f"OCR failed: {e}", - step_id=step_id) - out_words: List[Dict[str, Any]] = [] - for i, text in enumerate(data["text"]): - text = (text or "").strip() - if not text: - continue - try: - conf = int(data["conf"][i]) - except (TypeError, ValueError): - conf = 0 - if conf < min_conf: - continue - out_words.append({ - "text": text, "confidence": conf, - "bbox": {"x": int(data["left"][i]), "y": int(data["top"][i]), - "width": int(data["width"][i]), - "height": int(data["height"][i])}, - }) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, "window_uid": info.window_uid, - "words": out_words, - } - - -def get_screen_description(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Combined description: accessibility tree + OCR + VLM, returning every available source.""" - step_id, caused_by = _new_step_id("get_screen_description") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id) - max_tokens = args.get("max_tokens") - focus_id = args.get("focus_element") - - sub: UIElement = tree - if focus_id: - found = _find_by_id(tree, focus_id) - if found is not None: - sub = found - - parts: Dict[str, str] = {} - - # Accessibility tree — always attempted. - try: - parts["accessibility"] = ctx.describer.from_tree(sub, info) - except Exception as e: - logger.exception("[get_screen_description] accessibility failed: %s", e) - - # OCR — attempted when enabled in config. - ocr_enabled = (ctx.config.get("ocr", {}) or {}).get("enabled", True) - if ocr_enabled: - try: - shot = ctx.observer.get_screenshot(info.handle) - if shot: - parts["ocr"] = ctx.describer.from_ocr(shot) - else: - logger.warning("[get_screen_description] screenshot unavailable for OCR") - except Exception as e: - logger.exception("[get_screen_description] OCR failed: %s", e) - - # VLM — attempted when enabled in config. In multipass mode the VLM - # output is a structured envelope; the JSON-serialised form is folded - # into the concatenated body (for back-compat with the legacy text - # description) and the parsed dict is returned separately under - # ``vlm_structured`` so callers don't have to re-parse it. - vlm_structured: Any = None - vlm_enabled = (ctx.config.get("vlm", {}) or {}).get("enabled", False) - if vlm_enabled: - try: - shot = ctx.observer.get_screenshot(info.handle) - if shot: - vlm_mode = ( - (ctx.config.get("vlm", {}) or {}).get("mode") or "single" - ).lower() - if vlm_mode == "multipass": - env = ctx.describer.from_vlm_multipass( - shot, root=sub, window=info, - ) - if env is not None: - import json as _json - parts["vlm"] = _json.dumps(env, indent=2, - ensure_ascii=False) - vlm_structured = env - else: - vlm_out = ctx.describer.from_vlm( - shot, root=sub, window=info, - ) - if vlm_out is not None: - parts["vlm"] = vlm_out - else: - logger.warning("[get_screen_description] screenshot unavailable for VLM") - except Exception as e: - logger.exception("[get_screen_description] VLM failed: %s", e) - - body = "" - if parts: - body = "\n\n".join(f"[{k}]\n{v}" for k, v in parts.items()) - else: - body = "[no description available]" - - truncated = False - if max_tokens is not None: - char_cap = int(max_tokens) * 4 # rough chars-per-token - if len(body) > char_cap: - body = body[:char_cap] + "… [truncated]" - truncated = True - - result: Dict[str, Any] = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, "window_uid": info.window_uid, - "effective_mode": "combined", - "description": body, - "truncated": truncated, - } - if vlm_structured is not None: - result["vlm_structured"] = vlm_structured - return result - - -# ─── Dispatcher ─────────────────────────────────────────────────────────────── - -REGISTRY: Dict[str, Callable[[ToolContext, Dict[str, Any]], Dict[str, Any]]] = { - # Read-only - "list_windows": list_windows, - "get_capabilities": get_capabilities, - "get_monitors": get_monitors, - "find_element": find_element, - "get_window_structure": get_window_structure, - "get_screenshot": get_screenshot, - "get_visible_areas": get_visible_areas, - - # Element-targeted actions - "click_element": click_element, - "focus_element": focus_element, - "set_value": set_value, - "invoke_element": invoke_element, - "select_option": select_option, - - # Legacy actions - "click_at": click_at, - "type_text": type_text, - "press_key": press_key, - "scroll": scroll, - "bring_to_foreground": bring_to_foreground, - - # P2: sync, diff, snapshots, composites - "observe_window": observe_window, - "snapshot": snapshot, - "snapshot_get": snapshot_get, - "snapshot_diff": snapshot_diff, - "snapshot_drop": snapshot_drop, - "wait_for": wait_for, - "wait_idle": wait_idle, - "click_element_and_observe": click_element_and_observe, - "type_and_observe": type_and_observe, - "press_key_and_observe": press_key_and_observe, - - # P3 - "get_screenshot_cropped": get_screenshot_cropped, - "get_ocr": get_ocr, - "get_screen_description": get_screen_description, -} - -# Forward-declared P4 entries appended after definitions (below). - - -# ─── P4: tracing, replay, scenarios, oracles ───────────────────────────────── - -def trace_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import tracing as _tracing - step_id, caused_by = _new_step_id("trace_start") - sess = get_session() - if sess.active_trace is not None and not sess.active_trace.closed: - return error_dict(Code.BAD_REQUEST, "trace already active", - step_id=step_id, - trace_id=sess.active_trace.trace_id) - handle = _tracing.start(label=args.get("label", ""), config=ctx.config) - sess.active_trace = handle - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "trace_id": handle.trace_id, - "started_at": handle.started_at, - "dir": handle.dir, - } - - -def trace_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import tracing as _tracing - step_id, caused_by = _new_step_id("trace_stop") - sess = get_session() - if sess.active_trace is None: - return error_dict(Code.BAD_REQUEST, "no active trace", - step_id=step_id) - info = _tracing.stop(sess.active_trace) - sess.active_trace = None - info.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return info - - -def trace_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("trace_status") - sess = get_session() - if sess.active_trace is None: - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "active_trace_id": None, "step_count": 0, "dir": None} - h = sess.active_trace - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "active_trace_id": h.trace_id, - "step_count": h.counter.value, - "dir": h.dir, - } - - -# ─── Replay state ──────────────────────────────────────────────────────────── - -_REPLAYS: Dict[str, Any] = {} - - -def replay_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import replay as _replay - step_id, caused_by = _new_step_id("replay_start") - path = args.get("path") - if not path: - return error_dict(Code.BAD_REQUEST, "path is required", - step_id=step_id) - mode = args.get("mode", "execute") - on_div = args.get("on_divergence", "warn") - try: - rep = _replay.load(path, mode=mode, on_divergence=on_div) - except Exception as e: - return error_dict(Code.BAD_REQUEST, f"could not load trace: {e}", - step_id=step_id, path=path) - handle_id = "rep:" + str(len(_REPLAYS) + 1) - _REPLAYS[handle_id] = rep - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": handle_id, - "total": len(rep.rows), - "mode": rep.mode, - "label": rep.label, - } - - -def replay_step(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import replay as _replay - step_id, caused_by = _new_step_id("replay_step") - rid = args.get("replay_id") or "" - rep = _REPLAYS.get(rid) - if rep is None: - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - - def _disp(name: str, a: Dict[str, Any]) -> Dict[str, Any]: - return dispatch(ctx, name, a) - - out = _replay.step(rep, dispatch=_disp) - out.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": rid}) - return out - - -def replay_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("replay_status") - rid = args.get("replay_id") or "" - rep = _REPLAYS.get(rid) - if rep is None: - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": rid, - "position": rep.position, - "total": len(rep.rows), - "finished": rep.finished, - "divergences": rep.divergences, - "mode": rep.mode, - } - - -def replay_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("replay_stop") - rid = args.get("replay_id") - if rid in _REPLAYS: - _REPLAYS.pop(rid) - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "stopped": True} - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - - -# ─── Scenarios ─────────────────────────────────────────────────────────────── - -def load_scenario(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import scenarios as _scn - step_id, caused_by = _new_step_id("load_scenario") - path = args.get("path") - if not path: - return error_dict(Code.BAD_REQUEST, "path is required", - step_id=step_id) - try: - sc = _scn.load(path) - _scn.attach_to_observer(sc, ctx.observer) - # The scenario replaces the mock world — cached trees are stale. - get_session().tree_cache.invalidate_all() - except _scn.ScenarioError as e: - return error_dict(Code.SCENARIO_INVALID, str(e), - step_id=step_id, path=path) - except Exception as e: - return error_dict(Code.SCENARIO_INVALID, f"{type(e).__name__}: {e}", - step_id=step_id, path=path) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "scenario": sc.name, - "state": sc.current_state, - "states": list(sc.states.keys()), - } - - -# ─── Oracles ───────────────────────────────────────────────────────────────── - -def assert_state(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import oracles as _oracles - step_id, caused_by = _new_step_id("assert_state") - pred = args.get("predicate") or args.get("predicates") or [] - out = _oracles.evaluate(ctx.observer, pred, config=ctx.config) - if out.get("ok"): - out["step_id"] = step_id - out["caused_by_step_id"] = caused_by - return out - - -REGISTRY.update({ - "trace_start": trace_start, - "trace_stop": trace_stop, - "trace_status": trace_status, - "replay_start": replay_start, - "replay_step": replay_step, - "replay_status": replay_status, - "replay_stop": replay_stop, - "load_scenario": load_scenario, - "assert_state": assert_state, -}) - - -# ─── P5: budgets, propose_action, status reporters ─────────────────────────── - -def get_budget_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_budget_status") - sess = get_session() - if sess.budgets is None: - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "configured": False} - out = sess.budgets.status() - out.update({"ok": True, "success": True, "configured": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return out - - -def get_redaction_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_redaction_status") - sess = get_session() - out = sess.redactor.status() if sess.redactor else {"enabled": False, "active": False} - out.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return out - - -def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Issue a single-use confirm_token for a destructive action.""" - step_id, caused_by = _new_step_id("propose_action") - action = args.get("action") - inner_args = args.get("args") or {} - if not action: - return error_dict(Code.BAD_REQUEST, "action is required", - step_id=step_id) - - windows, res = _resolve_window(ctx, inner_args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id) - elem, selector_str, err = _resolve_element(tree, inner_args) - if err or elem is None: - return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, - "element resolution failed")), - "step_id": step_id} - - sess = get_session() - bbox = elem.bounds.to_dict() - ct = sess.confirms.issue( - action=action, window_uid=info.window_uid, - selector=selector_str or "", bbox=bbox, args=inner_args, - ) - # Optional preview crop. - preview_b64 = None - try: - shot = ctx.observer.get_screenshot(info.handle) - if shot is not None: - crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, - max_width=400) - if crop_bytes: - preview_b64 = base64.b64encode(crop_bytes).decode() - except Exception as e: - logger.debug(f"propose_action: preview crop failed: {e}") - - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "confirm_token": ct.token, - "expires_at": ct.expires_at, - "would_target": { - "window_uid": info.window_uid, - "selector": selector_str, - "bounds": bbox, - "screenshot_b64": preview_b64, - }, - } - - -REGISTRY.update({ - "get_budget_status": get_budget_status, - "get_redaction_status": get_redaction_status, - "propose_action": propose_action, -}) - - -# ─── P6: extra input verbs ──────────────────────────────────────────────────── - -def hover_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("hover_at") - x = int(args.get("x", 0)) - y = int(args.get("y", 0)) - hover_ms = int(args.get("hover_ms", 250)) - try: - import pyautogui - pyautogui.moveTo(x, y) - time.sleep(hover_ms / 1000.0) - ok = True - except Exception: - ok = False - return {"ok": ok, "success": ok, "action": "hover_at", - "step_id": step_id, "caused_by_step_id": step_id, - "x": x, "y": y, "hover_ms": hover_ms} - - -def hover_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - hover_ms = int(args.get("hover_ms", 250)) - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - try: - import pyautogui - pyautogui.moveTo(elem.bounds.center_x, elem.bounds.center_y) - time.sleep(hover_ms / 1000.0) - return {"success": True, "action": "hover", "hover_ms": hover_ms} - except Exception as e: - return {"success": False, "error": str(e)} - return _do_element_action(ctx, action_name="hover_element", - args=args, executor=_exec) - - -def right_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_at(ctx, dict(args, button="right")) - - -def right_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_element(ctx, dict(args, button="right")) - - -def double_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_at(ctx, dict(args, double=True)) - - -def double_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_element(ctx, dict(args, count=2)) - - -def drag(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("drag") - src = args.get("from") or {} - dst = args.get("to") or {} - modifiers = list(args.get("modifiers") or []) - # Resolve element references on either end. - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - tree = (ctx.observer.get_element_tree(info.handle, - window_uid=info.window_uid) - if info else None) - - def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: - if "x" in spec and "y" in spec: - return int(spec["x"]), int(spec["y"]) - eid = spec.get("element_id") - sel_text = spec.get("selector") - if tree is None: - return None - if sel_text: - try: - resu = sel.resolve(tree, sel.parse(sel_text)) - if resu.matches: - e = resu.matches[0] - return e.bounds.center_x, e.bounds.center_y - except sel.SelectorParseError: - return None - if eid: - e = _find_by_id(tree, eid) - if e is not None: - return e.bounds.center_x, e.bounds.center_y - return None - - p1 = _to_xy(src) - p2 = _to_xy(dst) - if p1 is None or p2 is None: - return error_dict(Code.BAD_REQUEST, - "drag requires from/to as {x,y} or {selector|element_id}", - step_id=step_id) - - # §21 confirmation gate: drag endpoints addressed by selector/element_id - # resolve to concrete elements, so a confirmation_required rule matching - # either endpoint (e.g. a "Trash" drop target) must demand a token, just - # like the other element-targeted verbs. The first matching endpoint is - # validated (propose_action(action="drag", args={"selector": …}) issues - # the token for that endpoint's selector). - if tree is not None and info is not None: - for spec in (src, dst): - if not (spec.get("selector") or spec.get("element_id")): - continue - elem, selector_str, res_err = _resolve_element(tree, spec) - if res_err or elem is None: - continue - tgt = {"window_uid": info.window_uid, - "element_id": elem.element_id, - "selector": selector_str, - "bounds": elem.bounds.to_dict()} - if _confirmation_rules_match(ctx, tgt): - confirm_check = _check_confirmation(ctx, "drag", args, tgt) - if confirm_check is not None: - return {**confirm_check, "step_id": step_id} - break # token validated for the matching endpoint - - duration = float(args.get("duration_s", 0.5)) - path = [p1, ((p1[0] + p2[0]) // 2, (p1[1] + p2[1]) // 2), p2] - try: - import pyautogui - for k in modifiers: - pyautogui.keyDown(k) - try: - pyautogui.moveTo(*p1) - pyautogui.dragTo(*p2, duration=duration, button="left") - finally: - for k in modifiers: - pyautogui.keyUp(k) - ok = True - err = None - except Exception as e: - ok = False - err = str(e) - out = {"ok": ok, "success": ok, "action": "drag", - "step_id": step_id, "caused_by_step_id": step_id, - "from": list(p1), "to": list(p2), - "modifiers": modifiers, "path": [list(p) for p in path]} - if err: - out["error"] = {"code": Code.INTERNAL, "message": err, - "recoverable": False, "suggested_next_tool": None, - "context": {}} - return out - - -def key_into_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - keys = args.get("keys", "") - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - return ctx.observer.perform_action("key", value=keys, hwnd=info.handle) - return _do_element_action(ctx, action_name="key_into_element", - args=args, executor=_exec) - - -def clear_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) - return ctx.observer.perform_action("key", value="delete", - hwnd=info.handle) - return _do_element_action(ctx, action_name="clear_text", - args=args, executor=_exec) - - -REGISTRY.update({ - "hover_at": hover_at, - "hover_element": hover_element, - "right_click_at": right_click_at, - "right_click_element": right_click_element, - "double_click_at": double_click_at, - "double_click_element": double_click_element, - "drag": drag, - "key_into_element": key_into_element, - "clear_text": clear_text, -}) - - -_ALLOWLIST_TOOLS = { - "get_capabilities", "get_monitors", "get_budget_status", - "get_redaction_status", "trace_status", "replay_status", - "list_windows", -} - - -def _check_allowlist(ctx: ToolContext, name: str - ) -> Optional[Dict[str, Any]]: - actions = ctx.config.get("actions") or {} - allow = set(actions.get("allow") or []) - deny = set(actions.get("deny") or []) - default = actions.get("default", "allow") - if name in _ALLOWLIST_TOOLS: - return None - if name in deny: - return error_dict(Code.PERMISSION_DENIED, - f"tool {name!r} is in actions.deny") - if allow and name not in allow and default == "deny": - return error_dict(Code.PERMISSION_DENIED, - f"tool {name!r} is not in actions.allow") - if not allow and default == "deny": - return error_dict(Code.PERMISSION_DENIED, - f"actions.default is 'deny' and no allowlist matches {name!r}") - return None - - -def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any]: - fn = REGISTRY.get(name) - if fn is None: - return error_dict(Code.BAD_REQUEST, f"unknown tool: {name}") - - blocked = _check_allowlist(ctx, name) - if blocked is not None: - sid, _ = _new_step_id(name) - blocked["step_id"] = sid - return blocked - - started = time.time() - sess = get_session() - tree_before = "" - if sess.active_trace is not None and not sess.active_trace.closed: - try: - windows0 = ctx.observer.list_windows() - focused0 = next((w for w in windows0 if w.is_focused), None) - if focused0: - t = ctx.observer.get_element_tree( - focused0.handle, window_uid=focused0.window_uid) - if t: - tree_before = tree_hash(t) - except Exception as e: - logger.debug(f"trace: pre-call tree hash unavailable: {e}") - - try: - result = fn(ctx, args or {}) - except Exception as e: - logger.exception(f"tool {name} crashed") - result = error_dict(Code.INTERNAL, f"{type(e).__name__}: {e}") - - duration_ms = int((time.time() - started) * 1000) - - # P1 tree cache invalidation — the single choke point. After any input - # tool (including bring_to_foreground) the affected window's cached tree - # is stale; drop it so subsequent reads re-walk. When the target window - # cannot be determined (legacy coordinate tools), drop everything. - if _is_input_tool(name): - try: - uid = None - if isinstance(result, dict): - uid = ((result.get("target") or {}).get("window_uid") - or result.get("window_uid")) - uid = uid or (args or {}).get("window_uid") - if uid: - sess.tree_cache.invalidate(uid) - else: - sess.tree_cache.invalidate_all() - except Exception: - logger.exception("tree cache invalidation failed") - - # Recurse-safety: don't trace meta tools that would recurse forever. - if name not in {"trace_start", "trace_stop", "trace_status", - "replay_start", "replay_step", "replay_status", - "replay_stop"}: - if sess.active_trace is not None and not sess.active_trace.closed: - try: - import tracing as _tracing - shot_full = ctx.observer.get_full_display_screenshot() - shot_window = None - tgt_uid = None - tgt = result.get("target") or {} - tgt_uid = tgt.get("window_uid") or args.get("window_uid") - if tgt_uid: - win = ctx.observer.window_by_uid( - ctx.observer.list_windows(), tgt_uid) - if win: - shot_window = ctx.observer.get_screenshot(win.handle) - tree_after = "" - try: - after_w = ctx.observer.list_windows() - f = next((w for w in after_w if w.is_focused), None) - if f: - t = ctx.observer.get_element_tree( - f.handle, window_uid=f.window_uid, - use_cache=False) - if t: - tree_after = tree_hash(t) - except Exception as e: - logger.debug(f"trace: post-call tree hash " - f"unavailable: {e}") - _tracing.record( - sess.active_trace, - tool=name, caller=args.get("_caller", "unknown"), - args={k: v for k, v in (args or {}).items() - if not k.startswith("_")}, - result=result, duration_ms=duration_ms, - tree_hash_before=tree_before, tree_hash_after=tree_after, - full_screenshot=shot_full, - window_screenshot=shot_window, - ) - except Exception: - logger.exception("trace.record failed") - - # Apply redaction to text-bearing read-only results. - if sess.redactor is not None: - try: - result = _apply_redaction(name, result, sess.redactor) - except Exception: - logger.exception("redaction failed") - - # Untrusted-content marking (always on): screen-derived text is - # attacker-influenced data (prompt injection), never instructions. - # Flag it and strip ANSI/control characters. - try: - result = mark_untrusted(name, result) - except Exception: - logger.exception("untrusted-content marking failed") - - # Budget accounting. - if sess.budgets is not None: - try: - sess.budgets.note(name, result) - except Exception: - logger.exception("budget accounting failed") - - # Audit log. - if sess.auditor is not None: - try: - sess.auditor.record( - tool=name, caller=args.get("_caller", "unknown"), - args=args or {}, result=result, - ) - except Exception: - logger.exception("audit failed") - - return result - - -def _apply_redaction(tool: str, result: Dict[str, Any], redactor: Any - ) -> Dict[str, Any]: - if not redactor.is_active() or not isinstance(result, dict): - return result - if tool == "get_window_structure" and "tree" in result: - title = result.get("window") or "" - result["tree"] = redactor.redact_tree(result["tree"], title) - elif tool == "observe_window": - if "tree" in result and isinstance(result["tree"], dict): - result["tree"] = redactor.redact_tree(result["tree"], - result.get("window") or "") - elif tool == "get_screen_description": - if isinstance(result.get("description"), str): - result["description"] = redactor.redact_ocr_text(result["description"]) - elif tool == "get_ocr": - if isinstance(result.get("words"), list): - result["words"] = redactor.redact_ocr_words(result["words"]) - return result diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..0f34505 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,160 @@ +""" +tools — Central tool implementations (package form of the former tools.py). + +Both mcp_server.py and web_inspector.py dispatch into this package; the +MCP and REST surfaces are thin wrappers. Every tool returns a dict in +one of two shapes: + + {ok: true, step_id: …, …tool-specific fields…} + {ok: false, success: false, step_id: …, error: {code, message, …}} + +For backwards compatibility (design doc D5) success-shaped legacy fields +are preserved alongside the new `ok` / `error` object on existing tools. + +P3 decomposition: the implementation now lives in submodules (context, +receipts, actions, observe, vision, snapshots, trace_replay, meta, +dispatch). This __init__ re-exports the entire pre-split public surface +so `import tools` / `from tools import X` keep working unchanged. +""" + +from __future__ import annotations + +from tools.context import ( + ToolContext, + _find_by_id, + _focused_window, + _is_input_tool, + _new_dialogs, + _new_step_id, + _resolve_element, + _resolve_window, +) +from tools.receipts import ( + _build_receipt, + _check_confirmation, + _confirmation_rules_match, +) +from tools.observe import ( + _count_nodes, + _effective_depth, + _filter_tree, + _flat_to_tree, + _flatten, + _intersects_any, + _observe_changed_only, + _page_tree, + _perf_dict, + _serialize_full_observation, + _truncate_depth, + find_element, + get_visible_areas, + get_window_structure, + observe_window, +) +from tools.vision import ( + _apply_crop, + get_ocr, + get_screen_description, + get_screenshot, + get_screenshot_cropped, +) +from tools.actions import ( + _compose_observe, + _do_element_action, + bring_to_foreground, + clear_text, + click_at, + click_element, + click_element_and_observe, + double_click_at, + double_click_element, + drag, + focus_element, + hover_at, + hover_element, + invoke_element, + key_into_element, + press_key, + press_key_and_observe, + right_click_at, + right_click_element, + scroll, + select_option, + set_value, + type_and_observe, + type_text, +) +from tools.snapshots import ( + _check_condition, + snapshot, + snapshot_diff, + snapshot_drop, + snapshot_get, + wait_for, + wait_idle, +) +from tools.trace_replay import ( + _REPLAYS, + assert_state, + load_scenario, + replay_start, + replay_status, + replay_step, + replay_stop, + trace_start, + trace_status, + trace_stop, +) +from tools.meta import ( + get_budget_status, + get_capabilities, + get_monitors, + get_redaction_status, + list_windows, + propose_action, +) +from tools.dispatch import ( + REGISTRY, + _ALLOWLIST_TOOLS, + _apply_redaction, + _check_allowlist, + dispatch, +) + +__all__ = [ + # context + "ToolContext", "_find_by_id", "_focused_window", "_is_input_tool", + "_new_dialogs", "_new_step_id", "_resolve_element", "_resolve_window", + # receipts + "_build_receipt", "_check_confirmation", "_confirmation_rules_match", + # observe + "_count_nodes", "_effective_depth", "_filter_tree", "_flat_to_tree", + "_flatten", "_intersects_any", "_observe_changed_only", "_page_tree", + "_perf_dict", "_serialize_full_observation", "_truncate_depth", + "find_element", "get_visible_areas", "get_window_structure", + "observe_window", + # vision + "_apply_crop", "get_ocr", "get_screen_description", "get_screenshot", + "get_screenshot_cropped", + # actions + "_compose_observe", "_do_element_action", "bring_to_foreground", + "clear_text", "click_at", "click_element", "click_element_and_observe", + "double_click_at", "double_click_element", "drag", "focus_element", + "hover_at", "hover_element", "invoke_element", "key_into_element", + "press_key", "press_key_and_observe", "right_click_at", + "right_click_element", "scroll", "select_option", "set_value", + "type_and_observe", "type_text", + # snapshots / waits + "_check_condition", "snapshot", "snapshot_diff", "snapshot_drop", + "snapshot_get", "wait_for", "wait_idle", + # trace / replay / scenarios / oracles + "_REPLAYS", "assert_state", "load_scenario", "replay_start", + "replay_status", "replay_step", "replay_stop", "trace_start", + "trace_status", "trace_stop", + # meta + "get_budget_status", "get_capabilities", "get_monitors", + "get_redaction_status", "list_windows", "propose_action", + # dispatch + "REGISTRY", "_ALLOWLIST_TOOLS", "_apply_redaction", "_check_allowlist", + "dispatch", +] diff --git a/tools/actions.py b/tools/actions.py new file mode 100644 index 0000000..9b59024 --- /dev/null +++ b/tools/actions.py @@ -0,0 +1,515 @@ +""" +Element-targeted and coordinate input verbs (plus composites). + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Dict, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict, annotate_legacy_result +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_dialogs, _new_step_id, + _resolve_element, _resolve_window, +) +from tools.observe import observe_window +from tools.receipts import ( + _build_receipt, _check_confirmation, _confirmation_rules_match, +) + +logger = logging.getLogger(__name__) + + +def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, Any], + executor: Callable[[UIElement, WindowInfo, Dict[str, Any]], + Dict[str, Any]]) -> Dict[str, Any]: + step_id, _ = _new_step_id(action_name) + dry_run = bool(args.get("dry_run")) + + # Budget gate (no-op until P5 plumbs budgets in). + sess = get_session() + if sess.budgets is not None: + gate = sess.budgets.gate(action_name) + if gate is not None: + return {**gate, "step_id": step_id} + + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + + elem, selector_str, err = _resolve_element(tree, args) + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} + + if not elem.enabled: + return error_dict(Code.ELEMENT_DISABLED, + f"element is disabled: {selector_str}", + step_id=step_id, selector=selector_str) + + occluded = ctx.observer.is_element_occluded(elem.bounds, info.handle, windows) + if occluded: + return error_dict(Code.ELEMENT_OCCLUDED, + f"element is occluded: {selector_str}", + step_id=step_id, selector=selector_str) + + target = { + "window_uid": info.window_uid, + "element_id": elem.element_id, + "selector": selector_str, + "bounds": elem.bounds.to_dict(), + } + + # Confirmation gate (no-op until P5 plumbs confirms in). + confirm_check = _check_confirmation(ctx, action_name, args, target) + if confirm_check is not None: + return {**confirm_check, "step_id": step_id} + + started = time.time() + executor_result: Dict[str, Any] + if dry_run: + executor_result = {"success": True, "dry_run": True} + else: + try: + executor_result = executor(elem, info, args) + except Exception as e: + logger.exception(f"{action_name} executor failed") + executor_result = {"success": False, "error": str(e)} + + duration_ms = int((time.time() - started) * 1000) + after_windows = ctx.observer.list_windows() + info_after = ctx.observer.window_by_uid(after_windows, info.window_uid) + # Post-action re-read must bypass the tree cache so the receipt's + # after-state reflects reality (the fresh capture refreshes the cache). + after_tree = (ctx.observer.get_element_tree(info_after.handle, + window_uid=info_after.window_uid, + use_cache=False) + if info_after else None) + + ok = bool(executor_result.get("success", True)) + err_obj: Optional[Dict[str, Any]] = None + if not ok: + err_obj = { + "code": executor_result.get("error_code", Code.INTERNAL), + "message": str(executor_result.get("error", "action failed")), + "recoverable": False, + "suggested_next_tool": None, + "context": {}, + } + + extra: Dict[str, Any] = {} + if "warning" in (res.warning or ""): + extra["warning"] = res.warning + if res.warning: + extra["warning"] = res.warning + + return _build_receipt( + step_id=step_id, action=action_name, target=target, + before_tree=tree, before_windows=windows, + after_tree=after_tree, after_windows=after_windows, + duration_ms=duration_ms, dry_run=dry_run, ok=ok, + extra=extra, error=err_obj, + ) + + +def click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + button = args.get("button", "left") + count = int(args.get("count", 1)) + + def _exec(elem: UIElement, info: WindowInfo, _args: Dict[str, Any] + ) -> Dict[str, Any]: + cx, cy = elem.bounds.center_x, elem.bounds.center_y + result = ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": cx, "y": cy, "button": button, + "double": (count >= 2)}, + hwnd=info.handle, + ) + return result + + return _do_element_action(ctx, action_name="click_element", + args=args, executor=_exec) + + +def focus_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + # Focus via center click (universal fallback). Adapter-specific + # SetFocus paths live in the platform adapters; today the universal + # click is sufficient on all three platforms. + return ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + return _do_element_action(ctx, action_name="focus_element", + args=args, executor=_exec) + + +def set_value(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + if "value" not in args: + step_id, _ = _new_step_id("set_value") + return error_dict(Code.BAD_REQUEST, "value is required", + step_id=step_id) + value = args["value"] + clear_first = bool(args.get("clear_first", True)) + + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + if clear_first: + ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) + ctx.observer.perform_action("key", value="delete", hwnd=info.handle) + return ctx.observer.perform_action("type", value=str(value), + hwnd=info.handle) + return _do_element_action(ctx, action_name="set_value", + args=args, executor=_exec) + + +def invoke_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + # No platform-specific InvokePattern surface yet; behaves like click. + return _do_element_action( + ctx, action_name="invoke_element", args=args, + executor=lambda elem, info, _a: ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ), + ) + + +def select_option(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Click the element to open the menu, then click the matching child.""" + option_name = args.get("option_name") + option_index = args.get("option_index") + if option_name is None and option_index is None: + step_id, _ = _new_step_id("select_option") + return error_dict(Code.BAD_REQUEST, + "option_name or option_index is required", + step_id=step_id) + + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + # Re-walk to see the now-visible option list (bypass the cache — + # the click above just changed the UI). + new_tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + target: Optional[UIElement] = None + if new_tree is not None: + descendants = [] + stack = [new_tree] + while stack: + e = stack.pop() + descendants.append(e) + stack.extend(e.children) + if option_name is not None: + target = next((d for d in descendants if d.name == option_name), None) + elif option_index is not None: + idx = int(option_index) + if 0 <= idx < len(elem.children): + target = elem.children[idx] + if target is None: + return {"success": False, + "error": "option not found after opening selector"} + return ctx.observer.perform_action( + "click_at", + value={"x": target.bounds.center_x, "y": target.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + + return _do_element_action(ctx, action_name="select_option", + args=args, executor=_exec) + + +def click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("click_at") + before_windows = ctx.observer.list_windows() + started = time.time() + result = ctx.observer.perform_action("click_at", value={ + "x": args.get("x", 0), "y": args.get("y", 0), + "button": args.get("button", "left"), + "double": args.get("double", False), + }) + duration_ms = int((time.time() - started) * 1000) + after_windows = ctx.observer.list_windows() + out = annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + out.setdefault("action", "click_at") + out["duration_ms"] = duration_ms + out["new_dialogs"] = _new_dialogs(before_windows, after_windows) + return out + + +def type_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("type_text") + result = ctx.observer.perform_action("type", value=args.get("text", "")) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def press_key(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("press_key") + result = ctx.observer.perform_action("key", value=args.get("keys", "")) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def scroll(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("scroll") + result = ctx.observer.perform_action("scroll", value=args) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def bring_to_foreground(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("bring_to_foreground") + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + result = ctx.observer.bring_to_foreground(info.handle, windows) + result["window"] = info.title + result["window_uid"] = info.window_uid + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def _compose_observe(ctx: ToolContext, base_result: Dict[str, Any], + wait_after_ms: int, since_token: Optional[str]) -> None: + if wait_after_ms > 0: + time.sleep(wait_after_ms / 1000.0) + args: Dict[str, Any] = {} + target = base_result.get("target") or {} + if target.get("window_uid"): + args["window_uid"] = target["window_uid"] + if since_token: + args["since"] = since_token + obs = observe_window(ctx, args) + base_result["observation"] = obs + + +def click_element_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = click_element(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def type_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = type_text(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def press_key_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = press_key(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def hover_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("hover_at") + x = int(args.get("x", 0)) + y = int(args.get("y", 0)) + hover_ms = int(args.get("hover_ms", 250)) + try: + import pyautogui + pyautogui.moveTo(x, y) + time.sleep(hover_ms / 1000.0) + ok = True + except Exception: + ok = False + return {"ok": ok, "success": ok, "action": "hover_at", + "step_id": step_id, "caused_by_step_id": step_id, + "x": x, "y": y, "hover_ms": hover_ms} + + +def hover_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + hover_ms = int(args.get("hover_ms", 250)) + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + try: + import pyautogui + pyautogui.moveTo(elem.bounds.center_x, elem.bounds.center_y) + time.sleep(hover_ms / 1000.0) + return {"success": True, "action": "hover", "hover_ms": hover_ms} + except Exception as e: + return {"success": False, "error": str(e)} + return _do_element_action(ctx, action_name="hover_element", + args=args, executor=_exec) + + +def right_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_at(ctx, dict(args, button="right")) + + +def right_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_element(ctx, dict(args, button="right")) + + +def double_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_at(ctx, dict(args, double=True)) + + +def double_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_element(ctx, dict(args, count=2)) + + +def drag(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("drag") + src = args.get("from") or {} + dst = args.get("to") or {} + modifiers = list(args.get("modifiers") or []) + # Resolve element references on either end. + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + tree = (ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if info else None) + + def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: + if "x" in spec and "y" in spec: + return int(spec["x"]), int(spec["y"]) + eid = spec.get("element_id") + sel_text = spec.get("selector") + if tree is None: + return None + if sel_text: + try: + resu = sel.resolve(tree, sel.parse(sel_text)) + if resu.matches: + e = resu.matches[0] + return e.bounds.center_x, e.bounds.center_y + except sel.SelectorParseError: + return None + if eid: + e = _find_by_id(tree, eid) + if e is not None: + return e.bounds.center_x, e.bounds.center_y + return None + + p1 = _to_xy(src) + p2 = _to_xy(dst) + if p1 is None or p2 is None: + return error_dict(Code.BAD_REQUEST, + "drag requires from/to as {x,y} or {selector|element_id}", + step_id=step_id) + + # §21 confirmation gate: drag endpoints addressed by selector/element_id + # resolve to concrete elements, so a confirmation_required rule matching + # either endpoint (e.g. a "Trash" drop target) must demand a token, just + # like the other element-targeted verbs. The first matching endpoint is + # validated (propose_action(action="drag", args={"selector": …}) issues + # the token for that endpoint's selector). + if tree is not None and info is not None: + for spec in (src, dst): + if not (spec.get("selector") or spec.get("element_id")): + continue + elem, selector_str, res_err = _resolve_element(tree, spec) + if res_err or elem is None: + continue + tgt = {"window_uid": info.window_uid, + "element_id": elem.element_id, + "selector": selector_str, + "bounds": elem.bounds.to_dict()} + if _confirmation_rules_match(ctx, tgt): + confirm_check = _check_confirmation(ctx, "drag", args, tgt) + if confirm_check is not None: + return {**confirm_check, "step_id": step_id} + break # token validated for the matching endpoint + + duration = float(args.get("duration_s", 0.5)) + path = [p1, ((p1[0] + p2[0]) // 2, (p1[1] + p2[1]) // 2), p2] + try: + import pyautogui + for k in modifiers: + pyautogui.keyDown(k) + try: + pyautogui.moveTo(*p1) + pyautogui.dragTo(*p2, duration=duration, button="left") + finally: + for k in modifiers: + pyautogui.keyUp(k) + ok = True + err = None + except Exception as e: + ok = False + err = str(e) + out = {"ok": ok, "success": ok, "action": "drag", + "step_id": step_id, "caused_by_step_id": step_id, + "from": list(p1), "to": list(p2), + "modifiers": modifiers, "path": [list(p) for p in path]} + if err: + out["error"] = {"code": Code.INTERNAL, "message": err, + "recoverable": False, "suggested_next_tool": None, + "context": {}} + return out + + +def key_into_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + keys = args.get("keys", "") + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + return ctx.observer.perform_action("key", value=keys, hwnd=info.handle) + return _do_element_action(ctx, action_name="key_into_element", + args=args, executor=_exec) + + +def clear_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) + return ctx.observer.perform_action("key", value="delete", + hwnd=info.handle) + return _do_element_action(ctx, action_name="clear_text", + args=args, executor=_exec) diff --git a/tools/context.py b/tools/context.py new file mode 100644 index 0000000..f09537d --- /dev/null +++ b/tools/context.py @@ -0,0 +1,142 @@ +""" +Tool execution context and window/element resolution helpers. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from observer import ScreenObserver, UIElement, WindowInfo, WindowResolution +from session import Session, get_session + +logger = logging.getLogger(__name__) + + +@dataclass +class ToolContext: + observer: ScreenObserver + renderer: Any + describer: Any + config: Dict[str, Any] + + @property + def session(self) -> Session: + return get_session() + + +def _is_input_tool(name: str) -> bool: + return name in { + "click_at", "type_text", "press_key", "scroll", "bring_to_foreground", + "click_element", "focus_element", "set_value", "invoke_element", + "select_option", "hover_at", "hover_element", + "right_click_at", "right_click_element", + "double_click_at", "double_click_element", + "drag", "key_into_element", "clear_text", + "click_element_and_observe", "type_and_observe", "press_key_and_observe", + } + + +def _new_step_id(name: str) -> Tuple[int, Optional[int]]: + return get_session().steps.next_id(is_input=_is_input_tool(name)) + + +def _resolve_window(ctx: ToolContext, args: Dict[str, Any] + ) -> Tuple[List[WindowInfo], WindowResolution]: + windows = ctx.observer.list_windows() + res = ctx.observer.resolve_window( + windows, + window_uid=args.get("window_uid"), + window_index=args.get("window_index"), + window_title=args.get("window_title"), + ) + return windows, res + + +def _focused_window(windows: List[WindowInfo]) -> Optional[WindowInfo]: + for w in windows: + if w.is_focused: + return w + return windows[0] if windows else None + + +def _resolve_element(tree: UIElement, args: Dict[str, Any] + ) -> Tuple[Optional[UIElement], Optional[str], Optional[Dict]]: + """ + Returns (element, selector_string, error_dict). Either *element* is set + or *error_dict* is. The selector string is resolved or derived. + """ + selector = args.get("selector") + element_id = args.get("element_id") + + # Try selector first, then element_id. When both are provided, element_id + # acts as a fallback so a bad/unmatched selector doesn't block the call. + selector_err = None + if selector: + try: + parsed = sel.parse(selector) + result = sel.resolve(tree, parsed) + if result.matches: + return result.matches[0], parsed.canonical(), None + selector_err = error_dict( + Code.ELEMENT_NOT_FOUND, + f"no element matches selector {selector!r}", + selector=selector, + ) + except sel.SelectorParseError as e: + selector_err = error_dict(Code.BAD_REQUEST, + f"selector parse error: {e}", + selector=selector) + + if element_id: + elem = _find_by_id(tree, element_id) + if elem is not None: + derived = sel.selector_for(tree, element_id) or "" + return elem, derived, None + # element_id didn't match any internal ID — try parsing it as a selector + # (LLMs often pass the display-format string e.g. 'TabItem "name"' as an id). + try: + parsed = sel.parse(element_id) + result = sel.resolve(tree, parsed) + if result.matches: + return result.matches[0], parsed.canonical(), None + except Exception as e: + logger.debug(f"element_id {element_id!r} not parseable as a " + f"selector either: {e}") + # Both failed — prefer selector error if we have one (more informative). + if selector_err: + return None, None, selector_err + return None, None, error_dict( + Code.ELEMENT_NOT_FOUND, + f"no element with id {element_id!r}", + element_id=element_id, + ) + + if selector_err: + return None, None, selector_err + + return None, None, error_dict(Code.BAD_REQUEST, + "either element_id or selector is required") + + +def _find_by_id(elem: UIElement, target: str) -> Optional[UIElement]: + if elem.element_id == target: + return elem + for c in elem.children: + r = _find_by_id(c, target) + if r is not None: + return r + return None + + +def _new_dialogs(before: List[WindowInfo], after: List[WindowInfo]) -> List[Dict]: + before_uids = {w.window_uid for w in before} + return [ + {"window_uid": w.window_uid, "title": w.title} + for w in after if w.window_uid and w.window_uid not in before_uids + ] diff --git a/tools/dispatch.py b/tools/dispatch.py new file mode 100644 index 0000000..4d82755 --- /dev/null +++ b/tools/dispatch.py @@ -0,0 +1,268 @@ +""" +Dispatch table, allowlist gate and cross-cutting hooks. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Dict, Optional + +from errors import Code, error_dict +from hashing import tree_hash +from redaction import mark_untrusted +from session import get_session + +from tools.context import ToolContext, _is_input_tool, _new_step_id +from tools import actions, meta, observe, snapshots, trace_replay, vision + +logger = logging.getLogger(__name__) + + +REGISTRY: Dict[str, Callable[[ToolContext, Dict[str, Any]], Dict[str, Any]]] = { + # Read-only + "list_windows": meta.list_windows, + "get_capabilities": meta.get_capabilities, + "get_monitors": meta.get_monitors, + "find_element": observe.find_element, + "get_window_structure": observe.get_window_structure, + "get_screenshot": vision.get_screenshot, + "get_visible_areas": observe.get_visible_areas, + + # Element-targeted actions + "click_element": actions.click_element, + "focus_element": actions.focus_element, + "set_value": actions.set_value, + "invoke_element": actions.invoke_element, + "select_option": actions.select_option, + + # Legacy actions + "click_at": actions.click_at, + "type_text": actions.type_text, + "press_key": actions.press_key, + "scroll": actions.scroll, + "bring_to_foreground": actions.bring_to_foreground, + + # P2: sync, diff, snapshots, composites + "observe_window": observe.observe_window, + "snapshot": snapshots.snapshot, + "snapshot_get": snapshots.snapshot_get, + "snapshot_diff": snapshots.snapshot_diff, + "snapshot_drop": snapshots.snapshot_drop, + "wait_for": snapshots.wait_for, + "wait_idle": snapshots.wait_idle, + "click_element_and_observe": actions.click_element_and_observe, + "type_and_observe": actions.type_and_observe, + "press_key_and_observe": actions.press_key_and_observe, + + # P3 + "get_screenshot_cropped": vision.get_screenshot_cropped, + "get_ocr": vision.get_ocr, + "get_screen_description": vision.get_screen_description, + + # P4: tracing, replay, scenarios, oracles + "trace_start": trace_replay.trace_start, + "trace_stop": trace_replay.trace_stop, + "trace_status": trace_replay.trace_status, + "replay_start": trace_replay.replay_start, + "replay_step": trace_replay.replay_step, + "replay_status": trace_replay.replay_status, + "replay_stop": trace_replay.replay_stop, + "load_scenario": trace_replay.load_scenario, + "assert_state": trace_replay.assert_state, + + # P5: budgets, propose_action, status reporters + "get_budget_status": meta.get_budget_status, + "get_redaction_status": meta.get_redaction_status, + "propose_action": meta.propose_action, + + # P6: extra input verbs + "hover_at": actions.hover_at, + "hover_element": actions.hover_element, + "right_click_at": actions.right_click_at, + "right_click_element": actions.right_click_element, + "double_click_at": actions.double_click_at, + "double_click_element": actions.double_click_element, + "drag": actions.drag, + "key_into_element": actions.key_into_element, + "clear_text": actions.clear_text, +} + + +_ALLOWLIST_TOOLS = { + "get_capabilities", "get_monitors", "get_budget_status", + "get_redaction_status", "trace_status", "replay_status", + "list_windows", +} + + +def _check_allowlist(ctx: ToolContext, name: str + ) -> Optional[Dict[str, Any]]: + actions = ctx.config.get("actions") or {} + allow = set(actions.get("allow") or []) + deny = set(actions.get("deny") or []) + default = actions.get("default", "allow") + if name in _ALLOWLIST_TOOLS: + return None + if name in deny: + return error_dict(Code.PERMISSION_DENIED, + f"tool {name!r} is in actions.deny") + if allow and name not in allow and default == "deny": + return error_dict(Code.PERMISSION_DENIED, + f"tool {name!r} is not in actions.allow") + if not allow and default == "deny": + return error_dict(Code.PERMISSION_DENIED, + f"actions.default is 'deny' and no allowlist matches {name!r}") + return None + + +def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any]: + fn = REGISTRY.get(name) + if fn is None: + return error_dict(Code.BAD_REQUEST, f"unknown tool: {name}") + + blocked = _check_allowlist(ctx, name) + if blocked is not None: + sid, _ = _new_step_id(name) + blocked["step_id"] = sid + return blocked + + started = time.time() + sess = get_session() + tree_before = "" + if sess.active_trace is not None and not sess.active_trace.closed: + try: + windows0 = ctx.observer.list_windows() + focused0 = next((w for w in windows0 if w.is_focused), None) + if focused0: + t = ctx.observer.get_element_tree( + focused0.handle, window_uid=focused0.window_uid) + if t: + tree_before = tree_hash(t) + except Exception as e: + logger.debug(f"trace: pre-call tree hash unavailable: {e}") + + try: + result = fn(ctx, args or {}) + except Exception as e: + logger.exception(f"tool {name} crashed") + result = error_dict(Code.INTERNAL, f"{type(e).__name__}: {e}") + + duration_ms = int((time.time() - started) * 1000) + + # P1 tree cache invalidation — the single choke point. After any input + # tool (including bring_to_foreground) the affected window's cached tree + # is stale; drop it so subsequent reads re-walk. When the target window + # cannot be determined (legacy coordinate tools), drop everything. + if _is_input_tool(name): + try: + uid = None + if isinstance(result, dict): + uid = ((result.get("target") or {}).get("window_uid") + or result.get("window_uid")) + uid = uid or (args or {}).get("window_uid") + if uid: + sess.tree_cache.invalidate(uid) + else: + sess.tree_cache.invalidate_all() + except Exception: + logger.exception("tree cache invalidation failed") + + # Recurse-safety: don't trace meta tools that would recurse forever. + if name not in {"trace_start", "trace_stop", "trace_status", + "replay_start", "replay_step", "replay_status", + "replay_stop"}: + if sess.active_trace is not None and not sess.active_trace.closed: + try: + import tracing as _tracing + shot_full = ctx.observer.get_full_display_screenshot() + shot_window = None + tgt_uid = None + tgt = result.get("target") or {} + tgt_uid = tgt.get("window_uid") or args.get("window_uid") + if tgt_uid: + win = ctx.observer.window_by_uid( + ctx.observer.list_windows(), tgt_uid) + if win: + shot_window = ctx.observer.get_screenshot(win.handle) + tree_after = "" + try: + after_w = ctx.observer.list_windows() + f = next((w for w in after_w if w.is_focused), None) + if f: + t = ctx.observer.get_element_tree( + f.handle, window_uid=f.window_uid, + use_cache=False) + if t: + tree_after = tree_hash(t) + except Exception as e: + logger.debug(f"trace: post-call tree hash " + f"unavailable: {e}") + _tracing.record( + sess.active_trace, + tool=name, caller=args.get("_caller", "unknown"), + args={k: v for k, v in (args or {}).items() + if not k.startswith("_")}, + result=result, duration_ms=duration_ms, + tree_hash_before=tree_before, tree_hash_after=tree_after, + full_screenshot=shot_full, + window_screenshot=shot_window, + ) + except Exception: + logger.exception("trace.record failed") + + # Apply redaction to text-bearing read-only results. + if sess.redactor is not None: + try: + result = _apply_redaction(name, result, sess.redactor) + except Exception: + logger.exception("redaction failed") + + # Untrusted-content marking (always on): screen-derived text is + # attacker-influenced data (prompt injection), never instructions. + # Flag it and strip ANSI/control characters. + try: + result = mark_untrusted(name, result) + except Exception: + logger.exception("untrusted-content marking failed") + + # Budget accounting. + if sess.budgets is not None: + try: + sess.budgets.note(name, result) + except Exception: + logger.exception("budget accounting failed") + + # Audit log. + if sess.auditor is not None: + try: + sess.auditor.record( + tool=name, caller=args.get("_caller", "unknown"), + args=args or {}, result=result, + ) + except Exception: + logger.exception("audit failed") + + return result + + +def _apply_redaction(tool: str, result: Dict[str, Any], redactor: Any + ) -> Dict[str, Any]: + if not redactor.is_active() or not isinstance(result, dict): + return result + if tool == "get_window_structure" and "tree" in result: + title = result.get("window") or "" + result["tree"] = redactor.redact_tree(result["tree"], title) + elif tool == "observe_window": + if "tree" in result and isinstance(result["tree"], dict): + result["tree"] = redactor.redact_tree(result["tree"], + result.get("window") or "") + elif tool == "get_screen_description": + if isinstance(result.get("description"), str): + result["description"] = redactor.redact_ocr_text(result["description"]) + elif tool == "get_ocr": + if isinstance(result.get("words"), list): + result["words"] = redactor.redact_ocr_words(result["words"]) + return result diff --git a/tools/meta.py b/tools/meta.py new file mode 100644 index 0000000..5ebb532 --- /dev/null +++ b/tools/meta.py @@ -0,0 +1,133 @@ +""" +Introspection tools: windows, capabilities, status, proposals. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +from typing import Any, Dict + +from errors import Code, error_dict +from session import get_session + +from tools.context import ( + ToolContext, _focused_window, _new_step_id, _resolve_element, + _resolve_window, +) +from tools.vision import _apply_crop + +logger = logging.getLogger(__name__) + + +def list_windows(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("list_windows") + try: + windows = ctx.observer.list_windows() + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "is_mock": ctx.observer.is_mock, + "count": len(windows), + "windows": [{"index": i, **w.to_dict()} for i, w in enumerate(windows)], + } + except Exception as e: + logger.exception("list_windows failed") + return error_dict(Code.INTERNAL, str(e), step_id=step_id) + + +def get_capabilities(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_capabilities") + out = ctx.observer.get_capabilities() + out.update({"step_id": step_id, "caused_by_step_id": caused_by, "success": True}) + return out + + +def get_monitors(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_monitors") + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "monitors": ctx.observer.get_monitors(), + } + + +def get_budget_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_budget_status") + sess = get_session() + if sess.budgets is None: + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "configured": False} + out = sess.budgets.status() + out.update({"ok": True, "success": True, "configured": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return out + + +def get_redaction_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_redaction_status") + sess = get_session() + out = sess.redactor.status() if sess.redactor else {"enabled": False, "active": False} + out.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return out + + +def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Issue a single-use confirm_token for a destructive action.""" + step_id, caused_by = _new_step_id("propose_action") + action = args.get("action") + inner_args = args.get("args") or {} + if not action: + return error_dict(Code.BAD_REQUEST, "action is required", + step_id=step_id) + + windows, res = _resolve_window(ctx, inner_args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id) + elem, selector_str, err = _resolve_element(tree, inner_args) + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} + + sess = get_session() + bbox = elem.bounds.to_dict() + ct = sess.confirms.issue( + action=action, window_uid=info.window_uid, + selector=selector_str or "", bbox=bbox, args=inner_args, + ) + # Optional preview crop. + preview_b64 = None + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot is not None: + crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, + max_width=400) + if crop_bytes: + preview_b64 = base64.b64encode(crop_bytes).decode() + except Exception as e: + logger.debug(f"propose_action: preview crop failed: {e}") + + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "confirm_token": ct.token, + "expires_at": ct.expires_at, + "would_target": { + "window_uid": info.window_uid, + "selector": selector_str, + "bounds": bbox, + "screenshot_b64": preview_b64, + }, + } diff --git a/tools/observe.py b/tools/observe.py new file mode 100644 index 0000000..b3d5d3f --- /dev/null +++ b/tools/observe.py @@ -0,0 +1,543 @@ +""" +Tree observation: structure, diffs, filtering/paging helpers. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +import time +from typing import Any, Dict, List, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from hashing import tree_hash +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ( + ToolContext, _focused_window, _new_step_id, _resolve_window, +) + + +def find_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("find_element") + selector_text = args.get("selector") + if not selector_text: + return error_dict(Code.BAD_REQUEST, "selector is required", + step_id=step_id) + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + try: + parsed = sel.parse(selector_text) + except sel.SelectorParseError as e: + return error_dict(Code.BAD_REQUEST, f"selector parse error: {e}", + step_id=step_id) + result = sel.resolve(tree, parsed) + if not result.matches: + return error_dict(Code.ELEMENT_NOT_FOUND, + f"no element matches {selector_text!r}", + step_id=step_id, selector=selector_text, + window_uid=info.window_uid) + first = result.matches[0] + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, + "element_id": first.element_id, + "selector": parsed.canonical(), + "bounds": first.bounds.to_dict(), + "ambiguous_matches": len(result.matches), + "all_matches": [ + {"element_id": m.element_id, "bounds": m.bounds.to_dict(), + "name": m.name, "role": m.role} + for m in result.matches + ], + } + + +def _effective_depth(ctx: ToolContext, args: Dict[str, Any]) -> int: + """Depth to return: caller's depth= (clamped to tree.max_depth) or + tree.default_depth when the caller passes none.""" + tree_cfg = ctx.config.get("tree", {}) or {} + hard_cap = int(tree_cfg.get("max_depth", 8)) + requested = args.get("depth") + if requested is None: + depth = int(tree_cfg.get("default_depth", 5)) + else: + try: + depth = int(requested) + except (TypeError, ValueError): + depth = int(tree_cfg.get("default_depth", 5)) + return max(0, min(depth, hard_cap)) + + +def _truncate_depth(node: Optional[Dict[str, Any]], max_depth: int, + _depth: int = 0) -> Tuple[Optional[Dict[str, Any]], bool]: + """Copy *node* limited to *max_depth* levels below it. + + Nodes whose children were dropped are marked ``truncated: true`` with a + ``child_count`` so the caller knows to drill in (via scope=/depth=). + Returns (tree, any_node_truncated). The input dict is not mutated.""" + if node is None: + return None, False + out = dict(node) + children = node.get("children") or [] + if _depth >= max_depth and children: + out["children"] = [] + out["truncated"] = True + out["child_count"] = len(children) + return out, True + truncated_any = False + new_children: List[Dict[str, Any]] = [] + for c in children: + nc, t = _truncate_depth(c, max_depth, _depth + 1) + if nc is not None: + new_children.append(nc) + truncated_any = truncated_any or t + out["children"] = new_children + return out, truncated_any + + +def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_window_structure") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + + depth = _effective_depth(ctx, args) + scope = args.get("scope") + token = None + + if scope: + # Drill into one branch only (element-id path, e.g. 'root.3.2'). + ttl = float((ctx.config.get("tree", {}) or {}).get("cache_ttl_s", 2.0)) + had_fresh_entry = (get_session().tree_cache.get( + info.window_uid, ttl_s=ttl) is not None) + started = time.time() + tree = ctx.observer.get_element_subtree( + info.handle, scope, max_depth=depth, + window_uid=info.window_uid) + capture_ms = int((time.time() - started) * 1000) + if tree is None: + return error_dict(Code.ELEMENT_NOT_FOUND, + f"no element with id {scope!r} to scope to", + step_id=step_id, scope=scope, + window_uid=info.window_uid) + perf = {"capture_ms": capture_ms, + "node_count": len(tree.flat_list()), + "cache": "hit" if had_fresh_entry else "miss", + "depth_used": depth} + # scoped captures are not valid diff baselines → no tree_token + else: + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + perf = {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} + full_serialized = tree.to_dict() + th = tree_hash(tree) + if not scope: + token = get_session().tree_tokens.put(info.window_uid, + full_serialized, th) + serialized, depth_truncated = _truncate_depth(full_serialized, depth) + + # P3 filtering / paging -------------------------------------------------- + roles = args.get("roles") + exclude_roles = args.get("exclude_roles") + visible_only = bool(args.get("visible_only")) + name_regex = args.get("name_regex") + max_text_len = args.get("max_text_len") + prune_empty = bool(args.get("prune_empty")) + max_nodes = args.get("max_nodes") + page_cursor = args.get("page_cursor") + + visible_regions: Optional[List[Dict[str, int]]] = None + if visible_only: + try: + visible_regions = ctx.observer.get_visible_areas(info.handle, windows) + except Exception: + visible_regions = [] + + filtered = _filter_tree( + serialized or {}, + roles=set(roles) if roles else None, + exclude_roles=set(exclude_roles) if exclude_roles else None, + visible_regions=visible_regions, + name_regex=name_regex, + max_text_len=max_text_len, + prune_empty=prune_empty, + ) + + truncated = False + next_cursor: Optional[str] = None + node_count = _count_nodes(filtered) if filtered else 0 + if max_nodes is not None or page_cursor is not None: + filtered, truncated, next_cursor, node_count = _page_tree( + filtered, max_nodes=max_nodes, page_cursor=page_cursor, + ) + + out = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, + "window_uid": info.window_uid, + "element_count": len(tree.flat_list()), + "node_count": node_count, + "tree": filtered, + "tree_hash": th, + "tree_token": token, + "truncated": truncated, + "next_cursor": next_cursor, + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": perf, + } + if scope: + out["scope"] = scope + else: + # Degradation signal: accessibility-dark windows (games, custom + # renderers) produce sparse trees — steer the agent to pixel-based + # fallbacks instead of letting it act on a near-empty tree. + named = sum(1 for e in tree.flat_list()[1:] if (e.name or "").strip()) + threshold = int((ctx.config.get("tree", {}) or {}) + .get("sparse_threshold", 5)) + if named < threshold: + out["degraded"] = { + "reason": "sparse_accessibility_tree", + "named_node_count": named, + "threshold": threshold, + "suggested_fallbacks": ["get_ocr", "get_screen_description"], + } + return out + + +def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_visible_areas") + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + areas = ctx.observer.get_visible_areas(info.handle, windows) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, + "window_uid": info.window_uid, + "visible_regions": areas, + } + + +def _perf_dict(meta: Dict[str, Any], depth: Optional[int]) -> Dict[str, Any]: + return {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} + + +def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, + depth: Optional[int] = None, + ) -> Tuple[Optional[UIElement], Dict[str, Any]]: + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return None, {"error": "no tree"} + serialized = tree.to_dict() + th = tree_hash(tree) + # Diff baselines keep the full capture; only the returned tree is + # depth-bounded (with truncated-node markers). + token = get_session().tree_tokens.put(info.window_uid, serialized, th) + out_tree: 'Optional[Dict[str, Any]]' = serialized + depth_truncated = False + if depth is not None: + out_tree, depth_truncated = _truncate_depth(serialized, depth) + return tree, { + "format": "full", + "window_uid": info.window_uid, + "window": info.title, + "tree": out_tree, + "tree_hash": th, + "tree_token": token, + "base_token": None, + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), + } + + +def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Return the current tree, optionally as a diff against a tree_token.""" + from diff import diff_custom, diff_json_patch + step_id, caused_by = _new_step_id("observe_window") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + since = args.get("since") + fmt = args.get("format", "custom") + depth = _effective_depth(ctx, args) + changed_only = bool(args.get("changed_only")) + + if not since and changed_only: + return _observe_changed_only(ctx, info, depth=depth, + step_id=step_id, caused_by=caused_by) + + if not since: + _, full = _serialize_full_observation(ctx, info, depth=depth) + full.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "format": "full"}) + return full + + entry = get_session().tree_tokens.get(since) + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + serialized = tree.to_dict() + th = tree_hash(tree) + new_token = get_session().tree_tokens.put(info.window_uid, serialized, th) + + if entry is None or entry.window_uid != info.window_uid: + # Token expired/wrong-window: return full tree (depth-bounded). + out_tree, depth_truncated = _truncate_depth(serialized, depth) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree": out_tree, "tree_hash": th, + "tree_token": new_token, "base_token": None, + "format": "full", + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), + } + + if fmt == "json-patch": + changes = diff_json_patch(entry.serialized, serialized) + out_format = "json-patch" + else: + changes = diff_custom(entry.serialized, serialized) + out_format = "custom" + + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree_token": new_token, "base_token": since, + "format": out_format, + "changes": changes, + "unchanged": len(changes) == 0, + "tree_hash": th, + "perf": _perf_dict(meta, depth), + } + + +def _observe_changed_only(ctx: ToolContext, info: WindowInfo, *, depth: int, + step_id: int, caused_by: Optional[int] + ) -> Dict[str, Any]: + """observe_window changed_only=true: compare a fresh capture against the + last cached capture of the window. Unchanged → a tiny + {unchanged: true, tree_hash} response; changed → a custom diff instead + of the full tree; no baseline → full tree.""" + from diff import diff_custom + sess = get_session() + # Baseline: the most recent capture, regardless of cache TTL. + baseline = sess.tree_cache.peek(info.window_uid) + + # Fresh capture (bypass the cache — the whole point is to detect drift). + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid, use_cache=False) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + serialized = tree.to_dict() + th = tree_hash(tree) + new_token = sess.tree_tokens.put(info.window_uid, serialized, th) + perf = _perf_dict(meta, depth) + + base: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree_hash": th, "tree_token": new_token, + "changed_only": True, + "perf": perf, + } + + if baseline is None: + # Nothing to compare against — return the (depth-bounded) full tree. + out_tree, depth_truncated = _truncate_depth(serialized, depth) + base.update({"format": "full", "tree": out_tree, "base_token": None, + "depth_used": depth, "depth_truncated": depth_truncated}) + return base + + if baseline.tree_hash == th: + base["unchanged"] = True + return base + + changes = diff_custom(baseline.serialized, serialized) + base.update({"format": "custom", "changes": changes, "unchanged": False}) + return base + + +def _filter_tree(node: Dict[str, Any], *, roles: Optional[set], + exclude_roles: Optional[set], + visible_regions: Optional[List[Dict[str, int]]], + name_regex: Optional[str], + max_text_len: Optional[int], + prune_empty: bool) -> Optional[Dict[str, Any]]: + if node is None: + return None + role = node.get("role") + name = node.get("name") or "" + bounds = node.get("bounds") or {} + + # Role filter + role_keep = True + if roles is not None and role not in roles: + role_keep = False + if exclude_roles is not None and role in exclude_roles: + role_keep = False + + # Name regex + name_keep = True + if name_regex: + try: + name_keep = bool(re.search(name_regex, name)) + except re.error: + name_keep = True + + # Visibility + visible_keep = True + if visible_regions is not None: + visible_keep = _intersects_any(bounds, visible_regions) + + self_keep = role_keep and name_keep and visible_keep + + # Recurse children regardless (so we can keep ancestors if descendants match) + new_children: List[Dict[str, Any]] = [] + for c in node.get("children", []) or []: + fc = _filter_tree( + c, roles=roles, exclude_roles=exclude_roles, + visible_regions=visible_regions, name_regex=name_regex, + max_text_len=max_text_len, prune_empty=prune_empty, + ) + if fc is not None: + new_children.append(fc) + + if prune_empty and not self_keep and not new_children: + return None + + # Truncate text fields if requested. + truncated_node = dict(node) + if max_text_len is not None: + n = int(max_text_len) + if isinstance(truncated_node.get("name"), str) and len(truncated_node["name"]) > n: + truncated_node["name"] = truncated_node["name"][:n] + "…" + v = truncated_node.get("value") + if isinstance(v, str) and len(v) > n: + truncated_node["value"] = v[:n] + "…" + truncated_node["children"] = new_children + return truncated_node + + +def _intersects_any(b: Dict[str, int], regions: List[Dict[str, int]]) -> bool: + if not b: + return False + bx, by = b.get("x", 0), b.get("y", 0) + bw, bh = b.get("width", 0), b.get("height", 0) + if bw <= 0 or bh <= 0: + return False + bx2, by2 = bx + bw, by + bh + for r in regions: + rx, ry = r.get("x", 0), r.get("y", 0) + rx2, ry2 = rx + r.get("width", 0), ry + r.get("height", 0) + if bx < rx2 and bx2 > rx and by < ry2 and by2 > ry: + return True + return False + + +def _count_nodes(node: Optional[Dict[str, Any]]) -> int: + if node is None: + return 0 + return 1 + sum(_count_nodes(c) for c in (node.get("children") or [])) + + +def _page_tree(node: Optional[Dict[str, Any]], *, + max_nodes: Optional[int], + page_cursor: Optional[str] + ) -> Tuple[Optional[Dict[str, Any]], bool, Optional[str], int]: + """ + Paginated DFS walk. Returns (subtree-shaped result containing only the + page slice, truncated flag, next_cursor, node_count_in_page). + + Cursors are post-order element_ids; resuming starts from the next sibling + in the original walk. This is a best-effort pager — if the tree changed, + callers will get SnapshotExpired-shaped semantics by virtue of an unknown + cursor returning truncated:false and node_count:0. + """ + if node is None: + return None, False, None, 0 + flat: List[Dict[str, Any]] = [] + _flatten(node, flat) + + if page_cursor is not None: + for i, n in enumerate(flat): + if n.get("id") == page_cursor: + flat = flat[i + 1:] + break + else: + return None, False, None, 0 + + if max_nodes is None or max_nodes >= len(flat): + # Return full (possibly trimmed) tree starting from cursor. + if page_cursor is None: + return node, False, None, len(flat) + return _flat_to_tree(flat), False, None, len(flat) + + page = flat[:max_nodes] + truncated = True + next_cursor = page[-1].get("id") if page else None + return _flat_to_tree(page), truncated, next_cursor, len(page) + + +def _flatten(node: Dict[str, Any], out: List[Dict[str, Any]]) -> None: + out.append(node) + for c in node.get("children") or []: + _flatten(c, out) + + +def _flat_to_tree(flat: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Wrap a list of nodes as children of a synthetic Window root.""" + if not flat: + return None + return { + "id": "page-root", + "name": "[paged]", + "role": "Group", + "value": None, + "bounds": {"x": 0, "y": 0, "width": 0, "height": 0}, + "enabled": True, "focused": False, + "keyboard_shortcut": None, "description": None, + "children": [dict(n, children=[]) for n in flat], + } diff --git a/tools/receipts.py b/tools/receipts.py new file mode 100644 index 0000000..ae08b69 --- /dev/null +++ b/tools/receipts.py @@ -0,0 +1,117 @@ +""" +Action receipts and confirmation gating. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +from errors import Code, error_dict +from hashing import focused_selector, tree_hash +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ToolContext, _new_dialogs + + +def _build_receipt(*, step_id: int, action: str, target: Dict[str, Any], + before_tree: Optional[UIElement], + before_windows: List[WindowInfo], + after_tree: Optional[UIElement], + after_windows: List[WindowInfo], + duration_ms: int, dry_run: bool, ok: bool, + extra: Optional[Dict[str, Any]] = None, + error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + before_hash = tree_hash(before_tree) if before_tree else "" + after_hash = tree_hash(after_tree) if after_tree else "" + receipt: Dict[str, Any] = { + "ok": ok, "success": ok, + "step_id": step_id, "caused_by_step_id": step_id, + "action": action, + "dry_run": dry_run, + "target": target, + "before": { + "tree_hash": before_hash, + "focused_selector": focused_selector(before_tree) if before_tree else "", + }, + "after": { + "tree_hash": after_hash, + "focused_selector": focused_selector(after_tree) if after_tree else "", + }, + "changed": (before_hash != after_hash) and not dry_run, + "new_dialogs": _new_dialogs(before_windows, after_windows), + "duration_ms": duration_ms, + } + if extra: + receipt.update(extra) + if error: + receipt["error"] = error + return receipt + + +def _confirmation_rules_match(ctx: ToolContext, + target: Dict[str, Any]) -> bool: + """True when any configured confirmation_required rule matches the + target's role/name (derived from the selector tail).""" + confirm = ctx.config.get("confirmation_required") or [] + if not confirm: + return False + name_to_test = "" + role_to_test = "" + # Best-effort: derive name/role from the selector tail. + sel_tail = (target.get("selector") or "").split("/")[-1] + m = re.match(r"([A-Za-z_*]\w*)", sel_tail) + if m: + role_to_test = m.group(1) + nm = re.search(r'name="([^"]*)"', sel_tail) + if nm: + name_to_test = nm.group(1) + + def _matches_rule(rule: Dict[str, Any]) -> bool: + rname = rule.get("name_regex") + rrole = rule.get("role") + if rrole and role_to_test != rrole: + return False + if rname and not re.search(rname, name_to_test or ""): + return False + return bool(rname or rrole) + + return any(_matches_rule(r) for r in confirm) + + +def _check_confirmation(ctx: ToolContext, action_name: str, + args: Dict[str, Any], + target: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Returns an error dict to short-circuit when confirmation is required.""" + if not _confirmation_rules_match(ctx, target): + return None + + token = args.get("confirm_token") + if not token: + return error_dict( + Code.CONFIRMATION_REQUIRED, + f"action {action_name} requires a confirm_token from propose_action", + action=action_name, target=target, + ) + sess = get_session() + ct = sess.confirms.consume(token) + if not ct: + return error_dict(Code.CONFIRMATION_INVALID, + "confirm_token expired, unknown, or already used", + token=token) + if ct.action != action_name or ct.window_uid != target["window_uid"] \ + or ct.selector != target["selector"]: + return error_dict(Code.CONFIRMATION_INVALID, + "confirm_token does not match the proposed action") + tol = (ctx.config.get("confirmation", {}) or {}).get("bbox_tolerance_px", 20) + bb = target["bounds"] + if (abs(bb["x"] - ct.bbox.get("x", 0)) > tol or + abs(bb["y"] - ct.bbox.get("y", 0)) > tol or + abs(bb["width"] - ct.bbox.get("width", 0)) > tol or + abs(bb["height"] - ct.bbox.get("height", 0)) > tol): + return error_dict(Code.CONFIRMATION_INVALID, + "element bounds drifted beyond confirmation tolerance") + return None diff --git a/tools/snapshots.py b/tools/snapshots.py new file mode 100644 index 0000000..af6ae1a --- /dev/null +++ b/tools/snapshots.py @@ -0,0 +1,278 @@ +""" +Snapshots and wait_for / wait_idle condition polling. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +import time +from typing import Any, Dict, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from hashing import tree_hash +from session import get_session + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_step_id, _resolve_window, +) + + +def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot") + windows = ctx.observer.list_windows() + trees: Dict[str, Dict[str, Any]] = {} + hashes: Dict[str, str] = {} + for w in windows: + try: + t = ctx.observer.get_element_tree(w.handle, + window_uid=w.window_uid) + if t is not None and w.window_uid: + trees[w.window_uid] = t.to_dict() + hashes[w.window_uid] = tree_hash(t) + except Exception: + continue + snap = get_session().snapshots.put( + windows=[w.to_dict() for w in windows], + trees=trees, tree_hashes=hashes, + ) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "snapshot_id": snap.snapshot_id, + "ts": snap.ts, + "summary": {"windows": len(snap.windows), "trees": len(trees)}, + } + + +def snapshot_get(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot_get") + sid = args.get("snapshot_id") + if not sid: + return error_dict(Code.BAD_REQUEST, "snapshot_id is required", + step_id=step_id) + snap = get_session().snapshots.get(sid) + if snap is None: + return error_dict(Code.SNAPSHOT_EXPIRED, + "snapshot expired or not found", + step_id=step_id, snapshot_id=sid) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "snapshot_id": snap.snapshot_id, "ts": snap.ts, + "windows": snap.windows, + "trees": snap.trees, + "tree_hashes": snap.tree_hashes, + } + + +def snapshot_diff(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + from diff import diff_custom, diff_json_patch + step_id, caused_by = _new_step_id("snapshot_diff") + a_id = args.get("a") + b_id = args.get("b") + if not a_id or not b_id: + return error_dict(Code.BAD_REQUEST, "a and b are required", + step_id=step_id) + sess = get_session() + a = sess.snapshots.get(a_id) + b = sess.snapshots.get(b_id) + if a is None or b is None: + return error_dict(Code.SNAPSHOT_EXPIRED, + "one or both snapshots are missing", + step_id=step_id) + fmt = args.get("format", "custom") + + a_uids = {w["window_uid"] for w in a.windows} + b_uids = {w["window_uid"] for w in b.windows} + windows_added = sorted(b_uids - a_uids) + windows_removed = sorted(a_uids - b_uids) + common = sorted(a_uids & b_uids) + + per_window: Dict[str, Any] = {} + for uid in common: + if uid in a.trees and uid in b.trees: + if fmt == "json-patch": + per_window[uid] = {"format": "json-patch", + "changes": diff_json_patch(a.trees[uid], b.trees[uid])} + else: + per_window[uid] = {"format": "custom", + "changes": diff_custom(a.trees[uid], b.trees[uid])} + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "windows_added": windows_added, + "windows_removed": windows_removed, + "per_window_changes": per_window, + } + + +def snapshot_drop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot_drop") + sid = args.get("snapshot_id") + dropped = get_session().snapshots.drop(sid) if sid else False + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "dropped": dropped, + } + + +def _check_condition(ctx: ToolContext, cond: Dict[str, Any], + window_uid_hint: Optional[str]) -> Tuple[bool, Dict[str, Any]]: + kind = cond.get("type") + sess = get_session() + windows = ctx.observer.list_windows() + info = ctx.observer.window_by_uid(windows, window_uid_hint) or _focused_window(windows) + + if kind == "window_appears": + rx = cond.get("title_regex", "") + for w in windows: + if re.search(rx, w.title): + return True, {"window_uid": w.window_uid, "title": w.title} + return False, {} + if kind == "window_disappears": + target = cond.get("window_uid") + for w in windows: + if w.window_uid == target: + return False, {} + return True, {"window_uid": target} + if kind == "focused_changes": + focus = next((w for w in windows if w.is_focused), None) + return (focus is not None), ({"focused_uid": focus.window_uid} if focus else {}) + if kind == "tree_changes": + token = cond.get("since") + entry = sess.tree_tokens.get(token) if token else None + if entry is None or info is None: + return False, {} + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + return tree is not None and tree_hash(tree) != entry.tree_hash, {} + + if info is None: + return False, {} + # Polling must observe fresh state — bypass the tree cache. + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + if tree is None: + return False, {} + + if kind == "element_appears": + sel_text = cond.get("selector") + if not sel_text: + return False, {} + try: + res = sel.resolve(tree, sel.parse(sel_text)) + except sel.SelectorParseError: + return False, {} + if res.matches: + m = res.matches[0] + return True, {"element_id": m.element_id, "bounds": m.bounds.to_dict()} + return False, {} + if kind == "element_disappears": + sel_text = cond.get("selector") + eid = cond.get("element_id") + if sel_text: + try: + res = sel.resolve(tree, sel.parse(sel_text)) + return not res.matches, {} + except sel.SelectorParseError: + return False, {} + if eid: + return _find_by_id(tree, eid) is None, {} + return False, {} + if kind == "text_visible": + rx = cond.get("regex", "") + # Walk tree names/values. + for elem in tree.flat_list(): + joined = (elem.name or "") + " " + (elem.value or "") + if re.search(rx, joined): + return True, {"element_id": elem.element_id} + return False, {} + return False, {} + + +def wait_for(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("wait_for") + timeout_ms = int(args.get("timeout_ms", 5000)) + cap = int((ctx.config.get("wait_for", {}) or {}).get("max_timeout_ms", 60000)) + timeout_ms = min(timeout_ms, cap) + poll_ms = max(50, int(args.get("poll_ms", 200))) + conditions = args.get("any_of", []) + if not conditions: + return error_dict(Code.BAD_REQUEST, "any_of is required", + step_id=step_id) + window_uid = args.get("window_uid") + + started = time.time() + polls = 0 + while True: + polls += 1 + for i, cond in enumerate(conditions): + try: + ok, detail = _check_condition(ctx, cond, window_uid) + except Exception: + ok, detail = False, {} + if ok: + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "matched_index": i, "matched_detail": detail, + "elapsed_ms": int((time.time() - started) * 1000), + "polls": polls, + } + elapsed = (time.time() - started) * 1000 + if elapsed >= timeout_ms: + err = error_dict( + Code.TIMEOUT, f"wait_for timed out after {int(elapsed)}ms", + step_id=step_id, + ) + err.update({ + "elapsed_ms": int(elapsed), "polls": polls, + "matched_index": None, + }) + return err + time.sleep(poll_ms / 1000.0) + + +def wait_idle(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("wait_idle") + timeout_ms = int(args.get("timeout_ms", 5000)) + quiet_ms = int(args.get("quiet_ms", 750)) + poll_ms = max(50, int(args.get("poll_ms", 100))) + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + + started = time.time() + last_hash = None + last_change_at = time.time() + while (time.time() - started) * 1000 < timeout_ms: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + if tree is None: + time.sleep(poll_ms / 1000.0) + continue + h = tree_hash(tree) + if h != last_hash: + last_hash = h + last_change_at = time.time() + elif (time.time() - last_change_at) * 1000 >= quiet_ms: + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "elapsed_ms": int((time.time() - started) * 1000), + "tree_hash": h, + } + time.sleep(poll_ms / 1000.0) + + err = error_dict(Code.TIMEOUT, "wait_idle timed out", step_id=step_id) + err["elapsed_ms"] = int((time.time() - started) * 1000) + return err diff --git a/tools/trace_replay.py b/tools/trace_replay.py new file mode 100644 index 0000000..606b4ff --- /dev/null +++ b/tools/trace_replay.py @@ -0,0 +1,183 @@ +""" +Tracing, replay, scenario loading and state oracles. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from errors import Code, error_dict +from session import get_session + +from tools.context import ToolContext, _new_step_id + + +def trace_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import tracing as _tracing + step_id, caused_by = _new_step_id("trace_start") + sess = get_session() + if sess.active_trace is not None and not sess.active_trace.closed: + return error_dict(Code.BAD_REQUEST, "trace already active", + step_id=step_id, + trace_id=sess.active_trace.trace_id) + handle = _tracing.start(label=args.get("label", ""), config=ctx.config) + sess.active_trace = handle + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "trace_id": handle.trace_id, + "started_at": handle.started_at, + "dir": handle.dir, + } + + +def trace_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import tracing as _tracing + step_id, caused_by = _new_step_id("trace_stop") + sess = get_session() + if sess.active_trace is None: + return error_dict(Code.BAD_REQUEST, "no active trace", + step_id=step_id) + info = _tracing.stop(sess.active_trace) + sess.active_trace = None + info.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return info + + +def trace_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("trace_status") + sess = get_session() + if sess.active_trace is None: + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "active_trace_id": None, "step_count": 0, "dir": None} + h = sess.active_trace + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "active_trace_id": h.trace_id, + "step_count": h.counter.value, + "dir": h.dir, + } + + +_REPLAYS: Dict[str, Any] = {} + + +def replay_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import replay as _replay + step_id, caused_by = _new_step_id("replay_start") + path = args.get("path") + if not path: + return error_dict(Code.BAD_REQUEST, "path is required", + step_id=step_id) + mode = args.get("mode", "execute") + on_div = args.get("on_divergence", "warn") + try: + rep = _replay.load(path, mode=mode, on_divergence=on_div) + except Exception as e: + return error_dict(Code.BAD_REQUEST, f"could not load trace: {e}", + step_id=step_id, path=path) + handle_id = "rep:" + str(len(_REPLAYS) + 1) + _REPLAYS[handle_id] = rep + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": handle_id, + "total": len(rep.rows), + "mode": rep.mode, + "label": rep.label, + } + + +def replay_step(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import replay as _replay + step_id, caused_by = _new_step_id("replay_step") + rid = args.get("replay_id") or "" + rep = _REPLAYS.get(rid) + if rep is None: + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + + def _disp(name: str, a: Dict[str, Any]) -> Dict[str, Any]: + # Local import: tools.dispatch imports this module to build REGISTRY. + from tools.dispatch import dispatch + return dispatch(ctx, name, a) + + out = _replay.step(rep, dispatch=_disp) + out.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": rid}) + return out + + +def replay_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("replay_status") + rid = args.get("replay_id") or "" + rep = _REPLAYS.get(rid) + if rep is None: + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": rid, + "position": rep.position, + "total": len(rep.rows), + "finished": rep.finished, + "divergences": rep.divergences, + "mode": rep.mode, + } + + +def replay_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("replay_stop") + rid = args.get("replay_id") + if rid in _REPLAYS: + _REPLAYS.pop(rid) + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "stopped": True} + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + + +def load_scenario(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import scenarios as _scn + step_id, caused_by = _new_step_id("load_scenario") + path = args.get("path") + if not path: + return error_dict(Code.BAD_REQUEST, "path is required", + step_id=step_id) + try: + sc = _scn.load(path) + _scn.attach_to_observer(sc, ctx.observer) + # The scenario replaces the mock world — cached trees are stale. + get_session().tree_cache.invalidate_all() + except _scn.ScenarioError as e: + return error_dict(Code.SCENARIO_INVALID, str(e), + step_id=step_id, path=path) + except Exception as e: + return error_dict(Code.SCENARIO_INVALID, f"{type(e).__name__}: {e}", + step_id=step_id, path=path) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "scenario": sc.name, + "state": sc.current_state, + "states": list(sc.states.keys()), + } + + +def assert_state(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import oracles as _oracles + step_id, caused_by = _new_step_id("assert_state") + pred = args.get("predicate") or args.get("predicates") or [] + out = _oracles.evaluate(ctx.observer, pred, config=ctx.config) + if out.get("ok"): + out["step_id"] = step_id + out["caused_by_step_id"] = caused_by + return out diff --git a/tools/vision.py b/tools/vision.py new file mode 100644 index 0000000..150b933 --- /dev/null +++ b/tools/vision.py @@ -0,0 +1,301 @@ +""" +Pixel-derived observations: screenshots, OCR, descriptions. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +from typing import Any, Dict, List, Optional, Tuple + +from errors import Code, error_dict +from observer import UIElement + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_step_id, _resolve_window, +) + +logger = logging.getLogger(__name__) + + +def get_screenshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_screenshot") + windows, res = _resolve_window(ctx, args) + info = res.info + hwnd = info.handle if info else None + shot = ctx.observer.get_screenshot(hwnd) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title if info else "(full screen)", + "format": "png", "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + + +def get_screenshot_cropped(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """get_screenshot with optional bbox / element_id / max_width / padding.""" + step_id, caused_by = _new_step_id("get_screenshot_cropped") + windows, res = _resolve_window(ctx, args) + info = res.info + hwnd = info.handle if info else None + shot = ctx.observer.get_screenshot(hwnd) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + + bbox: Optional[Dict[str, int]] = args.get("bbox") + element_id: Optional[str] = args.get("element_id") + padding = int(args.get("padding_px", 0)) + max_width: Optional[int] = args.get("max_width") + + if bbox is None and element_id and info is not None: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is not None: + elem = _find_by_id(tree, element_id) + if elem is not None: + # Convert to window-relative coordinates. + bbox = { + "x": max(0, elem.bounds.x - info.bounds.x), + "y": max(0, elem.bounds.y - info.bounds.y), + "width": elem.bounds.width, + "height": elem.bounds.height, + } + + if bbox or max_width: + shot, source_bbox = _apply_crop(shot, bbox, padding, max_width) + else: + source_bbox = None + + out: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title if info else "(full screen)", + "format": "png", "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + if source_bbox: + out["source_bbox"] = source_bbox + return out + + +def _apply_crop(png_bytes: bytes, bbox: Optional[Dict[str, int]], + padding: int, max_width: Optional[int] + ) -> Tuple[bytes, Optional[Dict[str, int]]]: + try: + import io as _io + from PIL import Image + except Exception: + return png_bytes, None + img: "Image.Image" = Image.open(_io.BytesIO(png_bytes)) + source_bbox: Optional[Dict[str, int]] = None + if bbox is not None: + x = max(0, int(bbox.get("x", 0)) - padding) + y = max(0, int(bbox.get("y", 0)) - padding) + x2 = min(img.width, int(bbox.get("x", 0)) + int(bbox.get("width", 0)) + padding) + y2 = min(img.height, int(bbox.get("y", 0)) + int(bbox.get("height", 0)) + padding) + if x2 > x and y2 > y: + img = img.crop((x, y, x2, y2)) + source_bbox = {"x": x, "y": y, "width": x2 - x, "height": y2 - y} + if max_width and img.width > int(max_width): + ratio = int(max_width) / float(img.width) + new_size = (int(max_width), max(1, int(img.height * ratio))) + img = img.resize(new_size) + buf = __import__("io").BytesIO() + img.save(buf, "PNG") + return buf.getvalue(), source_bbox + + +def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Region-scoped OCR; returns [{text, confidence, bbox}].""" + step_id, caused_by = _new_step_id("get_ocr") + try: + import io as _io + from PIL import Image + import pytesseract + from ocr_util import configure as _ocr_configure + _ocr_configure(ctx.config) + except Exception: + from ocr_util import INSTALL_HINT + return error_dict(Code.PLATFORM_UNSUPPORTED, + f"pytesseract / Pillow not installed. {INSTALL_HINT}", + step_id=step_id, hint=INSTALL_HINT) + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + shot = ctx.observer.get_screenshot(info.handle) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + bbox = args.get("bbox") + element_id = args.get("element_id") + if element_id and not bbox: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is not None: + elem = _find_by_id(tree, element_id) + if elem is not None: + bbox = { + "x": max(0, elem.bounds.x - info.bounds.x), + "y": max(0, elem.bounds.y - info.bounds.y), + "width": elem.bounds.width, + "height": elem.bounds.height, + } + + img: "Image.Image" = Image.open(_io.BytesIO(shot)) + if bbox: + x = max(0, int(bbox.get("x", 0))) + y = max(0, int(bbox.get("y", 0))) + x2 = min(img.width, x + int(bbox.get("width", 0))) + y2 = min(img.height, y + int(bbox.get("height", 0))) + if x2 > x and y2 > y: + img = img.crop((x, y, x2, y2)) + + min_conf = (ctx.config.get("ocr", {}) or {}).get("min_confidence", 30) + try: + data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT) + except pytesseract.TesseractNotFoundError: + from ocr_util import diagnose as _ocr_diag, INSTALL_HINT + return error_dict( + Code.PLATFORM_UNSUPPORTED, + ("tesseract binary not found — check ocr.tesseract_cmd in " + f"config.json. {INSTALL_HINT}"), + step_id=step_id, **_ocr_diag(ctx.config), + ) + except Exception as e: + return error_dict(Code.INTERNAL, f"OCR failed: {e}", + step_id=step_id) + out_words: List[Dict[str, Any]] = [] + for i, text in enumerate(data["text"]): + text = (text or "").strip() + if not text: + continue + try: + conf = int(data["conf"][i]) + except (TypeError, ValueError): + conf = 0 + if conf < min_conf: + continue + out_words.append({ + "text": text, "confidence": conf, + "bbox": {"x": int(data["left"][i]), "y": int(data["top"][i]), + "width": int(data["width"][i]), + "height": int(data["height"][i])}, + }) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, "window_uid": info.window_uid, + "words": out_words, + } + + +def get_screen_description(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Combined description: accessibility tree + OCR + VLM, returning every available source.""" + step_id, caused_by = _new_step_id("get_screen_description") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id) + max_tokens = args.get("max_tokens") + focus_id = args.get("focus_element") + + sub: UIElement = tree + if focus_id: + found = _find_by_id(tree, focus_id) + if found is not None: + sub = found + + parts: Dict[str, str] = {} + + # Accessibility tree — always attempted. + try: + parts["accessibility"] = ctx.describer.from_tree(sub, info) + except Exception as e: + logger.exception("[get_screen_description] accessibility failed: %s", e) + + # OCR — attempted when enabled in config. + ocr_enabled = (ctx.config.get("ocr", {}) or {}).get("enabled", True) + if ocr_enabled: + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot: + parts["ocr"] = ctx.describer.from_ocr(shot) + else: + logger.warning("[get_screen_description] screenshot unavailable for OCR") + except Exception as e: + logger.exception("[get_screen_description] OCR failed: %s", e) + + # VLM — attempted when enabled in config. In multipass mode the VLM + # output is a structured envelope; the JSON-serialised form is folded + # into the concatenated body (for back-compat with the legacy text + # description) and the parsed dict is returned separately under + # ``vlm_structured`` so callers don't have to re-parse it. + vlm_structured: Any = None + vlm_enabled = (ctx.config.get("vlm", {}) or {}).get("enabled", False) + if vlm_enabled: + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot: + vlm_mode = ( + (ctx.config.get("vlm", {}) or {}).get("mode") or "single" + ).lower() + if vlm_mode == "multipass": + env = ctx.describer.from_vlm_multipass( + shot, root=sub, window=info, + ) + if env is not None: + import json as _json + parts["vlm"] = _json.dumps(env, indent=2, + ensure_ascii=False) + vlm_structured = env + else: + vlm_out = ctx.describer.from_vlm( + shot, root=sub, window=info, + ) + if vlm_out is not None: + parts["vlm"] = vlm_out + else: + logger.warning("[get_screen_description] screenshot unavailable for VLM") + except Exception as e: + logger.exception("[get_screen_description] VLM failed: %s", e) + + body = "" + if parts: + body = "\n\n".join(f"[{k}]\n{v}" for k, v in parts.items()) + else: + body = "[no description available]" + + truncated = False + if max_tokens is not None: + char_cap = int(max_tokens) * 4 # rough chars-per-token + if len(body) > char_cap: + body = body[:char_cap] + "… [truncated]" + truncated = True + + result: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, "window_uid": info.window_uid, + "effective_mode": "combined", + "description": body, + "truncated": truncated, + } + if vlm_structured is not None: + result["vlm_structured"] = vlm_structured + return result From 770e24c4414c8d25c95f0a764c69a43b8e206cc2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 20:05:27 +0000 Subject: [PATCH 11/14] =?UTF-8?q?[P3]=20decompose=20observer.py=20?= =?UTF-8?q?=E2=86=92=20observer/=20package=20(shim=20preserves=20imports)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical split of the 2,048-line observer.py into a package: observer/models.py Bounds/UIElement/WindowInfo/WindowResolution + find_element_by_path / prune_tree_depth observer/platform_info.py PLATFORM / IS_WSL / EFFECTIVE_PLATFORM observer/adapters/ mock.py, windows.py (incl. UIA constants), macos.py, linux.py, wsl.py observer/core.py ScreenObserver + adapter selection + cache wiring observer/activation.py bring_to_foreground + _activate_* + title-bar targeting (ActivationMixin) observer/occlusion.py get_visible_areas / is_element_occluded + _intersect_bounds / _subtract_rect (OcclusionMixin) ScreenObserver now inherits the activation/occlusion mixins; method bodies are byte-identical. Top-level mac_adapter.py / linux_adapter.py (the optional pyobjc/pyatspi runtime upgrades) are untouched and still lazily imported by core. observer/__init__.py re-exports the entire pre-split surface (adapters, models, UIA constants, platform flags, geometry helpers) so every existing import path keeps working; all tests pass unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- observer.py | 2048 --------------------------------- observer/__init__.py | 82 ++ observer/activation.py | 287 +++++ observer/adapters/__init__.py | 47 + observer/adapters/linux.py | 147 +++ observer/adapters/macos.py | 76 ++ observer/adapters/mock.py | 184 +++ observer/adapters/windows.py | 640 +++++++++++ observer/adapters/wsl.py | 119 ++ observer/core.py | 367 ++++++ observer/models.py | 197 ++++ observer/occlusion.py | 116 ++ observer/platform_info.py | 24 + 13 files changed, 2286 insertions(+), 2048 deletions(-) delete mode 100644 observer.py create mode 100644 observer/__init__.py create mode 100644 observer/activation.py create mode 100644 observer/adapters/__init__.py create mode 100644 observer/adapters/linux.py create mode 100644 observer/adapters/macos.py create mode 100644 observer/adapters/mock.py create mode 100644 observer/adapters/windows.py create mode 100644 observer/adapters/wsl.py create mode 100644 observer/core.py create mode 100644 observer/models.py create mode 100644 observer/occlusion.py create mode 100644 observer/platform_info.py diff --git a/observer.py b/observer.py deleted file mode 100644 index ff890fa..0000000 --- a/observer.py +++ /dev/null @@ -1,2048 +0,0 @@ -""" -observer.py — Core screen observation module. - -Provides a platform-aware ScreenObserver that exposes a uniform interface -for: enumerating windows, walking the accessibility element tree, capturing -screenshots, and dispatching input actions. Platform adapters (Windows/macOS/ -Linux/Mock) share a common base and are selected automatically at runtime. - -Data model ----------- - Bounds — screen-coordinate bounding rectangle - UIElement — one node of the accessibility tree - WindowInfo — top-level window metadata -""" - -import io -import logging -import os -import platform -import time -import traceback -from dataclasses import dataclass, field, replace as _dc_replace -from typing import Any, Callable, Dict, List, Optional, Tuple - -logger = logging.getLogger(__name__) - -PLATFORM = platform.system() - - -def _is_wsl() -> bool: - """True when running inside Windows Subsystem for Linux (WSL 1 or WSL 2).""" - if PLATFORM != "Linux": - return False - try: - with open("/proc/version") as _f: - return "microsoft" in _f.read().lower() - except Exception: - return False - - -IS_WSL = _is_wsl() -EFFECTIVE_PLATFORM = "WSL" if IS_WSL else PLATFORM - - -# ───────────────────────────────────────────────────────────────────────────── -# Data Structures -# ───────────────────────────────────────────────────────────────────────────── - -@dataclass -class Bounds: - x: int - y: int - width: int - height: int - - @property - def right(self) -> int: - return self.x + self.width - - @property - def bottom(self) -> int: - return self.y + self.height - - @property - def center_x(self) -> int: - return self.x + self.width // 2 - - @property - def center_y(self) -> int: - return self.y + self.height // 2 - - def to_dict(self) -> Dict: - return {"x": self.x, "y": self.y, "width": self.width, "height": self.height} - - def __bool__(self) -> bool: - return self.width > 0 and self.height > 0 - - -@dataclass -class UIElement: - element_id: str - name: str - role: str - value: Optional[str] = None - bounds: Bounds = field(default_factory=lambda: Bounds(0, 0, 0, 0)) - enabled: bool = True - focused: bool = False - keyboard_shortcut: Optional[str] = None - description: Optional[str] = None - children: List["UIElement"] = field(default_factory=list) - # Extended a11y signals — None means "adapter could not determine". - # Populated where the platform exposes the pattern (UIA SelectionItem / - # ExpandCollapse / RangeValue, AX AXValue/AXMinValue/AXMaxValue, AT-SPI - # STATE_SELECTED / STATE_EXPANDED / IValue) and consumed by the ASCII - # renderer for role-aware glyphs and the structured sidecar. - selected: Optional[bool] = None - expanded: Optional[bool] = None - value_now: Optional[float] = None - value_min: Optional[float] = None - value_max: Optional[float] = None - identifier: Optional[str] = None - - def to_dict(self) -> Dict: - d: Dict[str, Any] = { - "id": self.element_id, - "name": self.name, - "role": self.role, - "value": self.value, - "bounds": self.bounds.to_dict(), - "enabled": self.enabled, - "focused": self.focused, - "keyboard_shortcut": self.keyboard_shortcut, - "description": self.description, - "children": [c.to_dict() for c in self.children], - } - # Omit extended fields when unset so existing API consumers do not - # see a flood of nulls; include them when populated. - for k in ("selected", "expanded", "value_now", "value_min", - "value_max", "identifier"): - v = getattr(self, k) - if v is not None: - d[k] = v - return d - - def flat_list(self) -> List["UIElement"]: - """Return all elements in this subtree as a flat list (DFS order).""" - result = [self] - for child in self.children: - result.extend(child.flat_list()) - return result - - -@dataclass -class WindowInfo: - handle: Any # platform-specific: HWND (int) on Windows; int index elsewhere - title: str - process_name: str - pid: int - bounds: Bounds - is_focused: bool - # Stable cross-call identifier; populated by adapters (design doc §6.1). - window_uid: str = "" - # Optional multi-monitor metadata (design doc §6.3). Populated when the - # adapter knows; left None on adapters that do not. - monitor_index: Optional[int] = None - scale_factor: Optional[float] = None - logical_bounds: Optional[Bounds] = None - physical_bounds: Optional[Bounds] = None - - def to_dict(self) -> Dict: - d: Dict[str, Any] = { - "handle": str(self.handle), - "title": self.title, - "process": self.process_name, - "pid": self.pid, - "bounds": self.bounds.to_dict(), - "focused": self.is_focused, - "window_uid": self.window_uid, - } - if self.monitor_index is not None: - d["monitor_index"] = self.monitor_index - if self.scale_factor is not None: - d["scale_factor"] = self.scale_factor - if self.logical_bounds is not None: - d["logical_bounds"] = self.logical_bounds.to_dict() - if self.physical_bounds is not None: - d["physical_bounds"] = self.physical_bounds.to_dict() - return d - - -# ─── Window resolution result ──────────────────────────────────────────────── - -@dataclass -class WindowResolution: - info: Optional[WindowInfo] - warning: Optional[str] - used_uid: bool - requested_uid: Optional[str] - - -# ─── Subtree helpers (P1 perf: scoped drill-in) ────────────────────────────── - -def find_element_by_path(root: Optional[UIElement], - element_path: str) -> Optional[UIElement]: - """Locate an element by its positional element-id path (e.g. 'root.3.2'). - - Prefers walking the id prefixes level by level (cheap); falls back to a - full DFS by exact id for trees whose ids are not strictly positional - (e.g. nodes injected by tree synthesis get ids like 'root.2.x1').""" - if root is None or not element_path: - return None - if root.element_id == element_path: - return root - # Fast path: navigate children whose ids extend the current prefix. - if element_path.startswith(root.element_id + "."): - node = root - prefix = root.element_id - rest = element_path[len(prefix) + 1:] - found = True - for seg in rest.split("."): - prefix = f"{prefix}.{seg}" - nxt = next((c for c in node.children if c.element_id == prefix), - None) - if nxt is None: - found = False - break - node = nxt - if found: - return node - # Fallback: exhaustive search by exact id. - stack = [root] - while stack: - e = stack.pop() - if e.element_id == element_path: - return e - stack.extend(e.children) - return None - - -def prune_tree_depth(elem: Optional[UIElement], - max_depth: Optional[int]) -> Optional[UIElement]: - """Return a copy of *elem* limited to *max_depth* levels below it. - - Nodes are shallow-copied (bounds objects are shared) so the original — - possibly cache-resident — tree is never mutated.""" - if elem is None or max_depth is None: - return elem - - def _copy(e: UIElement, depth: int) -> UIElement: - kids = ([] if depth >= max_depth - else [_copy(c, depth + 1) for c in e.children]) - return _dc_replace(e, children=kids) - - return _copy(elem, 0) - - -# ───────────────────────────────────────────────────────────────────────────── -# Mock Adapter (no OS dependencies; safe to run in any environment) -# ───────────────────────────────────────────────────────────────────────────── - -class MockAdapter: - """Synthetic data adapter for development and testing.""" - - def __init__(self) -> None: - import secrets as _s - self._nonce = _s.token_hex(4) - # Optional scenario hook (design doc §15.5). Set by main.py when - # --scenario is supplied; methods route through the scenario when - # active so that input actions can drive state transitions. - self.scenario: Optional[Any] = None - # Test hooks (P1 perf work): capture_count increments on every tree - # walk so tests can assert cache hits avoided adapter work; - # tree_mutator, when set, post-processes (or replaces) each captured - # tree so tests can simulate UI changes between captures. - self.capture_count: int = 0 - self.tree_mutator: Optional[Callable[[UIElement], UIElement]] = None - - def get_windows_above_bounds(self, hwnd) -> List["Bounds"]: - return [] # Mock assumes the target window is on top - - def list_windows(self) -> List[WindowInfo]: - if self.scenario is not None: - return self.scenario.list_windows(self._nonce) - return [ - WindowInfo(1001, "Untitled — Notepad", "notepad.exe", 1234, - Bounds(80, 60, 800, 600), True, - window_uid=f"mock:0:{self._nonce}"), - WindowInfo(1002, "GitHub · Where software is built — Google Chrome", - "chrome.exe", 5678, Bounds(0, 0, 1920, 1050), False, - window_uid=f"mock:1:{self._nonce}"), - WindowInfo(1003, "screen_observer.py — Visual Studio Code", - "code.exe", 9012, Bounds(960, 0, 960, 1050), False, - window_uid=f"mock:2:{self._nonce}"), - ] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - self.capture_count += 1 - tree = self._build_tree(hwnd) - if tree is not None and self.tree_mutator is not None: - mutated = self.tree_mutator(tree) - if mutated is not None: - tree = mutated - return tree - - def get_element_subtree(self, hwnd=None, element_path: str = "root", - max_depth: Optional[int] = None - ) -> Optional[UIElement]: - """Walk only the subtree rooted at *element_path*, to *max_depth* - levels below it. The mock world is synthetic, so this navigates the - positional element-id path of a fresh capture.""" - tree = self.get_element_tree(hwnd) - sub = find_element_by_path(tree, element_path) - return prune_tree_depth(sub, max_depth) - - def _build_tree(self, hwnd=None) -> Optional[UIElement]: - if self.scenario is not None: - return self.scenario.get_element_tree(hwnd) - root = UIElement("root", "Untitled — Notepad", "Window", - bounds=Bounds(80, 60, 800, 600)) - - # Menu bar - menubar = UIElement("root.0", "MenuBar", "MenuBar", - bounds=Bounds(80, 60, 800, 22)) - for i, lbl in enumerate(["File", "Edit", "Format", "View", "Help"]): - menubar.children.append(UIElement( - f"root.0.{i}", lbl, "MenuItem", - bounds=Bounds(80 + i * 58, 60, 56, 22) - )) - root.children.append(menubar) - - # Main editing area - editor = UIElement( - "root.1", "Text Editor", "Document", - value="Hello, world!\nThis is a test document.\nLine 3 has some content here.\n", - bounds=Bounds(80, 82, 800, 514), focused=True - ) - root.children.append(editor) - - # Horizontal scroll bar - hscroll = UIElement("root.2", "Horizontal ScrollBar", "ScrollBar", - bounds=Bounds(80, 596, 784, 18)) - root.children.append(hscroll) - - # Vertical scroll bar - vscroll = UIElement("root.3", "Vertical ScrollBar", "ScrollBar", - bounds=Bounds(864, 82, 18, 532)) - root.children.append(vscroll) - - # Progress bar — exercises value_now/min/max for the role glyph path. - root.children.append(UIElement( - "root.5", "Saving", "ProgressBar", - bounds=Bounds(560, 600, 200, 10), - value_now=40.0, value_min=0.0, value_max=100.0, - )) - - # Word-wrap toggle checkbox — exercises selected=True path. - root.children.append(UIElement( - "root.6", "Word Wrap", "CheckBox", - bounds=Bounds(80, 636, 120, 18), selected=True, - )) - - # Status bar - sb = UIElement("root.4", "Status Bar", "StatusBar", - bounds=Bounds(80, 614, 800, 22)) - for i, (lbl, val) in enumerate([ - ("Position", "Ln 1, Col 1"), - ("Zoom", "100%"), - ("Encoding", "UTF-8"), - ("EOL", "Windows (CRLF)"), - ]): - sb.children.append(UIElement( - f"root.4.{i}", lbl, "Text", value=val, - bounds=Bounds(80 + i * 190, 614, 188, 22) - )) - root.children.append(sb) - - return root - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - from PIL import Image, ImageDraw - img = Image.new("RGB", (800, 600), "#1a1e2e") - draw = ImageDraw.Draw(img) - - # Title bar - draw.rectangle([0, 0, 800, 30], fill="#2d3250") - draw.text((10, 8), "Untitled — Notepad", fill="#c8d3f5") - - # Menu bar - draw.rectangle([0, 30, 800, 52], fill="#1e2030") - for i, m in enumerate(["File", "Edit", "Format", "View", "Help"]): - draw.text((10 + i * 58, 36), m, fill="#a9b1d6") - - # Editor area - draw.rectangle([0, 52, 782, 570], fill="#1a1e2e") - for i, ln in enumerate(["Hello, world!", "This is a test document.", - "Line 3 has some content here.", ""]): - draw.text((6, 58 + i * 18), ln, fill="#c0caf5") - - # Scrollbars - draw.rectangle([782, 52, 800, 570], fill="#24283b") - draw.rectangle([0, 570, 782, 586], fill="#24283b") - - # Status bar - draw.rectangle([0, 586, 800, 600], fill="#16161e") - draw.text((6, 589), "Ln 1, Col 1 100% UTF-8 Windows (CRLF)", - fill="#565f89") - - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - print(f"[MockAdapter:get_screenshot] {e}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id: Optional[str] = None, - value: Any = None, hwnd=None) -> Dict: - if self.scenario is not None: - handled = self.scenario.handle_action(action=action, - element_id=element_id, - value=value, hwnd=hwnd) - if handled is not None: - return handled - return { - "success": True, - "action": action, - "note": "Mock adapter — no real OS action performed", - } - - -# ───────────────────────────────────────────────────────────────────────────── -# UIA control-type ID → pywinauto-compatible role name. -# Values from UIAutomationClient.h; must match what selectors/descriptions expect. -_UIA_TYPE_TO_ROLE: Dict[int, str] = { - 50000: "Button", 50001: "Calendar", 50002: "CheckBox", - 50003: "ComboBox", 50004: "Edit", 50005: "Hyperlink", - 50006: "Image", 50007: "ListItem", 50008: "ListBox", - 50009: "Menu", 50010: "MenuBar", 50011: "MenuItem", - 50012: "ProgressBar", 50013: "RadioButton", 50014: "ScrollBar", - 50015: "Slider", 50016: "Spinner", 50017: "StatusBar", - 50018: "TabControl", 50019: "TabItem", 50020: "Text", - 50021: "Toolbar", 50022: "ToolTip", 50023: "Tree", - 50024: "TreeItem", 50025: "Custom", 50026: "GroupBox", - 50027: "Thumb", 50028: "DataGrid", 50029: "DataItem", - 50030: "Document", 50031: "SplitButton", 50032: "Dialog", - 50033: "Pane", 50034: "Header", 50035: "HeaderItem", - 50036: "Table", 50037: "TitleBar", 50038: "Separator", - 50039: "SemanticZoom",50040: "AppBar", -} - -# UIA property IDs (UIAutomationClient.h) used by the raw-COM walker. -_UIA_BOUNDING_RECT = 30001 -_UIA_NAME = 30005 -_UIA_CTRL_TYPE = 30003 -_UIA_ENABLED = 30010 -_UIA_FOCUSED = 30008 -_UIA_VALUE = 30045 -_UIA_ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" -_UIA_ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" -_UIA_HELP_TEXT = 30013 # tooltip / description -_UIA_AUTOMATION_ID = 30011 -_UIA_RANGE_VALUE = 30047 -_UIA_RANGE_MIN = 30049 -_UIA_RANGE_MAX = 30050 -_UIA_IS_SELECTED = 30079 -_UIA_EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf -_UIA_SCOPE_CHILDREN = 0x2 - -# Properties bulk-fetched via a UIA CacheRequest so each level of the walk is -# one COM round trip (FindAllBuildCache) instead of ~15 per node. -_UIA_CACHED_PROPS = ( - _UIA_BOUNDING_RECT, _UIA_NAME, _UIA_CTRL_TYPE, _UIA_ENABLED, - _UIA_FOCUSED, _UIA_VALUE, _UIA_ACCESS_KEY, _UIA_ACCEL_KEY, - _UIA_HELP_TEXT, _UIA_AUTOMATION_ID, _UIA_RANGE_VALUE, _UIA_RANGE_MIN, - _UIA_RANGE_MAX, _UIA_IS_SELECTED, _UIA_EXPAND_STATE, -) - - -# Windows Adapter (requires: pywinauto, pywin32, psutil) -# ───────────────────────────────────────────────────────────────────────────── - -class WindowsAdapter: - """Full Windows UIA adapter using pywinauto + pywin32.""" - - def __init__(self, config: dict): - self.config = config - try: - import win32gui # noqa: F401 - import win32process # noqa: F401 - import psutil # noqa: F401 - from pywinauto import Application # noqa: F401 - self._Application = Application - logger.info("[WindowsAdapter:__init__] pywinauto/pywin32 ready") - except ImportError as e: - print(f"[WindowsAdapter:__init__] Missing dependency: {e}") - traceback.print_exc() - raise - - def list_windows(self) -> List[WindowInfo]: - try: - import win32gui - import win32process - import psutil - - results: List[WindowInfo] = [] - fg = win32gui.GetForegroundWindow() - - def _cb(hwnd, _): - try: - if not win32gui.IsWindowVisible(hwnd): - return - title = win32gui.GetWindowText(hwnd) - if not title: - return - rect = win32gui.GetWindowRect(hwnd) - w, h = rect[2] - rect[0], rect[3] - rect[1] - if w <= 0 or h <= 0: - return - try: - _, pid = win32process.GetWindowThreadProcessId(hwnd) - proc_name = psutil.Process(pid).name() - except Exception: - pid, proc_name = 0, "unknown" - results.append(WindowInfo( - handle=hwnd, title=title, process_name=proc_name, pid=pid, - bounds=Bounds(rect[0], rect[1], w, h), - is_focused=(hwnd == fg), - window_uid=f"win:{pid}:{hwnd}", - )) - except Exception as inner: - logger.debug(f"[WindowsAdapter:list_windows:_cb] {inner}") - - win32gui.EnumWindows(_cb, None) - return sorted(results, key=lambda w: (not w.is_focused, w.title.lower())) - except Exception as e: - print(f"[WindowsAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - """Return bounds of visible windows that are above hwnd in Z-order.""" - try: - import win32gui - GW_HWNDNEXT = 2 - above: List[Bounds] = [] - h = win32gui.GetTopWindow(None) - while h and h != hwnd: - try: - if win32gui.IsWindowVisible(h): - rect = win32gui.GetWindowRect(h) - w = rect[2] - rect[0] - hh = rect[3] - rect[1] - if w > 0 and hh > 0: - above.append(Bounds(rect[0], rect[1], w, hh)) - except Exception as e: - logger.debug(f"[occlusion] window rect probe failed " - f"for hwnd {h}: {e}") - try: - h = win32gui.GetWindow(h, GW_HWNDNEXT) - except Exception: - break - return above - except Exception as e: - logger.debug(f"[WindowsAdapter:get_windows_above_bounds] {e}") - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - try: - import win32gui - if hwnd is None: - hwnd = win32gui.GetForegroundWindow() - except Exception as e: - print(f"[WindowsAdapter:get_element_tree] win32gui: {e}") - return None - - # Run both walkers and synthesise: UIA crosses Chromium fragment boundaries - # (gets web content); pywinauto may surface native-control properties that - # UIA omits. The merged result gives the LLM everything either source sees. - # tree.strategy == "uia_only" skips the second pywinauto walk and the - # synthesis pass entirely — roughly halving capture time — at the cost - # of the extra native-control properties the merge would contribute. - strategy = str(self.config.get("tree", {}).get("strategy", - "merged")).lower() - uia_tree = self._uia_walk(hwnd) - if strategy == "uia_only" and uia_tree is not None: - return uia_tree - pw_tree = None - try: - app = self._Application(backend="uia").connect(handle=hwnd) - window = app.window(handle=hwnd) - wrapper = window.wrapper_object() - max_depth = self.config.get("tree", {}).get("max_depth", 8) - pw_tree = self._walk(wrapper, "root", 0, max_depth) - except Exception as e: - logger.debug(f"[WindowsAdapter:get_element_tree] pywinauto: {e}") - - if uia_tree is None and pw_tree is None: - return None - if uia_tree is None: - return pw_tree - if pw_tree is None: - return uia_tree - return self._synthesize_trees(uia_tree, pw_tree) - - def get_element_subtree(self, hwnd=None, element_path: str = "root", - max_depth: Optional[int] = None - ) -> Optional[UIElement]: - """Walk only the subtree rooted at *element_path* via raw UIA. - - Navigates the positional child indices of the element-id path - ('root.3.2' → child 3 → child 2) so only the requested branch is - traversed. Falls back to a full walk plus extraction when the path - contains non-positional segments (synthesized ids) or navigation - fails.""" - try: - import win32gui - if hwnd is None: - hwnd = win32gui.GetForegroundWindow() - except Exception as e: - print(f"[WindowsAdapter:get_element_subtree] win32gui: {e}") - return None - - if max_depth is None: - max_depth = self.config.get("tree", {}).get("max_depth", 8) - - indices = self._parse_positional_path(element_path) - if indices is not None: - try: - raw_uia, true_cond = self._uia_handles() - elem = raw_uia.ElementFromHandle(hwnd) - for idx in indices: - kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) - if idx < 0 or idx >= kids.Length: - elem = None - break - elem = kids.GetElement(idx) - if elem is not None: - return self._uia_walk_element( - elem, element_path, 0, max_depth, true_cond, - cache_request=self._uia_cache_request(raw_uia)) - except Exception as e: - logger.debug(f"[WindowsAdapter:get_element_subtree] " - f"navigation failed ({e}); falling back") - - # Fallback: full walk + extraction. - tree = self.get_element_tree(hwnd) - sub = find_element_by_path(tree, element_path) - return prune_tree_depth(sub, max_depth) - - @staticmethod - def _parse_positional_path(element_path: str) -> Optional[List[int]]: - """'root.3.2' → [3, 2]; None when the path is not purely positional.""" - segs = (element_path or "").split(".") - if not segs or segs[0] != "root": - return None - try: - return [int(s) for s in segs[1:]] - except ValueError: - return None # synthesized ids like 'root.2.x1' - - def _uia_handles(self): - """Return (raw IUIAutomation interface, true condition). - - pywinauto already initialised comtypes/UIA at import time; retrieve - the raw COM interface from its singleton (pywinauto ≥0.6 exposes it - as .iuia; older versions expose it directly).""" - from pywinauto.uia_defines import IUIA - _iuia_obj = IUIA() - raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) - return raw_uia, raw_uia.CreateTrueCondition() - - def _uia_walk(self, hwnd: int) -> Optional[UIElement]: - """Walk the accessibility tree via raw IUIAutomation COM calls. - - pywinauto's children() falls back to EnumChildWindows for HWND-backed - elements (e.g. Chrome_RenderWidgetHostHWND), missing all web content. - Using FindAll(TreeScope_Children) directly on the IUIAutomationElement - always uses UIA and correctly crosses Chromium fragment boundaries. - - When the COM API supports it, a CacheRequest bulk-fetches the walked - properties per level (FindAllBuildCache + GetCachedPropertyValue) — - one cross-process round trip per node's children instead of ~15 per - node. Any CacheRequest failure falls back to the per-property path. - """ - try: - raw_uia, true_cond = self._uia_handles() - root = raw_uia.ElementFromHandle(hwnd) - max_depth = self.config.get("tree", {}).get("max_depth", 8) - cache_request = self._uia_cache_request(raw_uia) - return self._uia_walk_element(root, "root", 0, max_depth, - true_cond, - cache_request=cache_request) - except Exception as e: - logger.debug(f"[WindowsAdapter:_uia_walk] {e}") - return None - - @staticmethod - def _uia_cache_request(raw_uia): - """Build a CacheRequest covering the properties the walker reads. - Returns None (per-property fallback) when construction fails.""" - try: - cr = raw_uia.CreateCacheRequest() - for pid in _UIA_CACHED_PROPS: - cr.AddProperty(pid) - return cr - except Exception as e: - logger.debug(f"[WindowsAdapter:_uia_cache_request] CacheRequest " - f"unavailable ({e}); using per-property fetches") - return None - - @staticmethod - def _uia_prop(elem, pid, default=None, cached=False): - if cached: - try: - v = elem.GetCachedPropertyValue(pid) - return v if v is not None else default - except Exception: - pass # cache miss/failure — fall back to a live fetch - try: - v = elem.GetCurrentPropertyValue(pid) - return v if v is not None else default - except Exception: - return default - - @staticmethod - def _uia_bounds(elem, cached=False) -> Bounds: - if cached: - try: - r = elem.CachedBoundingRectangle - return Bounds(r.left, r.top, - r.right - r.left, r.bottom - r.top) - except Exception: - try: - arr = elem.GetCachedPropertyValue(_UIA_BOUNDING_RECT) - if arr is not None and len(arr) == 4: - # VT_R8 SAFEARRAY: [left, top, width, height] - return Bounds(int(arr[0]), int(arr[1]), - int(arr[2]), int(arr[3])) - except Exception: - pass - try: - r = elem.CurrentBoundingRectangle - return Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) - except Exception: - return Bounds(0, 0, 0, 0) - - def _uia_walk_element(self, elem, elem_id: str, depth: int, - max_depth: int, true_cond, - cache_request=None, cached: bool = False - ) -> UIElement: - """Build a UIElement for *elem* and recurse into its children. - - *cached* means this element was fetched via FindAllBuildCache and its - properties can be read with GetCachedPropertyValue (no round trip). - """ - def _prop(pid, default=None): - return self._uia_prop(elem, pid, default, cached=cached) - - name = _prop(_UIA_NAME, "") or "" - ctrl = _prop(_UIA_CTRL_TYPE, 0) or 0 - role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") - bounds = self._uia_bounds(elem, cached=cached) - enabled = bool(_prop(_UIA_ENABLED, True)) - focused = bool(_prop(_UIA_FOCUSED, False)) - value = _prop(_UIA_VALUE) or None - # Keyboard shortcut: prefer access key, fall back to accelerator. - ks = _prop(_UIA_ACCESS_KEY) or _prop(_UIA_ACCEL_KEY) or None - desc = _prop(_UIA_HELP_TEXT) or None - aid = _prop(_UIA_AUTOMATION_ID) or None - # RangeValue pattern: slider / progress / scrollbar - vn = _prop(_UIA_RANGE_VALUE) - vmin = _prop(_UIA_RANGE_MIN) - vmax = _prop(_UIA_RANGE_MAX) - # SelectionItem.IsSelected: checkbox / radio / tab / menuitem - sel_raw = _prop(_UIA_IS_SELECTED) - sel = bool(sel_raw) if sel_raw is not None else None - # ExpandCollapse: combobox / treeitem / menuitem - exp_raw = _prop(_UIA_EXPAND_STATE) - if exp_raw in (0, 1): - exp = bool(exp_raw) - else: - exp = None - try: - vn_f = float(vn) if vn is not None else None - except Exception: - vn_f = None - try: - vmin_f = float(vmin) if vmin is not None else None - except Exception: - vmin_f = None - try: - vmax_f = float(vmax) if vmax is not None else None - except Exception: - vmax_f = None - node = UIElement( - element_id=elem_id, name=name, role=role, value=value, - bounds=bounds, enabled=enabled, focused=focused, - keyboard_shortcut=ks or None, - description=desc or None, - selected=sel, expanded=exp, - value_now=vn_f, value_min=vmin_f, value_max=vmax_f, - identifier=str(aid) if aid else None, - ) - if depth < max_depth: - kids = None - kids_cached = False - if cache_request is not None: - try: - kids = elem.FindAllBuildCache(_UIA_SCOPE_CHILDREN, - true_cond, cache_request) - kids_cached = True - except Exception: - kids = None # per-node fallback below - if kids is None: - try: - kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) - except Exception: - kids = None - if kids is not None: - try: - for i in range(kids.Length): - node.children.append(self._uia_walk_element( - kids.GetElement(i), f"{elem_id}.{i}", - depth + 1, max_depth, true_cond, - cache_request=cache_request, - cached=kids_cached)) - except Exception as e: - logger.debug(f"[uia] child walk truncated at " - f"{elem_id}: {e}") - return node - - def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: - try: - try: - rect = wrapper.rectangle() - bounds = Bounds(rect.left, rect.top, - rect.right - rect.left, rect.bottom - rect.top) - except Exception: - bounds = Bounds(0, 0, 0, 0) - - try: - name = wrapper.window_text() or "" - except Exception: - name = "" - - try: - role = wrapper.friendly_class_name() or "Unknown" - except Exception: - role = "Unknown" - - try: - value = wrapper.get_value() if hasattr(wrapper, "get_value") else None - except Exception: - value = None - - try: - enabled = wrapper.is_enabled() - except Exception: - enabled = True - - try: - focused = wrapper.has_keyboard_focus() - except Exception: - focused = False - - elem = UIElement( - element_id=elem_id, name=name, role=role, value=value, - bounds=bounds, enabled=enabled, focused=focused, - ) - - if depth < max_depth: - try: - for i, child in enumerate(wrapper.children()): - elem.children.append( - self._walk(child, f"{elem_id}.{i}", depth + 1, max_depth) - ) - except Exception as ce: - logger.debug(f"[WindowsAdapter:_walk:{elem_id}:children] {ce}") - - return elem - except Exception as e: - print(f"[WindowsAdapter:_walk:{elem_id}] {e}") - traceback.print_exc() - return UIElement(elem_id, "[error]", "Unknown", bounds=Bounds(0, 0, 0, 0)) - - def _synthesize_trees(self, primary: UIElement, secondary: UIElement) -> UIElement: - """Merge two accessibility trees into one richer tree. - - Uses the primary (UIA) tree as the base — it sees web content. - For each node in secondary (pywinauto) matched by bounds, enrich the - primary node with any non-empty properties the primary is missing. - Secondary nodes whose bounds don't appear anywhere in the primary tree - are injected under the deepest primary ancestor that contains them. - """ - # Build a flat index of primary nodes keyed by (x, y, w, h). - bounds_index: Dict[tuple, UIElement] = {} - - def _index(node: UIElement) -> None: - key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) - if key not in bounds_index: - bounds_index[key] = node - for c in node.children: - _index(c) - - _index(primary) - - # Enrich matched nodes; collect unmatched ones with their bounds. - unmatched: List[UIElement] = [] - - def _enrich(node: UIElement) -> None: - key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) - target = bounds_index.get(key) - if target is not None: - # Copy over properties the primary left empty. - if not target.keyboard_shortcut and node.keyboard_shortcut: - target.keyboard_shortcut = node.keyboard_shortcut - if not target.description and node.description: - target.description = node.description - if not target.value and node.value: - target.value = node.value - else: - # Not in primary — keep for injection. - w, h = node.bounds.width, node.bounds.height - if w > 0 and h > 0: # ignore zero-size ghost elements - unmatched.append(node) - for c in node.children: - _enrich(c) - - _enrich(secondary) - - # Inject unmatched secondary nodes under the deepest primary ancestor - # whose bounds contain them (largest-area match wins → most specific). - def _contains(outer: Bounds, inner: Bounds) -> bool: - return (outer.x <= inner.x and outer.y <= inner.y and - outer.x + outer.width >= inner.x + inner.width and - outer.y + outer.height >= inner.y + inner.height) - - for node in unmatched: - best: Optional[UIElement] = None - best_area = float("inf") - for pnode in bounds_index.values(): - if _contains(pnode.bounds, node.bounds): - area = pnode.bounds.width * pnode.bounds.height - if area < best_area: - best_area = area - best = pnode - if best is None: - best = primary - # Renumber the injected element_id to avoid collisions. - node.element_id = f"{best.element_id}.x{len(best.children)}" - best.children.append(node) - - return primary - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - import win32gui - import win32ui - from PIL import Image - import ctypes - - if hwnd is None: - hwnd = win32gui.GetForegroundWindow() - - rect = win32gui.GetWindowRect(hwnd) - width = rect[2] - rect[0] - height = rect[3] - rect[1] - - if width <= 0 or height <= 0: - return None - - hwnd_dc = win32gui.GetWindowDC(hwnd) - mfc_dc = win32ui.CreateDCFromHandle(hwnd_dc) - save_dc = mfc_dc.CreateCompatibleDC() - save_bmp = win32ui.CreateBitmap() - save_bmp.CreateCompatibleBitmap(mfc_dc, width, height) - save_dc.SelectObject(save_bmp) - - # PW_RENDERFULLCONTENT (0x2) renders hardware-accelerated content too - ok = ctypes.windll.user32.PrintWindow( # type: ignore[attr-defined] - hwnd, save_dc.GetSafeHdc(), 2) - - bmpinfo = save_bmp.GetInfo() - bmpstr = save_bmp.GetBitmapBits(True) - - win32gui.DeleteObject(save_bmp.GetHandle()) - save_dc.DeleteDC() - mfc_dc.DeleteDC() - win32gui.ReleaseDC(hwnd, hwnd_dc) - - if ok: - img = Image.frombuffer( - "RGB", - (bmpinfo["bmWidth"], bmpinfo["bmHeight"]), - bmpstr, "raw", "BGRX", 0, 1, - ) - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - - # PrintWindow failed — fall back to screen-region capture - logger.warning("[WindowsAdapter:get_screenshot] PrintWindow failed; falling back to mss") - raise RuntimeError("PrintWindow returned 0") - - except Exception as e: - logger.debug(f"[WindowsAdapter:get_screenshot] PrintWindow path failed ({e}); trying mss") - try: - import mss - from PIL import Image - import win32gui - - with mss.MSS() as sct: - if hwnd: - rect = win32gui.GetWindowRect(hwnd) - region = {"left": rect[0], "top": rect[1], - "width": rect[2] - rect[0], "height": rect[3] - rect[1]} - else: - region = sct.monitors[1] - raw = sct.grab(region) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e2: - print(f"[WindowsAdapter:get_screenshot] {e2}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id: Optional[str] = None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True, "action": "type", "text": value} - - elif action == "key" and value: - keys = str(value).lower().split("+") - pyautogui.hotkey(*keys) - return {"success": True, "action": "key", "keys": value} - - elif action == "click_at" and isinstance(value, dict): - pyautogui.click(value["x"], value["y"]) - return {"success": True, "action": "click_at", **value} - - elif action == "scroll" and isinstance(value, dict): - pyautogui.scroll(value.get("clicks", 3), x=value.get("x"), y=value.get("y")) - return {"success": True, "action": "scroll"} - - else: - return {"success": False, "error": f"Unsupported action: {action}"} - except Exception as e: - print(f"[WindowsAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# macOS Adapter (screenshot works; AX tree is a stub pending pyobjc work) -# ───────────────────────────────────────────────────────────────────────────── - -class MacOSAdapter: - def __init__(self, config: dict): - self.config = config - logger.info("[MacOSAdapter:__init__] macOS adapter loaded (AX tree is stub)") - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - return [] # Z-order unavailable without Quartz CGWindowList - - def list_windows(self) -> List[WindowInfo]: - try: - import subprocess - script = ('tell application "System Events" to get name of every ' - 'process whose background only is false') - r = subprocess.run(["osascript", "-e", script], - capture_output=True, text=True, timeout=5) - apps = [a.strip() for a in r.stdout.split(",") if a.strip()] - return [WindowInfo(i, a, a, 0, Bounds(0, 0, 1920, 1080), i == 0, - window_uid=f"mac:{i}") - for i, a in enumerate(apps)] - except Exception as e: - print(f"[MacOSAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - logger.warning("[MacOSAdapter:get_element_tree] Full AX tree requires pyobjc; returning stub") - return UIElement("root", "macOS Application (AX tree stub)", "Window", - bounds=Bounds(0, 0, 1920, 1080)) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[1]) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - print(f"[MacOSAdapter:get_screenshot] {e}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id=None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True} - if action == "key" and value: - pyautogui.hotkey(*str(value).lower().split("+")) - return {"success": True} - return {"success": False, "error": f"Unsupported: {action}"} - except Exception as e: - print(f"[MacOSAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# Linux Adapter (screenshot works; AT-SPI tree is stub pending pyatspi work) -# ───────────────────────────────────────────────────────────────────────────── - -class LinuxAdapter: - def __init__(self, config: dict): - self.config = config - logger.info("[LinuxAdapter:__init__] Linux adapter loaded (AT-SPI tree is stub)") - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - return [] # Z-order unavailable without Xlib/wnck - - def list_windows(self) -> List[WindowInfo]: - # Prefer wmctrl when present; otherwise fall back to Xlib so the - # adapter still works on systems without the wmctrl binary installed. - import subprocess - try: - r = subprocess.run(["wmctrl", "-lG"], capture_output=True, text=True, timeout=5) - except FileNotFoundError: - logger.info("[LinuxAdapter:list_windows] wmctrl not installed; trying Xlib fallback") - return self._list_windows_xlib() - except Exception as e: - logger.warning("[LinuxAdapter:list_windows] wmctrl failed: %s; trying Xlib fallback", e) - return self._list_windows_xlib() - try: - windows = [] - for line in r.stdout.strip().split("\n"): - if not line: - continue - parts = line.split(None, 8) - if len(parts) < 8: - continue - hwnd = int(parts[0], 16) - x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]) - title = parts[8] if len(parts) > 8 else "(no title)" - windows.append(WindowInfo(hwnd, title, title, 0, - Bounds(x, y, w, h), False, - window_uid=f"x11:{hwnd:x}")) - return windows - except Exception as e: - print(f"[LinuxAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def _list_windows_xlib(self) -> List[WindowInfo]: - try: - from Xlib import display, X # noqa: F401 - from Xlib.error import XError - except ImportError: - logger.warning( - "[LinuxAdapter:list_windows] neither 'wmctrl' nor python-xlib " - "is available; install one (e.g. `apt install wmctrl` or " - "`pip install python-xlib`) to enumerate windows" - ) - return [] - try: - d = display.Display() - root = d.screen().root - NET_CLIENT_LIST = d.intern_atom("_NET_CLIENT_LIST") - NET_WM_NAME = d.intern_atom("_NET_WM_NAME") - UTF8_STRING = d.intern_atom("UTF8_STRING") - prop = root.get_full_property(NET_CLIENT_LIST, X.AnyPropertyType) - if prop is None: - return [] - windows: List[WindowInfo] = [] - for wid in prop.value: - try: - w = d.create_resource_object("window", wid) - geom = w.get_geometry() - coords = w.translate_coords(root, 0, 0) - x, y = -coords.x, -coords.y - title = "" - name_prop = w.get_full_property(NET_WM_NAME, UTF8_STRING) - if name_prop and name_prop.value: - title = name_prop.value.decode("utf-8", errors="replace") - else: - wm_name = w.get_wm_name() - if wm_name: - title = wm_name if isinstance(wm_name, str) else wm_name.decode("utf-8", errors="replace") - title = title or "(no title)" - windows.append(WindowInfo(int(wid), title, title, 0, - Bounds(x, y, geom.width, geom.height), False, - window_uid=f"x11:{int(wid):x}")) - except XError: - continue - return windows - except Exception as e: - print(f"[LinuxAdapter:_list_windows_xlib] {e}") - traceback.print_exc() - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - logger.warning("[LinuxAdapter:get_element_tree] Full tree requires pyatspi; returning stub") - return UIElement("root", "Linux Application (AT-SPI stub)", "Window", - bounds=Bounds(0, 0, 1920, 1080)) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - # mss needs a running X server (DISPLAY must be set). - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[1]) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - logger.debug("[LinuxAdapter:get_screenshot] mss failed: %s; trying scrot", e) - # scrot can write PNG directly to stdout (-z suppresses notifications). - try: - import subprocess - r = subprocess.run(["scrot", "-z", "-"], capture_output=True, timeout=10) - if r.returncode == 0 and r.stdout: - return r.stdout - except Exception as e: - logger.debug("[LinuxAdapter:get_screenshot] scrot failed: %s", e) - return None - - def perform_action(self, action: str, element_id=None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True} - if action == "key" and value: - pyautogui.hotkey(*str(value).lower().split("+")) - return {"success": True} - return {"success": False, "error": f"Unsupported: {action}"} - except Exception as e: - print(f"[LinuxAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# WSL Adapter (WSL 1 + WSL 2: X11 when DISPLAY is set, PowerShell fallback) -# ───────────────────────────────────────────────────────────────────────────── - -class WSLAdapter(LinuxAdapter): - """Adapter for Windows Subsystem for Linux. - - Prefers X11-based tools (wmctrl, mss) when DISPLAY is set. Falls back to - PowerShell / cmd.exe interop, which is always available in both WSL 1 and - WSL 2 via the Windows binary execution layer. - """ - - def __init__(self, config: dict) -> None: - self.config = config - self._has_display = bool(os.environ.get("DISPLAY")) - logger.info( - "[WSLAdapter:__init__] WSL detected; " - "DISPLAY=%s", "set" if self._has_display else "not set (PowerShell fallback active)", - ) - - # ── Window listing ──────────────────────────────────────────────────────── - - def list_windows(self) -> List[WindowInfo]: - if self._has_display: - result = LinuxAdapter.list_windows(self) - if result: - return result - return self._list_windows_ps() - - def _list_windows_ps(self) -> List[WindowInfo]: - """Enumerate visible Windows windows via PowerShell ConvertTo-Json.""" - try: - import json - import subprocess - ps = ( - "Get-Process " - "| Where-Object { $_.MainWindowTitle -ne '' } " - "| Select-Object Id,ProcessName,MainWindowTitle " - "| ConvertTo-Json -Compress" - ) - r = subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], - capture_output=True, text=True, timeout=10, - ) - if r.returncode != 0 or not r.stdout.strip(): - return [] - data = json.loads(r.stdout) - if isinstance(data, dict): - data = [data] - results: List[WindowInfo] = [] - for i, item in enumerate(data or []): - pid = int(item.get("Id", 0)) - name = str(item.get("ProcessName", "unknown")) - title = str(item.get("MainWindowTitle", "")) - if not title: - continue - results.append(WindowInfo( - handle=pid, title=title, process_name=name, pid=pid, - bounds=Bounds(0, 0, 1920, 1080), is_focused=(i == 0), - window_uid=f"wsl:{pid}", - )) - return results - except Exception as e: - logger.debug("[WSLAdapter:_list_windows_ps] %s", e) - return [] - - # ── Screenshot ──────────────────────────────────────────────────────────── - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - if self._has_display: - result = LinuxAdapter.get_screenshot(self, hwnd) - if result: - return result - return self._screenshot_ps() - - def _screenshot_ps(self) -> Optional[bytes]: - """Capture the primary screen via PowerShell, returning PNG bytes.""" - try: - import base64 - import subprocess - # Capture screen to a MemoryStream and emit as base64 — avoids - # WSL↔Windows path translation issues entirely. - ps = ( - "Add-Type -AssemblyName System.Windows.Forms,System.Drawing;" - "$b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds;" - "$bmp=New-Object System.Drawing.Bitmap $b.Width,$b.Height;" - "$g=[System.Drawing.Graphics]::FromImage($bmp);" - "$g.CopyFromScreen($b.Location,[System.Drawing.Point]::Empty,$b.Size);" - "$ms=New-Object System.IO.MemoryStream;" - "$bmp.Save($ms,[System.Drawing.Imaging.ImageFormat]::Png);" - "$g.Dispose();$bmp.Dispose();" - "[Convert]::ToBase64String($ms.ToArray())" - ) - r = subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], - capture_output=True, text=True, timeout=20, - ) - if r.returncode == 0 and r.stdout.strip(): - return base64.b64decode(r.stdout.strip()) - except Exception as e: - logger.debug("[WSLAdapter:_screenshot_ps] %s", e) - return None - - # ── get_windows_above_bounds: returns [] (inherited from LinuxAdapter) ──── - # ── get_element_tree: upgraded by linux_adapter.install_into if pyatspi ── - # ── perform_action: inherited (pyautogui; needs DISPLAY) ───────────────── - - -# ───────────────────────────────────────────────────────────────────────────── -# ScreenObserver (public interface; delegates to platform adapter) -# ───────────────────────────────────────────────────────────────────────────── - -class ScreenObserver: - """ - Platform-aware screen observer. All consumers should program against - this class rather than the platform adapters directly. - """ - - def __init__(self, config: dict): - self.config = config - self._adapter = self._select_adapter() - # Try to upgrade stub adapters to real AX implementations. - try: - if isinstance(self._adapter, MacOSAdapter): - import mac_adapter - if mac_adapter.install_into(self): - logger.info("[ScreenObserver] mac_adapter installed (pyobjc)") - elif isinstance(self._adapter, LinuxAdapter): - import linux_adapter - if linux_adapter.install_into(self): - logger.info("[ScreenObserver] linux_adapter installed (pyatspi)") - except Exception: - logger.exception("real adapter upgrade failed") - - def _select_adapter(self): - if self.config.get("mock", False): - logger.info("[ScreenObserver] Using MockAdapter") - return MockAdapter() - - target = self.config.get("platform", "auto") - # EFFECTIVE_PLATFORM is "WSL" when running inside WSL, otherwise same - # as platform.system(). Explicit config overrides auto-detection. - sys_plat = EFFECTIVE_PLATFORM if target == "auto" else target - - adapters = { - "Windows": WindowsAdapter, - "Darwin": MacOSAdapter, - "Linux": LinuxAdapter, - "WSL": WSLAdapter, - } - - cls = adapters.get(sys_plat) - if cls is None: - logger.warning("[ScreenObserver] Unknown platform '%s'; using MockAdapter", sys_plat) - return MockAdapter() - - try: - return cls(self.config) - except Exception as e: - print(f"[ScreenObserver:_select_adapter] Platform adapter failed: {e}; falling back to Mock") - traceback.print_exc() - return MockAdapter() - - @property - def is_mock(self) -> bool: - return isinstance(self._adapter, MockAdapter) - - def list_windows(self) -> List[WindowInfo]: - return self._adapter.list_windows() - - def get_element_tree(self, hwnd=None, window_uid: Optional[str] = None, - use_cache: bool = True) -> Optional[UIElement]: - """Return the accessibility tree for a window. - - When *window_uid* is supplied the per-window tree cache is consulted: - a fresh cached capture (within ``tree.cache_ttl_s``) is returned - without walking the adapter; a miss walks and stores. Pass - ``use_cache=False`` to force a fresh walk (post-action re-reads); - the fresh capture still refreshes the cache. Calls without a - *window_uid* always walk and are never cached. - """ - tree, _meta = self.get_element_tree_with_meta( - hwnd, window_uid=window_uid, use_cache=use_cache) - return tree - - def get_element_tree_with_meta( - self, hwnd=None, *, window_uid: Optional[str] = None, - use_cache: bool = True, - ) -> Tuple[Optional[UIElement], Dict[str, Any]]: - """get_element_tree plus capture metadata. - - Returns (tree, meta) where meta is - ``{"cache": "hit"|"miss"|"bypass", "capture_ms": int, - "node_count": int}``. - """ - tree_cfg = self.config.get("tree", {}) or {} - ttl = float(tree_cfg.get("cache_ttl_s", 2.0)) - cache = self._tree_cache() - - if use_cache and window_uid and cache is not None: - entry = cache.get(window_uid, ttl_s=ttl) - if entry is not None: - return entry.tree, { - "cache": "hit", - "capture_ms": 0, - "node_count": entry.node_count, - } - - started = time.time() - tree = self._adapter.get_element_tree(hwnd) - capture_ms = int((time.time() - started) * 1000) - node_count = len(tree.flat_list()) if tree is not None else 0 - - if tree is not None and window_uid and cache is not None: - from hashing import tree_hash as _tree_hash - named = sum( - 1 for e in tree.flat_list()[1:] if (e.name or "").strip() - ) - cache.put( - window_uid, - tree=tree, - serialized=tree.to_dict(), - tree_hash=_tree_hash(tree), - max_depth=int(tree_cfg.get("max_depth", 8)), - capture_ms=capture_ms, - node_count=node_count, - named_node_count=named, - ) - - return tree, { - "cache": "bypass" if not use_cache else "miss", - "capture_ms": capture_ms, - "node_count": node_count, - } - - @staticmethod - def _tree_cache(): - """The session-scoped TreeCache (lazy import avoids cycles).""" - try: - from session import get_session - return get_session().tree_cache - except Exception: - return None - - def get_element_subtree(self, hwnd=None, element_path: str = "root", - max_depth: Optional[int] = None, - window_uid: Optional[str] = None, - use_cache: bool = True) -> Optional[UIElement]: - """Return only the subtree rooted at *element_path* ('root.3.2'). - - Resolution order: - 1. a fresh cached full capture (no walk at all), - 2. the adapter's native get_element_subtree (walks just the branch), - 3. full walk + extraction. - The result is depth-limited to *max_depth* levels below the subtree - root and safe to mutate (cache extraction returns copies).""" - tree_cfg = self.config.get("tree", {}) or {} - if max_depth is None: - max_depth = int(tree_cfg.get("max_depth", 8)) - - # 1. Serve from a fresh cached capture. - cache = self._tree_cache() - if use_cache and window_uid and cache is not None: - entry = cache.get(window_uid, - ttl_s=float(tree_cfg.get("cache_ttl_s", 2.0))) - if entry is not None: - sub = find_element_by_path(entry.tree, element_path) - if sub is not None: - return prune_tree_depth(sub, max_depth) - - # 2. Adapter-native scoped walk. - native = getattr(self._adapter, "get_element_subtree", None) - if native is not None: - try: - sub = native(hwnd, element_path, max_depth) - if sub is not None: - return sub - except Exception: - logger.exception("[ScreenObserver:get_element_subtree] " - "adapter subtree walk failed; falling back") - - # 3. Full walk + extraction. - tree = self.get_element_tree(hwnd, window_uid=window_uid, - use_cache=use_cache) - sub = find_element_by_path(tree, element_path) - return prune_tree_depth(sub, max_depth) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - return self._adapter.get_screenshot(hwnd) - - def get_full_display_screenshot(self) -> Optional[bytes]: - """Capture the entire virtual desktop (all monitors combined) as a PNG.""" - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[0]) # 0 = union of all monitors - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - logger.warning(f"[ScreenObserver:get_full_display_screenshot] {e}; falling back") - return self._adapter.get_screenshot() - - def perform_action(self, action: str, element_id: Optional[str] = None, - value: Any = None, hwnd=None) -> Dict: - return self._adapter.perform_action(action, element_id, value, hwnd) - - def window_by_index(self, windows: List[WindowInfo], - index: Optional[int]) -> Optional[WindowInfo]: - """Convenience: return a WindowInfo by list index, or None.""" - if index is None or not windows: - return None - if 0 <= index < len(windows): - return windows[index] - return None - - def window_by_uid(self, windows: List[WindowInfo], - uid: Optional[str]) -> Optional[WindowInfo]: - """Resolve a window by stable uid; returns None if not found.""" - if not uid or not windows: - return None - for w in windows: - if w.window_uid == uid: - return w - return None - - def resolve_window(self, windows: List[WindowInfo], - window_uid: Optional[str], - window_index: Optional[int], - window_title: Optional[str] = None) -> "WindowResolution": - """Resolve a window by uid (preferred), index, or title substring.""" - if window_uid: - info = self.window_by_uid(windows, window_uid) - warning = ("both window_index and window_uid given; window_uid used" - if window_index is not None else None) - return WindowResolution(info=info, warning=warning, - used_uid=True, requested_uid=window_uid) - if window_index is not None: - info = self.window_by_index(windows, window_index) - resolved_uid = info.window_uid if info else None - return WindowResolution(info=info, warning=None, - used_uid=bool(resolved_uid), - requested_uid=resolved_uid) - if window_title: - needle = window_title.lower() - info = next((w for w in windows if needle in (w.title or "").lower()), None) - resolved_uid = info.window_uid if info else None - return WindowResolution(info=info, warning=None, - used_uid=bool(resolved_uid), - requested_uid=resolved_uid) - return WindowResolution(info=None, warning=None, - used_uid=False, requested_uid=None) - - # ── Monitors / DPI (design doc §6.3) ────────────────────────────────────── - - def get_monitors(self) -> List[Dict[str, Any]]: - """Return per-monitor metadata via mss.""" - try: - import mss - with mss.MSS() as sct: - mons = sct.monitors # [0] is union; [1..] are individual - out: List[Dict[str, Any]] = [] - for i, m in enumerate(mons[1:]): - out.append({ - "index": i, - "primary": (i == 0), - "bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - "scale_factor": 1.0, - "logical_bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - "physical_bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - }) - return out - except Exception: - return [] - - # ── Capability discovery (design doc §6.4) ──────────────────────────────── - - def get_capabilities(self) -> Dict[str, Any]: - adapter_name = type(self._adapter).__name__ - is_windows = adapter_name == "WindowsAdapter" - is_macos = adapter_name == "MacOSAdapter" - is_wsl = adapter_name == "WSLAdapter" - is_linux = adapter_name in ("LinuxAdapter", "WSLAdapter") - is_mock = adapter_name == "MockAdapter" - - # Probe optional libs. - def _has(mod: str) -> bool: - try: - __import__(mod) - return True - except Exception: - return False - - if is_macos: - ax_tree = _has("AppKit") or _has("ApplicationServices") or _has("Cocoa") - elif is_linux: - ax_tree = _has("pyatspi") - else: - ax_tree = is_windows or is_mock - - return { - "ok": True, - "platform": EFFECTIVE_PLATFORM, - "adapter": adapter_name, - "version": (self.config.get("mcp", {}) or {}).get("version", "0.2.0"), - "protocol_version": 2, - "supports": { - "accessibility_tree": bool(ax_tree), - "uia_invoke": is_windows, - "occlusion_detection": is_windows or is_mock or _has("Quartz") or _has("Xlib"), - "drag": True, - "ocr": _has("pytesseract"), - "vlm": bool((self.config.get("vlm") or {}).get("enabled") - and (self.config.get("vlm") or {}).get("model")), - "redaction": True, - "scenarios": is_mock, - "tracing": True, - "replay": True, - "image_blur": _has("PIL"), - "wsl_powershell": is_wsl, - # Action capabilities always present via REST + MCP. - "bring_to_foreground": True, - "element_targeting": bool(ax_tree), # click/focus/invoke/set_value via element_id - "observe_with_diff": True, # /api/observe returns diff token - }, - "config": { - "tree_max_depth": (self.config.get("tree", {}) or {}).get("max_depth", 8), - "tree_default_depth": (self.config.get("tree", {}) or {}).get("default_depth", 5), - "ascii_grid": { - "width": (self.config.get("ascii_sketch", {}) or {}).get("grid_width", 110), - "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), - }, - }, - # Per-window last-capture statistics (node_count, - # named_node_count, capture_ms, captured_at) so agents can spot - # accessibility-dark windows without another walk. - "tree_stats": self._last_capture_stats(), - } - - def _last_capture_stats(self) -> Dict[str, Dict[str, Any]]: - cache = self._tree_cache() - try: - return cache.stats() if cache is not None else {} - except Exception: - return {} - - # ── Element occlusion (design doc D14) ──────────────────────────────────── - - def is_element_occluded(self, element_bounds: Bounds, target_hwnd: Any, - all_windows: List[WindowInfo]) -> bool: - """ - True iff every pixel of *element_bounds* is covered by another window - above the target in Z-order, or the element lies entirely off-screen. - - On platforms without Z-order info, returns False (assumed visible). - """ - screen = self.get_screen_bounds() - clipped = _intersect_bounds(element_bounds, screen) - if not clipped: - return True - regions = [clipped] - try: - occluders = self._adapter.get_windows_above_bounds(target_hwnd) - except Exception: - occluders = [] - for occ in occluders: - regions = _subtract_rect(regions, occ) - return not regions - - # ── Visibility helpers ──────────────────────────────────────────────────── - - def get_screen_bounds(self) -> Bounds: - """Return the bounding rect of the combined virtual screen (all monitors).""" - try: - import mss - with mss.MSS() as sct: - m = sct.monitors[0] # index 0 = union of all monitors - return Bounds(m["left"], m["top"], m["width"], m["height"]) - except Exception: - return Bounds(0, 0, 65535, 65535) - - def get_visible_areas(self, target_hwnd: Any, - all_windows: List[WindowInfo]) -> List[Dict]: - """ - Return a list of {x, y, width, height} dicts for the portions of the - window identified by *target_hwnd* that are on-screen and not covered - by windows above it in Z-order. - - On Windows the Z-order is queried precisely via win32gui. - On macOS/Linux Z-order is unavailable, so the full clipped-to-screen - bounds are returned (assuming the window is on top). - """ - target = next((w for w in all_windows if w.handle == target_hwnd), None) - if target is None: - return [] - - screen = self.get_screen_bounds() - clipped = _intersect_bounds(target.bounds, screen) - if not clipped: - return [] - - visible: List[Bounds] = [clipped] - occluders = self._adapter.get_windows_above_bounds(target_hwnd) - for occ in occluders: - visible = _subtract_rect(visible, occ) - - return [b.to_dict() for b in visible] - - def bring_to_foreground(self, target_hwnd: Any, - all_windows: List[WindowInfo]) -> Dict: - """ - Bring a window to the foreground. - - Strategy (in order) - ------------------- - 1. Platform API — SetForegroundWindow / NSRunningApplication.activate / - wmctrl. Does not click, so it cannot accidentally maximise the window - and works even when the window is fully occluded. - 2. Click on title bar — fallback when the API call is unavailable or - blocked (e.g. Windows foreground-lock policy). Requires at least one - visible pixel; returns an error only when this is also impossible. - """ - target = next((w for w in all_windows if w.handle == target_hwnd), None) - if target is None: - return {"success": False, "error": "Window not found"} - - # ── 1. Try platform API first ───────────────────────────────────────── - api_ok, api_note = self._activate_via_api(target_hwnd, target) - if api_ok: - return {"success": True, "action": "activate_api", - "window": target.title, "note": api_note} - - # ── 2. Fall back to clicking the title bar ──────────────────────────── - regions = self.get_visible_areas(target_hwnd, all_windows) - if not regions: - return { - "success": False, - "error": ( - "Window has no visible area (fully occluded or off-screen) " - f"and platform API also failed: {api_note}" - ), - } - - best = min(regions, key=lambda r: (r["y"], -r["width"])) - click_x, click_y = self._title_bar_click_point(target_hwnd, target, best) - result = self.perform_action("click_at", - value={"x": click_x, "y": click_y, - "button": "left", "double": False}) - result["clicked_x"] = click_x - result["clicked_y"] = click_y - result.setdefault("action", "click_title_bar") - result["api_note"] = api_note - return result - - def _activate_via_api(self, hwnd: Any, - info: "WindowInfo") -> tuple: - """ - Attempt to raise *hwnd* using the platform's native window-focus API. - - Returns (success: bool, note: str). Never raises — all errors are - caught and returned as (False, reason). - """ - if PLATFORM == "Windows": - return self._activate_windows(hwnd) - if PLATFORM == "Darwin": - return self._activate_macos(info) - if PLATFORM == "Linux": - # WSL: try X11 tools (wmctrl/xdotool) when DISPLAY is set; they - # will gracefully fail and return (False, reason) when absent. - return self._activate_linux(hwnd) - return False, f"Platform {PLATFORM!r} has no API activate implementation" - - def _activate_windows(self, hwnd: int) -> tuple: - try: - import ctypes - import ctypes.wintypes - user32 = ctypes.windll.user32 # type: ignore[attr-defined] - kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] - - # Restore if minimised (IsIconic) or hidden. - if user32.IsIconic(hwnd): - user32.ShowWindow(hwnd, 9) # SW_RESTORE - - def _is_foreground() -> bool: - return user32.GetForegroundWindow() == hwnd - - if _is_foreground(): - return True, "already foreground" - - # ── Attempt 1: AttachThreadInput trick ──────────────────────────── - fg_hwnd = user32.GetForegroundWindow() - fg_tid = user32.GetWindowThreadProcessId(fg_hwnd, None) - this_tid = kernel32.GetCurrentThreadId() - attached = False - if fg_tid and fg_tid != this_tid: - attached = bool(user32.AttachThreadInput(this_tid, fg_tid, True)) - try: - user32.SetForegroundWindow(hwnd) - finally: - if attached: - user32.AttachThreadInput(this_tid, fg_tid, False) - - if _is_foreground(): - return True, "SetForegroundWindow (AttachThreadInput)" - - # ── Attempt 2: synthesise a keypress to acquire foreground lock ─── - # Windows grants foreground rights to processes that just received - # user input. A zero-vkey keybd_event satisfies that requirement. - KEYEVENTF_KEYUP = 0x0002 - user32.keybd_event(0, 0, 0, 0) - user32.keybd_event(0, 0, KEYEVENTF_KEYUP, 0) - user32.SetForegroundWindow(hwnd) - - if _is_foreground(): - return True, "SetForegroundWindow (keybd_event unlock)" - - # ── Attempt 3: BringWindowToTop + SetForegroundWindow ───────────── - user32.BringWindowToTop(hwnd) - user32.SetForegroundWindow(hwnd) - - if _is_foreground(): - return True, "BringWindowToTop + SetForegroundWindow" - - return False, "SetForegroundWindow failed after all attempts" - except Exception as e: - return False, f"Windows API error: {e}" - - def _activate_macos(self, info: "WindowInfo") -> tuple: - try: - import AppKit - ws = AppKit.NSWorkspace.sharedWorkspace() - apps = ws.runningApplications() - pid = info.pid - for app in apps: - if int(app.processIdentifier()) == pid: - NSApplicationActivateIgnoringOtherApps = 1 << 1 - app.activateWithOptions_(NSApplicationActivateIgnoringOtherApps) - return True, f"NSRunningApplication.activate pid={pid}" - return False, f"No running application found for pid={pid}" - except Exception as e: - return False, f"macOS API error: {e}" - - def _activate_linux(self, hwnd: int) -> tuple: - import subprocess - # Try wmctrl first (X11; commonly available). - try: - r = subprocess.run( - ["wmctrl", "-ia", hex(hwnd)], - capture_output=True, timeout=3, - ) - if r.returncode == 0: - return True, f"wmctrl -ia {hex(hwnd)} succeeded" - except FileNotFoundError: - pass - except Exception as e: - return False, f"wmctrl error: {e}" - # Try xdotool as a second option. - try: - r = subprocess.run( - ["xdotool", "windowactivate", "--sync", str(hwnd)], - capture_output=True, timeout=3, - ) - if r.returncode == 0: - return True, f"xdotool windowactivate {hwnd} succeeded" - return False, f"xdotool returned {r.returncode}: {r.stderr.decode().strip()}" - except FileNotFoundError: - return False, "neither wmctrl nor xdotool found" - except Exception as e: - return False, f"xdotool error: {e}" - - # ── Title-bar targeting (used by bring_to_foreground) ──────────────────── - - # Reserve these pixel margins for native window-control widgets. - _LEFT_CONTROL_MARGIN = 90 # macOS traffic-lights - _RIGHT_CONTROL_MARGIN = 160 # Windows / Linux minimize/maximize/close - _MIN_SAFE_WIDTH = 260 # below this, fall back to centre - - _TITLEBAR_ROLES = {"TitleBar", "AXTitleBar", "title bar", "Title bar", - "AXWindow"} - - def _title_bar_click_point(self, target_hwnd: Any, - window: Optional["WindowInfo"], - region: Dict) -> tuple: - """Return (x, y) for a safe click inside the title bar of *region*.""" - # 1. Try the accessibility tree. - try: - tree = self.get_element_tree(target_hwnd) - except Exception: - tree = None - bar = self._find_title_bar(tree, window.title if window else "") if tree else None - if bar is not None and bar.bounds and bar.bounds.width > 0: - tb = bar.bounds - # Intersect with the visible region so we don't click off-screen. - x1 = max(region["x"], tb.x) - y1 = max(region["y"], tb.y) - x2 = min(region["x"] + region["width"], tb.right) - y2 = min(region["y"] + region["height"], tb.bottom) - if x2 > x1 and y2 > y1: - # Click 1/4 from the left of the title bar's visible width, - # but skip the leftmost control margin. - safe_left = x1 + min(self._LEFT_CONTROL_MARGIN, (x2 - x1) // 4) - safe_right = x2 - min(self._RIGHT_CONTROL_MARGIN, (x2 - x1) // 4) - if safe_right <= safe_left: - cx = (x1 + x2) // 2 - else: - quarter = x1 + (x2 - x1) // 4 - cx = max(safe_left, min(safe_right, quarter)) - cy = y1 + max(1, (y2 - y1) // 2) - return cx, cy - - # 2. Fall back to the visible region's top strip. - rx = region["x"] - rw = region["width"] - ry = region["y"] - rh = region["height"] - if rw < self._MIN_SAFE_WIDTH: - cx = rx + rw // 2 - else: - safe_left = rx + self._LEFT_CONTROL_MARGIN - safe_right = rx + rw - self._RIGHT_CONTROL_MARGIN - candidate = rx + rw // 4 - if safe_right <= safe_left: - cx = rx + rw // 2 - else: - cx = max(safe_left, min(safe_right, candidate)) - title_bar_offset = min(12, max(1, (rh - 1) // 2)) - cy = ry + title_bar_offset - # Clamp inside the region. - cx = max(rx, min(rx + rw - 1, cx)) - cy = max(ry, min(ry + rh - 1, cy)) - return cx, cy - - def _find_title_bar(self, root: Optional["UIElement"], - window_title: str) -> Optional["UIElement"]: - """Locate a title-bar-like element by role or by matching name. - - Returns None for the root window itself (clicking the whole window's - center is precisely what we are trying to avoid). - """ - if root is None: - return None - from collections import deque - queue = deque([(root, 0)]) - while queue: - elem, depth = queue.popleft() - # Never return the root — its centre is the window body, not the - # title bar. - if depth > 0: - if (elem.role or "") in self._TITLEBAR_ROLES: - return elem - if window_title and (elem.name or "").strip() == window_title.strip(): - # Must be near the top edge AND short — a real title bar. - if (elem.bounds and root.bounds - and (elem.bounds.y - root.bounds.y) < 40 - and elem.bounds.height <= 48): - return elem - if depth > 3: - continue - for c in elem.children: - queue.append((c, depth + 1)) - return None - - -# ───────────────────────────────────────────────────────────────────────────── -# Rectangle geometry helpers -# ───────────────────────────────────────────────────────────────────────────── - -def _intersect_bounds(a: Bounds, b: Bounds) -> Optional[Bounds]: - x1 = max(a.x, b.x) - y1 = max(a.y, b.y) - x2 = min(a.right, b.right) - y2 = min(a.bottom, b.bottom) - if x2 <= x1 or y2 <= y1: - return None - return Bounds(x1, y1, x2 - x1, y2 - y1) - - -def _subtract_rect(rects: List[Bounds], occluder: Bounds) -> List[Bounds]: - """Subtract occluder from each rect, splitting into up to 4 sub-rects.""" - result: List[Bounds] = [] - for r in rects: - ix1 = max(r.x, occluder.x) - iy1 = max(r.y, occluder.y) - ix2 = min(r.right, occluder.right) - iy2 = min(r.bottom, occluder.bottom) - - if ix2 <= ix1 or iy2 <= iy1: - result.append(r) - continue - - # Top strip - if iy1 > r.y: - result.append(Bounds(r.x, r.y, r.width, iy1 - r.y)) - # Bottom strip - if iy2 < r.bottom: - result.append(Bounds(r.x, iy2, r.width, r.bottom - iy2)) - # Left strip (height = intersection height) - if ix1 > r.x: - result.append(Bounds(r.x, iy1, ix1 - r.x, iy2 - iy1)) - # Right strip (height = intersection height) - if ix2 < r.right: - result.append(Bounds(ix2, iy1, r.right - ix2, iy2 - iy1)) - return result diff --git a/observer/__init__.py b/observer/__init__.py new file mode 100644 index 0000000..3009ce8 --- /dev/null +++ b/observer/__init__.py @@ -0,0 +1,82 @@ +""" +observer — Core screen observation package (package form of observer.py). + +Provides a platform-aware ScreenObserver that exposes a uniform interface +for: enumerating windows, walking the accessibility element tree, capturing +screenshots, and dispatching input actions. Platform adapters (Windows/macOS/ +Linux/WSL/Mock) share a common protocol and are selected automatically at +runtime. + +Data model +---------- + Bounds — screen-coordinate bounding rectangle + UIElement — one node of the accessibility tree + WindowInfo — top-level window metadata + +P3 decomposition: the implementation now lives in submodules (models, +platform_info, adapters/, core, activation, occlusion). This __init__ +re-exports the entire pre-split public surface so `import observer` / +`from observer import X` keep working unchanged. +""" + +from __future__ import annotations + +from observer.adapters import ( + LinuxAdapter, + MacOSAdapter, + MockAdapter, + WindowsAdapter, + WSLAdapter, + _UIA_ACCEL_KEY, + _UIA_ACCESS_KEY, + _UIA_AUTOMATION_ID, + _UIA_BOUNDING_RECT, + _UIA_CACHED_PROPS, + _UIA_CTRL_TYPE, + _UIA_ENABLED, + _UIA_EXPAND_STATE, + _UIA_FOCUSED, + _UIA_HELP_TEXT, + _UIA_IS_SELECTED, + _UIA_NAME, + _UIA_RANGE_MAX, + _UIA_RANGE_MIN, + _UIA_RANGE_VALUE, + _UIA_SCOPE_CHILDREN, + _UIA_TYPE_TO_ROLE, + _UIA_VALUE, +) +from observer.activation import ActivationMixin +from observer.core import ScreenObserver +from observer.models import ( + Bounds, + UIElement, + WindowInfo, + WindowResolution, + find_element_by_path, + prune_tree_depth, +) +from observer.occlusion import OcclusionMixin, _intersect_bounds, _subtract_rect +from observer.platform_info import EFFECTIVE_PLATFORM, IS_WSL, PLATFORM, _is_wsl + +__all__ = [ + # models + "Bounds", "UIElement", "WindowInfo", "WindowResolution", + "find_element_by_path", "prune_tree_depth", + # platform detection + "EFFECTIVE_PLATFORM", "IS_WSL", "PLATFORM", "_is_wsl", + # adapters + "LinuxAdapter", "MacOSAdapter", "MockAdapter", "WindowsAdapter", + "WSLAdapter", + # UIA constants + "_UIA_ACCEL_KEY", "_UIA_ACCESS_KEY", "_UIA_AUTOMATION_ID", + "_UIA_BOUNDING_RECT", "_UIA_CACHED_PROPS", "_UIA_CTRL_TYPE", + "_UIA_ENABLED", "_UIA_EXPAND_STATE", "_UIA_FOCUSED", "_UIA_HELP_TEXT", + "_UIA_IS_SELECTED", "_UIA_NAME", "_UIA_RANGE_MAX", "_UIA_RANGE_MIN", + "_UIA_RANGE_VALUE", "_UIA_SCOPE_CHILDREN", "_UIA_TYPE_TO_ROLE", + "_UIA_VALUE", + # core + mixins + "ScreenObserver", "ActivationMixin", "OcclusionMixin", + # geometry helpers + "_intersect_bounds", "_subtract_rect", +] diff --git a/observer/activation.py b/observer/activation.py new file mode 100644 index 0000000..60286a5 --- /dev/null +++ b/observer/activation.py @@ -0,0 +1,287 @@ +""" +Window activation: bring_to_foreground, platform API +activation strategies and title-bar click targeting (mixin consumed by +ScreenObserver). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from observer.models import UIElement, WindowInfo +from observer.platform_info import PLATFORM + + +class ActivationMixin: + """Foreground-activation methods of ScreenObserver.""" + + # Provided by the concrete ScreenObserver / OcclusionMixin. + _adapter: Any + + if TYPE_CHECKING: # pragma: no cover — typing-only method stubs + def get_element_tree(self, hwnd: Any = None, + window_uid: Optional[str] = None, + use_cache: bool = True + ) -> Optional[UIElement]: ... + + def get_visible_areas(self, target_hwnd: Any, + all_windows: List[WindowInfo] + ) -> List[Dict]: ... + + def perform_action(self, action: str, + element_id: Optional[str] = None, + value: Any = None, hwnd: Any = None + ) -> Dict: ... + def bring_to_foreground(self, target_hwnd: Any, + all_windows: List[WindowInfo]) -> Dict: + """ + Bring a window to the foreground. + + Strategy (in order) + ------------------- + 1. Platform API — SetForegroundWindow / NSRunningApplication.activate / + wmctrl. Does not click, so it cannot accidentally maximise the window + and works even when the window is fully occluded. + 2. Click on title bar — fallback when the API call is unavailable or + blocked (e.g. Windows foreground-lock policy). Requires at least one + visible pixel; returns an error only when this is also impossible. + """ + target = next((w for w in all_windows if w.handle == target_hwnd), None) + if target is None: + return {"success": False, "error": "Window not found"} + + # ── 1. Try platform API first ───────────────────────────────────────── + api_ok, api_note = self._activate_via_api(target_hwnd, target) + if api_ok: + return {"success": True, "action": "activate_api", + "window": target.title, "note": api_note} + + # ── 2. Fall back to clicking the title bar ──────────────────────────── + regions = self.get_visible_areas(target_hwnd, all_windows) + if not regions: + return { + "success": False, + "error": ( + "Window has no visible area (fully occluded or off-screen) " + f"and platform API also failed: {api_note}" + ), + } + + best = min(regions, key=lambda r: (r["y"], -r["width"])) + click_x, click_y = self._title_bar_click_point(target_hwnd, target, best) + result = self.perform_action("click_at", + value={"x": click_x, "y": click_y, + "button": "left", "double": False}) + result["clicked_x"] = click_x + result["clicked_y"] = click_y + result.setdefault("action", "click_title_bar") + result["api_note"] = api_note + return result + + def _activate_via_api(self, hwnd: Any, + info: "WindowInfo") -> tuple: + """ + Attempt to raise *hwnd* using the platform's native window-focus API. + + Returns (success: bool, note: str). Never raises — all errors are + caught and returned as (False, reason). + """ + if PLATFORM == "Windows": + return self._activate_windows(hwnd) + if PLATFORM == "Darwin": + return self._activate_macos(info) + if PLATFORM == "Linux": + # WSL: try X11 tools (wmctrl/xdotool) when DISPLAY is set; they + # will gracefully fail and return (False, reason) when absent. + return self._activate_linux(hwnd) + return False, f"Platform {PLATFORM!r} has no API activate implementation" + + def _activate_windows(self, hwnd: int) -> tuple: + try: + import ctypes + import ctypes.wintypes + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + + # Restore if minimised (IsIconic) or hidden. + if user32.IsIconic(hwnd): + user32.ShowWindow(hwnd, 9) # SW_RESTORE + + def _is_foreground() -> bool: + return user32.GetForegroundWindow() == hwnd + + if _is_foreground(): + return True, "already foreground" + + # ── Attempt 1: AttachThreadInput trick ──────────────────────────── + fg_hwnd = user32.GetForegroundWindow() + fg_tid = user32.GetWindowThreadProcessId(fg_hwnd, None) + this_tid = kernel32.GetCurrentThreadId() + attached = False + if fg_tid and fg_tid != this_tid: + attached = bool(user32.AttachThreadInput(this_tid, fg_tid, True)) + try: + user32.SetForegroundWindow(hwnd) + finally: + if attached: + user32.AttachThreadInput(this_tid, fg_tid, False) + + if _is_foreground(): + return True, "SetForegroundWindow (AttachThreadInput)" + + # ── Attempt 2: synthesise a keypress to acquire foreground lock ─── + # Windows grants foreground rights to processes that just received + # user input. A zero-vkey keybd_event satisfies that requirement. + KEYEVENTF_KEYUP = 0x0002 + user32.keybd_event(0, 0, 0, 0) + user32.keybd_event(0, 0, KEYEVENTF_KEYUP, 0) + user32.SetForegroundWindow(hwnd) + + if _is_foreground(): + return True, "SetForegroundWindow (keybd_event unlock)" + + # ── Attempt 3: BringWindowToTop + SetForegroundWindow ───────────── + user32.BringWindowToTop(hwnd) + user32.SetForegroundWindow(hwnd) + + if _is_foreground(): + return True, "BringWindowToTop + SetForegroundWindow" + + return False, "SetForegroundWindow failed after all attempts" + except Exception as e: + return False, f"Windows API error: {e}" + + def _activate_macos(self, info: "WindowInfo") -> tuple: + try: + import AppKit + ws = AppKit.NSWorkspace.sharedWorkspace() + apps = ws.runningApplications() + pid = info.pid + for app in apps: + if int(app.processIdentifier()) == pid: + NSApplicationActivateIgnoringOtherApps = 1 << 1 + app.activateWithOptions_(NSApplicationActivateIgnoringOtherApps) + return True, f"NSRunningApplication.activate pid={pid}" + return False, f"No running application found for pid={pid}" + except Exception as e: + return False, f"macOS API error: {e}" + + def _activate_linux(self, hwnd: int) -> tuple: + import subprocess + # Try wmctrl first (X11; commonly available). + try: + r = subprocess.run( + ["wmctrl", "-ia", hex(hwnd)], + capture_output=True, timeout=3, + ) + if r.returncode == 0: + return True, f"wmctrl -ia {hex(hwnd)} succeeded" + except FileNotFoundError: + pass + except Exception as e: + return False, f"wmctrl error: {e}" + # Try xdotool as a second option. + try: + r = subprocess.run( + ["xdotool", "windowactivate", "--sync", str(hwnd)], + capture_output=True, timeout=3, + ) + if r.returncode == 0: + return True, f"xdotool windowactivate {hwnd} succeeded" + return False, f"xdotool returned {r.returncode}: {r.stderr.decode().strip()}" + except FileNotFoundError: + return False, "neither wmctrl nor xdotool found" + except Exception as e: + return False, f"xdotool error: {e}" + + # ── Title-bar targeting (used by bring_to_foreground) ──────────────────── + + # Reserve these pixel margins for native window-control widgets. + _LEFT_CONTROL_MARGIN = 90 # macOS traffic-lights + _RIGHT_CONTROL_MARGIN = 160 # Windows / Linux minimize/maximize/close + _MIN_SAFE_WIDTH = 260 # below this, fall back to centre + + _TITLEBAR_ROLES = {"TitleBar", "AXTitleBar", "title bar", "Title bar", + "AXWindow"} + + def _title_bar_click_point(self, target_hwnd: Any, + window: Optional["WindowInfo"], + region: Dict) -> tuple: + """Return (x, y) for a safe click inside the title bar of *region*.""" + # 1. Try the accessibility tree. + try: + tree = self.get_element_tree(target_hwnd) + except Exception: + tree = None + bar = self._find_title_bar(tree, window.title if window else "") if tree else None + if bar is not None and bar.bounds and bar.bounds.width > 0: + tb = bar.bounds + # Intersect with the visible region so we don't click off-screen. + x1 = max(region["x"], tb.x) + y1 = max(region["y"], tb.y) + x2 = min(region["x"] + region["width"], tb.right) + y2 = min(region["y"] + region["height"], tb.bottom) + if x2 > x1 and y2 > y1: + # Click 1/4 from the left of the title bar's visible width, + # but skip the leftmost control margin. + safe_left = x1 + min(self._LEFT_CONTROL_MARGIN, (x2 - x1) // 4) + safe_right = x2 - min(self._RIGHT_CONTROL_MARGIN, (x2 - x1) // 4) + if safe_right <= safe_left: + cx = (x1 + x2) // 2 + else: + quarter = x1 + (x2 - x1) // 4 + cx = max(safe_left, min(safe_right, quarter)) + cy = y1 + max(1, (y2 - y1) // 2) + return cx, cy + + # 2. Fall back to the visible region's top strip. + rx = region["x"] + rw = region["width"] + ry = region["y"] + rh = region["height"] + if rw < self._MIN_SAFE_WIDTH: + cx = rx + rw // 2 + else: + safe_left = rx + self._LEFT_CONTROL_MARGIN + safe_right = rx + rw - self._RIGHT_CONTROL_MARGIN + candidate = rx + rw // 4 + if safe_right <= safe_left: + cx = rx + rw // 2 + else: + cx = max(safe_left, min(safe_right, candidate)) + title_bar_offset = min(12, max(1, (rh - 1) // 2)) + cy = ry + title_bar_offset + # Clamp inside the region. + cx = max(rx, min(rx + rw - 1, cx)) + cy = max(ry, min(ry + rh - 1, cy)) + return cx, cy + + def _find_title_bar(self, root: Optional["UIElement"], + window_title: str) -> Optional["UIElement"]: + """Locate a title-bar-like element by role or by matching name. + + Returns None for the root window itself (clicking the whole window's + center is precisely what we are trying to avoid). + """ + if root is None: + return None + from collections import deque + queue = deque([(root, 0)]) + while queue: + elem, depth = queue.popleft() + # Never return the root — its centre is the window body, not the + # title bar. + if depth > 0: + if (elem.role or "") in self._TITLEBAR_ROLES: + return elem + if window_title and (elem.name or "").strip() == window_title.strip(): + # Must be near the top edge AND short — a real title bar. + if (elem.bounds and root.bounds + and (elem.bounds.y - root.bounds.y) < 40 + and elem.bounds.height <= 48): + return elem + if depth > 3: + continue + for c in elem.children: + queue.append((c, depth + 1)) + return None diff --git a/observer/adapters/__init__.py b/observer/adapters/__init__.py new file mode 100644 index 0000000..7ed2dd8 --- /dev/null +++ b/observer/adapters/__init__.py @@ -0,0 +1,47 @@ +""" +Platform adapters for ScreenObserver. + +Split out of observer.py (P3); behavior is unchanged. The top-level +mac_adapter.py / linux_adapter.py modules still hold the optional +pyobjc / pyatspi runtime upgrades that ScreenObserver installs over the +macOS / Linux stub adapters defined here. +""" + +from __future__ import annotations + +from observer.adapters.linux import LinuxAdapter +from observer.adapters.macos import MacOSAdapter +from observer.adapters.mock import MockAdapter +from observer.adapters.windows import ( + WindowsAdapter, + _UIA_ACCEL_KEY, + _UIA_ACCESS_KEY, + _UIA_AUTOMATION_ID, + _UIA_BOUNDING_RECT, + _UIA_CACHED_PROPS, + _UIA_CTRL_TYPE, + _UIA_ENABLED, + _UIA_EXPAND_STATE, + _UIA_FOCUSED, + _UIA_HELP_TEXT, + _UIA_IS_SELECTED, + _UIA_NAME, + _UIA_RANGE_MAX, + _UIA_RANGE_MIN, + _UIA_RANGE_VALUE, + _UIA_SCOPE_CHILDREN, + _UIA_TYPE_TO_ROLE, + _UIA_VALUE, +) +from observer.adapters.wsl import WSLAdapter + +__all__ = [ + "LinuxAdapter", "MacOSAdapter", "MockAdapter", "WindowsAdapter", + "WSLAdapter", + "_UIA_ACCEL_KEY", "_UIA_ACCESS_KEY", "_UIA_AUTOMATION_ID", + "_UIA_BOUNDING_RECT", "_UIA_CACHED_PROPS", "_UIA_CTRL_TYPE", + "_UIA_ENABLED", "_UIA_EXPAND_STATE", "_UIA_FOCUSED", "_UIA_HELP_TEXT", + "_UIA_IS_SELECTED", "_UIA_NAME", "_UIA_RANGE_MAX", "_UIA_RANGE_MIN", + "_UIA_RANGE_VALUE", "_UIA_SCOPE_CHILDREN", "_UIA_TYPE_TO_ROLE", + "_UIA_VALUE", +] diff --git a/observer/adapters/linux.py b/observer/adapters/linux.py new file mode 100644 index 0000000..ff74605 --- /dev/null +++ b/observer/adapters/linux.py @@ -0,0 +1,147 @@ +""" +Linux adapter (screenshot works; AT-SPI tree is a stub +pending pyatspi work — upgraded at runtime by top-level linux_adapter.py). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, UIElement, WindowInfo + +logger = logging.getLogger(__name__) + + +class LinuxAdapter: + def __init__(self, config: dict): + self.config = config + logger.info("[LinuxAdapter:__init__] Linux adapter loaded (AT-SPI tree is stub)") + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + return [] # Z-order unavailable without Xlib/wnck + + def list_windows(self) -> List[WindowInfo]: + # Prefer wmctrl when present; otherwise fall back to Xlib so the + # adapter still works on systems without the wmctrl binary installed. + import subprocess + try: + r = subprocess.run(["wmctrl", "-lG"], capture_output=True, text=True, timeout=5) + except FileNotFoundError: + logger.info("[LinuxAdapter:list_windows] wmctrl not installed; trying Xlib fallback") + return self._list_windows_xlib() + except Exception as e: + logger.warning("[LinuxAdapter:list_windows] wmctrl failed: %s; trying Xlib fallback", e) + return self._list_windows_xlib() + try: + windows = [] + for line in r.stdout.strip().split("\n"): + if not line: + continue + parts = line.split(None, 8) + if len(parts) < 8: + continue + hwnd = int(parts[0], 16) + x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]) + title = parts[8] if len(parts) > 8 else "(no title)" + windows.append(WindowInfo(hwnd, title, title, 0, + Bounds(x, y, w, h), False, + window_uid=f"x11:{hwnd:x}")) + return windows + except Exception as e: + print(f"[LinuxAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def _list_windows_xlib(self) -> List[WindowInfo]: + try: + from Xlib import display, X # noqa: F401 + from Xlib.error import XError + except ImportError: + logger.warning( + "[LinuxAdapter:list_windows] neither 'wmctrl' nor python-xlib " + "is available; install one (e.g. `apt install wmctrl` or " + "`pip install python-xlib`) to enumerate windows" + ) + return [] + try: + d = display.Display() + root = d.screen().root + NET_CLIENT_LIST = d.intern_atom("_NET_CLIENT_LIST") + NET_WM_NAME = d.intern_atom("_NET_WM_NAME") + UTF8_STRING = d.intern_atom("UTF8_STRING") + prop = root.get_full_property(NET_CLIENT_LIST, X.AnyPropertyType) + if prop is None: + return [] + windows: List[WindowInfo] = [] + for wid in prop.value: + try: + w = d.create_resource_object("window", wid) + geom = w.get_geometry() + coords = w.translate_coords(root, 0, 0) + x, y = -coords.x, -coords.y + title = "" + name_prop = w.get_full_property(NET_WM_NAME, UTF8_STRING) + if name_prop and name_prop.value: + title = name_prop.value.decode("utf-8", errors="replace") + else: + wm_name = w.get_wm_name() + if wm_name: + title = wm_name if isinstance(wm_name, str) else wm_name.decode("utf-8", errors="replace") + title = title or "(no title)" + windows.append(WindowInfo(int(wid), title, title, 0, + Bounds(x, y, geom.width, geom.height), False, + window_uid=f"x11:{int(wid):x}")) + except XError: + continue + return windows + except Exception as e: + print(f"[LinuxAdapter:_list_windows_xlib] {e}") + traceback.print_exc() + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + logger.warning("[LinuxAdapter:get_element_tree] Full tree requires pyatspi; returning stub") + return UIElement("root", "Linux Application (AT-SPI stub)", "Window", + bounds=Bounds(0, 0, 1920, 1080)) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + # mss needs a running X server (DISPLAY must be set). + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[1]) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + logger.debug("[LinuxAdapter:get_screenshot] mss failed: %s; trying scrot", e) + # scrot can write PNG directly to stdout (-z suppresses notifications). + try: + import subprocess + r = subprocess.run(["scrot", "-z", "-"], capture_output=True, timeout=10) + if r.returncode == 0 and r.stdout: + return r.stdout + except Exception as e: + logger.debug("[LinuxAdapter:get_screenshot] scrot failed: %s", e) + return None + + def perform_action(self, action: str, element_id=None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True} + if action == "key" and value: + pyautogui.hotkey(*str(value).lower().split("+")) + return {"success": True} + return {"success": False, "error": f"Unsupported: {action}"} + except Exception as e: + print(f"[LinuxAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/macos.py b/observer/adapters/macos.py new file mode 100644 index 0000000..8c83cd8 --- /dev/null +++ b/observer/adapters/macos.py @@ -0,0 +1,76 @@ +""" +macOS adapter (screenshot works; AX tree is a stub +pending pyobjc work — upgraded at runtime by top-level mac_adapter.py). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, UIElement, WindowInfo + +logger = logging.getLogger(__name__) + + +class MacOSAdapter: + def __init__(self, config: dict): + self.config = config + logger.info("[MacOSAdapter:__init__] macOS adapter loaded (AX tree is stub)") + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + return [] # Z-order unavailable without Quartz CGWindowList + + def list_windows(self) -> List[WindowInfo]: + try: + import subprocess + script = ('tell application "System Events" to get name of every ' + 'process whose background only is false') + r = subprocess.run(["osascript", "-e", script], + capture_output=True, text=True, timeout=5) + apps = [a.strip() for a in r.stdout.split(",") if a.strip()] + return [WindowInfo(i, a, a, 0, Bounds(0, 0, 1920, 1080), i == 0, + window_uid=f"mac:{i}") + for i, a in enumerate(apps)] + except Exception as e: + print(f"[MacOSAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + logger.warning("[MacOSAdapter:get_element_tree] Full AX tree requires pyobjc; returning stub") + return UIElement("root", "macOS Application (AX tree stub)", "Window", + bounds=Bounds(0, 0, 1920, 1080)) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[1]) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + print(f"[MacOSAdapter:get_screenshot] {e}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id=None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True} + if action == "key" and value: + pyautogui.hotkey(*str(value).lower().split("+")) + return {"success": True} + return {"success": False, "error": f"Unsupported: {action}"} + except Exception as e: + print(f"[MacOSAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/mock.py b/observer/adapters/mock.py new file mode 100644 index 0000000..9fd995d --- /dev/null +++ b/observer/adapters/mock.py @@ -0,0 +1,184 @@ +""" +Mock adapter (no OS dependencies; safe anywhere). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import traceback +from typing import Any, Callable, Dict, List, Optional + +from observer.models import ( + Bounds, UIElement, WindowInfo, find_element_by_path, prune_tree_depth, +) + + +class MockAdapter: + """Synthetic data adapter for development and testing.""" + + def __init__(self) -> None: + import secrets as _s + self._nonce = _s.token_hex(4) + # Optional scenario hook (design doc §15.5). Set by main.py when + # --scenario is supplied; methods route through the scenario when + # active so that input actions can drive state transitions. + self.scenario: Optional[Any] = None + # Test hooks (P1 perf work): capture_count increments on every tree + # walk so tests can assert cache hits avoided adapter work; + # tree_mutator, when set, post-processes (or replaces) each captured + # tree so tests can simulate UI changes between captures. + self.capture_count: int = 0 + self.tree_mutator: Optional[Callable[[UIElement], UIElement]] = None + + def get_windows_above_bounds(self, hwnd) -> List["Bounds"]: + return [] # Mock assumes the target window is on top + + def list_windows(self) -> List[WindowInfo]: + if self.scenario is not None: + return self.scenario.list_windows(self._nonce) + return [ + WindowInfo(1001, "Untitled — Notepad", "notepad.exe", 1234, + Bounds(80, 60, 800, 600), True, + window_uid=f"mock:0:{self._nonce}"), + WindowInfo(1002, "GitHub · Where software is built — Google Chrome", + "chrome.exe", 5678, Bounds(0, 0, 1920, 1050), False, + window_uid=f"mock:1:{self._nonce}"), + WindowInfo(1003, "screen_observer.py — Visual Studio Code", + "code.exe", 9012, Bounds(960, 0, 960, 1050), False, + window_uid=f"mock:2:{self._nonce}"), + ] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + self.capture_count += 1 + tree = self._build_tree(hwnd) + if tree is not None and self.tree_mutator is not None: + mutated = self.tree_mutator(tree) + if mutated is not None: + tree = mutated + return tree + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path*, to *max_depth* + levels below it. The mock world is synthetic, so this navigates the + positional element-id path of a fresh capture.""" + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + def _build_tree(self, hwnd=None) -> Optional[UIElement]: + if self.scenario is not None: + return self.scenario.get_element_tree(hwnd) + root = UIElement("root", "Untitled — Notepad", "Window", + bounds=Bounds(80, 60, 800, 600)) + + # Menu bar + menubar = UIElement("root.0", "MenuBar", "MenuBar", + bounds=Bounds(80, 60, 800, 22)) + for i, lbl in enumerate(["File", "Edit", "Format", "View", "Help"]): + menubar.children.append(UIElement( + f"root.0.{i}", lbl, "MenuItem", + bounds=Bounds(80 + i * 58, 60, 56, 22) + )) + root.children.append(menubar) + + # Main editing area + editor = UIElement( + "root.1", "Text Editor", "Document", + value="Hello, world!\nThis is a test document.\nLine 3 has some content here.\n", + bounds=Bounds(80, 82, 800, 514), focused=True + ) + root.children.append(editor) + + # Horizontal scroll bar + hscroll = UIElement("root.2", "Horizontal ScrollBar", "ScrollBar", + bounds=Bounds(80, 596, 784, 18)) + root.children.append(hscroll) + + # Vertical scroll bar + vscroll = UIElement("root.3", "Vertical ScrollBar", "ScrollBar", + bounds=Bounds(864, 82, 18, 532)) + root.children.append(vscroll) + + # Progress bar — exercises value_now/min/max for the role glyph path. + root.children.append(UIElement( + "root.5", "Saving", "ProgressBar", + bounds=Bounds(560, 600, 200, 10), + value_now=40.0, value_min=0.0, value_max=100.0, + )) + + # Word-wrap toggle checkbox — exercises selected=True path. + root.children.append(UIElement( + "root.6", "Word Wrap", "CheckBox", + bounds=Bounds(80, 636, 120, 18), selected=True, + )) + + # Status bar + sb = UIElement("root.4", "Status Bar", "StatusBar", + bounds=Bounds(80, 614, 800, 22)) + for i, (lbl, val) in enumerate([ + ("Position", "Ln 1, Col 1"), + ("Zoom", "100%"), + ("Encoding", "UTF-8"), + ("EOL", "Windows (CRLF)"), + ]): + sb.children.append(UIElement( + f"root.4.{i}", lbl, "Text", value=val, + bounds=Bounds(80 + i * 190, 614, 188, 22) + )) + root.children.append(sb) + + return root + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + from PIL import Image, ImageDraw + img = Image.new("RGB", (800, 600), "#1a1e2e") + draw = ImageDraw.Draw(img) + + # Title bar + draw.rectangle([0, 0, 800, 30], fill="#2d3250") + draw.text((10, 8), "Untitled — Notepad", fill="#c8d3f5") + + # Menu bar + draw.rectangle([0, 30, 800, 52], fill="#1e2030") + for i, m in enumerate(["File", "Edit", "Format", "View", "Help"]): + draw.text((10 + i * 58, 36), m, fill="#a9b1d6") + + # Editor area + draw.rectangle([0, 52, 782, 570], fill="#1a1e2e") + for i, ln in enumerate(["Hello, world!", "This is a test document.", + "Line 3 has some content here.", ""]): + draw.text((6, 58 + i * 18), ln, fill="#c0caf5") + + # Scrollbars + draw.rectangle([782, 52, 800, 570], fill="#24283b") + draw.rectangle([0, 570, 782, 586], fill="#24283b") + + # Status bar + draw.rectangle([0, 586, 800, 600], fill="#16161e") + draw.text((6, 589), "Ln 1, Col 1 100% UTF-8 Windows (CRLF)", + fill="#565f89") + + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + print(f"[MockAdapter:get_screenshot] {e}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + if self.scenario is not None: + handled = self.scenario.handle_action(action=action, + element_id=element_id, + value=value, hwnd=hwnd) + if handled is not None: + return handled + return { + "success": True, + "action": action, + "note": "Mock adapter — no real OS action performed", + } diff --git a/observer/adapters/windows.py b/observer/adapters/windows.py new file mode 100644 index 0000000..c1f866a --- /dev/null +++ b/observer/adapters/windows.py @@ -0,0 +1,640 @@ +""" +Windows UIA adapter (pywinauto, pywin32, raw COM walker). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import ( + Bounds, UIElement, WindowInfo, find_element_by_path, prune_tree_depth, +) + +logger = logging.getLogger(__name__) + + +# UIA control-type ID → pywinauto-compatible role name. +# Values from UIAutomationClient.h; must match what selectors/descriptions expect. +_UIA_TYPE_TO_ROLE: Dict[int, str] = { + 50000: "Button", 50001: "Calendar", 50002: "CheckBox", + 50003: "ComboBox", 50004: "Edit", 50005: "Hyperlink", + 50006: "Image", 50007: "ListItem", 50008: "ListBox", + 50009: "Menu", 50010: "MenuBar", 50011: "MenuItem", + 50012: "ProgressBar", 50013: "RadioButton", 50014: "ScrollBar", + 50015: "Slider", 50016: "Spinner", 50017: "StatusBar", + 50018: "TabControl", 50019: "TabItem", 50020: "Text", + 50021: "Toolbar", 50022: "ToolTip", 50023: "Tree", + 50024: "TreeItem", 50025: "Custom", 50026: "GroupBox", + 50027: "Thumb", 50028: "DataGrid", 50029: "DataItem", + 50030: "Document", 50031: "SplitButton", 50032: "Dialog", + 50033: "Pane", 50034: "Header", 50035: "HeaderItem", + 50036: "Table", 50037: "TitleBar", 50038: "Separator", + 50039: "SemanticZoom",50040: "AppBar", +} + +# UIA property IDs (UIAutomationClient.h) used by the raw-COM walker. +_UIA_BOUNDING_RECT = 30001 +_UIA_NAME = 30005 +_UIA_CTRL_TYPE = 30003 +_UIA_ENABLED = 30010 +_UIA_FOCUSED = 30008 +_UIA_VALUE = 30045 +_UIA_ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" +_UIA_ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" +_UIA_HELP_TEXT = 30013 # tooltip / description +_UIA_AUTOMATION_ID = 30011 +_UIA_RANGE_VALUE = 30047 +_UIA_RANGE_MIN = 30049 +_UIA_RANGE_MAX = 30050 +_UIA_IS_SELECTED = 30079 +_UIA_EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf +_UIA_SCOPE_CHILDREN = 0x2 + +# Properties bulk-fetched via a UIA CacheRequest so each level of the walk is +# one COM round trip (FindAllBuildCache) instead of ~15 per node. +_UIA_CACHED_PROPS = ( + _UIA_BOUNDING_RECT, _UIA_NAME, _UIA_CTRL_TYPE, _UIA_ENABLED, + _UIA_FOCUSED, _UIA_VALUE, _UIA_ACCESS_KEY, _UIA_ACCEL_KEY, + _UIA_HELP_TEXT, _UIA_AUTOMATION_ID, _UIA_RANGE_VALUE, _UIA_RANGE_MIN, + _UIA_RANGE_MAX, _UIA_IS_SELECTED, _UIA_EXPAND_STATE, +) + + +class WindowsAdapter: + """Full Windows UIA adapter using pywinauto + pywin32.""" + + def __init__(self, config: dict): + self.config = config + try: + import win32gui # noqa: F401 + import win32process # noqa: F401 + import psutil # noqa: F401 + from pywinauto import Application # noqa: F401 + self._Application = Application + logger.info("[WindowsAdapter:__init__] pywinauto/pywin32 ready") + except ImportError as e: + print(f"[WindowsAdapter:__init__] Missing dependency: {e}") + traceback.print_exc() + raise + + def list_windows(self) -> List[WindowInfo]: + try: + import win32gui + import win32process + import psutil + + results: List[WindowInfo] = [] + fg = win32gui.GetForegroundWindow() + + def _cb(hwnd, _): + try: + if not win32gui.IsWindowVisible(hwnd): + return + title = win32gui.GetWindowText(hwnd) + if not title: + return + rect = win32gui.GetWindowRect(hwnd) + w, h = rect[2] - rect[0], rect[3] - rect[1] + if w <= 0 or h <= 0: + return + try: + _, pid = win32process.GetWindowThreadProcessId(hwnd) + proc_name = psutil.Process(pid).name() + except Exception: + pid, proc_name = 0, "unknown" + results.append(WindowInfo( + handle=hwnd, title=title, process_name=proc_name, pid=pid, + bounds=Bounds(rect[0], rect[1], w, h), + is_focused=(hwnd == fg), + window_uid=f"win:{pid}:{hwnd}", + )) + except Exception as inner: + logger.debug(f"[WindowsAdapter:list_windows:_cb] {inner}") + + win32gui.EnumWindows(_cb, None) + return sorted(results, key=lambda w: (not w.is_focused, w.title.lower())) + except Exception as e: + print(f"[WindowsAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + """Return bounds of visible windows that are above hwnd in Z-order.""" + try: + import win32gui + GW_HWNDNEXT = 2 + above: List[Bounds] = [] + h = win32gui.GetTopWindow(None) + while h and h != hwnd: + try: + if win32gui.IsWindowVisible(h): + rect = win32gui.GetWindowRect(h) + w = rect[2] - rect[0] + hh = rect[3] - rect[1] + if w > 0 and hh > 0: + above.append(Bounds(rect[0], rect[1], w, hh)) + except Exception as e: + logger.debug(f"[occlusion] window rect probe failed " + f"for hwnd {h}: {e}") + try: + h = win32gui.GetWindow(h, GW_HWNDNEXT) + except Exception: + break + return above + except Exception as e: + logger.debug(f"[WindowsAdapter:get_windows_above_bounds] {e}") + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + try: + import win32gui + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + except Exception as e: + print(f"[WindowsAdapter:get_element_tree] win32gui: {e}") + return None + + # Run both walkers and synthesise: UIA crosses Chromium fragment boundaries + # (gets web content); pywinauto may surface native-control properties that + # UIA omits. The merged result gives the LLM everything either source sees. + # tree.strategy == "uia_only" skips the second pywinauto walk and the + # synthesis pass entirely — roughly halving capture time — at the cost + # of the extra native-control properties the merge would contribute. + strategy = str(self.config.get("tree", {}).get("strategy", + "merged")).lower() + uia_tree = self._uia_walk(hwnd) + if strategy == "uia_only" and uia_tree is not None: + return uia_tree + pw_tree = None + try: + app = self._Application(backend="uia").connect(handle=hwnd) + window = app.window(handle=hwnd) + wrapper = window.wrapper_object() + max_depth = self.config.get("tree", {}).get("max_depth", 8) + pw_tree = self._walk(wrapper, "root", 0, max_depth) + except Exception as e: + logger.debug(f"[WindowsAdapter:get_element_tree] pywinauto: {e}") + + if uia_tree is None and pw_tree is None: + return None + if uia_tree is None: + return pw_tree + if pw_tree is None: + return uia_tree + return self._synthesize_trees(uia_tree, pw_tree) + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path* via raw UIA. + + Navigates the positional child indices of the element-id path + ('root.3.2' → child 3 → child 2) so only the requested branch is + traversed. Falls back to a full walk plus extraction when the path + contains non-positional segments (synthesized ids) or navigation + fails.""" + try: + import win32gui + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + except Exception as e: + print(f"[WindowsAdapter:get_element_subtree] win32gui: {e}") + return None + + if max_depth is None: + max_depth = self.config.get("tree", {}).get("max_depth", 8) + + indices = self._parse_positional_path(element_path) + if indices is not None: + try: + raw_uia, true_cond = self._uia_handles() + elem = raw_uia.ElementFromHandle(hwnd) + for idx in indices: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + if idx < 0 or idx >= kids.Length: + elem = None + break + elem = kids.GetElement(idx) + if elem is not None: + return self._uia_walk_element( + elem, element_path, 0, max_depth, true_cond, + cache_request=self._uia_cache_request(raw_uia)) + except Exception as e: + logger.debug(f"[WindowsAdapter:get_element_subtree] " + f"navigation failed ({e}); falling back") + + # Fallback: full walk + extraction. + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + @staticmethod + def _parse_positional_path(element_path: str) -> Optional[List[int]]: + """'root.3.2' → [3, 2]; None when the path is not purely positional.""" + segs = (element_path or "").split(".") + if not segs or segs[0] != "root": + return None + try: + return [int(s) for s in segs[1:]] + except ValueError: + return None # synthesized ids like 'root.2.x1' + + def _uia_handles(self): + """Return (raw IUIAutomation interface, true condition). + + pywinauto already initialised comtypes/UIA at import time; retrieve + the raw COM interface from its singleton (pywinauto ≥0.6 exposes it + as .iuia; older versions expose it directly).""" + from pywinauto.uia_defines import IUIA + _iuia_obj = IUIA() + raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) + return raw_uia, raw_uia.CreateTrueCondition() + + def _uia_walk(self, hwnd: int) -> Optional[UIElement]: + """Walk the accessibility tree via raw IUIAutomation COM calls. + + pywinauto's children() falls back to EnumChildWindows for HWND-backed + elements (e.g. Chrome_RenderWidgetHostHWND), missing all web content. + Using FindAll(TreeScope_Children) directly on the IUIAutomationElement + always uses UIA and correctly crosses Chromium fragment boundaries. + + When the COM API supports it, a CacheRequest bulk-fetches the walked + properties per level (FindAllBuildCache + GetCachedPropertyValue) — + one cross-process round trip per node's children instead of ~15 per + node. Any CacheRequest failure falls back to the per-property path. + """ + try: + raw_uia, true_cond = self._uia_handles() + root = raw_uia.ElementFromHandle(hwnd) + max_depth = self.config.get("tree", {}).get("max_depth", 8) + cache_request = self._uia_cache_request(raw_uia) + return self._uia_walk_element(root, "root", 0, max_depth, + true_cond, + cache_request=cache_request) + except Exception as e: + logger.debug(f"[WindowsAdapter:_uia_walk] {e}") + return None + + @staticmethod + def _uia_cache_request(raw_uia): + """Build a CacheRequest covering the properties the walker reads. + Returns None (per-property fallback) when construction fails.""" + try: + cr = raw_uia.CreateCacheRequest() + for pid in _UIA_CACHED_PROPS: + cr.AddProperty(pid) + return cr + except Exception as e: + logger.debug(f"[WindowsAdapter:_uia_cache_request] CacheRequest " + f"unavailable ({e}); using per-property fetches") + return None + + @staticmethod + def _uia_prop(elem, pid, default=None, cached=False): + if cached: + try: + v = elem.GetCachedPropertyValue(pid) + return v if v is not None else default + except Exception: + pass # cache miss/failure — fall back to a live fetch + try: + v = elem.GetCurrentPropertyValue(pid) + return v if v is not None else default + except Exception: + return default + + @staticmethod + def _uia_bounds(elem, cached=False) -> Bounds: + if cached: + try: + r = elem.CachedBoundingRectangle + return Bounds(r.left, r.top, + r.right - r.left, r.bottom - r.top) + except Exception: + try: + arr = elem.GetCachedPropertyValue(_UIA_BOUNDING_RECT) + if arr is not None and len(arr) == 4: + # VT_R8 SAFEARRAY: [left, top, width, height] + return Bounds(int(arr[0]), int(arr[1]), + int(arr[2]), int(arr[3])) + except Exception: + pass + try: + r = elem.CurrentBoundingRectangle + return Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) + except Exception: + return Bounds(0, 0, 0, 0) + + def _uia_walk_element(self, elem, elem_id: str, depth: int, + max_depth: int, true_cond, + cache_request=None, cached: bool = False + ) -> UIElement: + """Build a UIElement for *elem* and recurse into its children. + + *cached* means this element was fetched via FindAllBuildCache and its + properties can be read with GetCachedPropertyValue (no round trip). + """ + def _prop(pid, default=None): + return self._uia_prop(elem, pid, default, cached=cached) + + name = _prop(_UIA_NAME, "") or "" + ctrl = _prop(_UIA_CTRL_TYPE, 0) or 0 + role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") + bounds = self._uia_bounds(elem, cached=cached) + enabled = bool(_prop(_UIA_ENABLED, True)) + focused = bool(_prop(_UIA_FOCUSED, False)) + value = _prop(_UIA_VALUE) or None + # Keyboard shortcut: prefer access key, fall back to accelerator. + ks = _prop(_UIA_ACCESS_KEY) or _prop(_UIA_ACCEL_KEY) or None + desc = _prop(_UIA_HELP_TEXT) or None + aid = _prop(_UIA_AUTOMATION_ID) or None + # RangeValue pattern: slider / progress / scrollbar + vn = _prop(_UIA_RANGE_VALUE) + vmin = _prop(_UIA_RANGE_MIN) + vmax = _prop(_UIA_RANGE_MAX) + # SelectionItem.IsSelected: checkbox / radio / tab / menuitem + sel_raw = _prop(_UIA_IS_SELECTED) + sel = bool(sel_raw) if sel_raw is not None else None + # ExpandCollapse: combobox / treeitem / menuitem + exp_raw = _prop(_UIA_EXPAND_STATE) + if exp_raw in (0, 1): + exp = bool(exp_raw) + else: + exp = None + try: + vn_f = float(vn) if vn is not None else None + except Exception: + vn_f = None + try: + vmin_f = float(vmin) if vmin is not None else None + except Exception: + vmin_f = None + try: + vmax_f = float(vmax) if vmax is not None else None + except Exception: + vmax_f = None + node = UIElement( + element_id=elem_id, name=name, role=role, value=value, + bounds=bounds, enabled=enabled, focused=focused, + keyboard_shortcut=ks or None, + description=desc or None, + selected=sel, expanded=exp, + value_now=vn_f, value_min=vmin_f, value_max=vmax_f, + identifier=str(aid) if aid else None, + ) + if depth < max_depth: + kids = None + kids_cached = False + if cache_request is not None: + try: + kids = elem.FindAllBuildCache(_UIA_SCOPE_CHILDREN, + true_cond, cache_request) + kids_cached = True + except Exception: + kids = None # per-node fallback below + if kids is None: + try: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + except Exception: + kids = None + if kids is not None: + try: + for i in range(kids.Length): + node.children.append(self._uia_walk_element( + kids.GetElement(i), f"{elem_id}.{i}", + depth + 1, max_depth, true_cond, + cache_request=cache_request, + cached=kids_cached)) + except Exception as e: + logger.debug(f"[uia] child walk truncated at " + f"{elem_id}: {e}") + return node + + def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: + try: + try: + rect = wrapper.rectangle() + bounds = Bounds(rect.left, rect.top, + rect.right - rect.left, rect.bottom - rect.top) + except Exception: + bounds = Bounds(0, 0, 0, 0) + + try: + name = wrapper.window_text() or "" + except Exception: + name = "" + + try: + role = wrapper.friendly_class_name() or "Unknown" + except Exception: + role = "Unknown" + + try: + value = wrapper.get_value() if hasattr(wrapper, "get_value") else None + except Exception: + value = None + + try: + enabled = wrapper.is_enabled() + except Exception: + enabled = True + + try: + focused = wrapper.has_keyboard_focus() + except Exception: + focused = False + + elem = UIElement( + element_id=elem_id, name=name, role=role, value=value, + bounds=bounds, enabled=enabled, focused=focused, + ) + + if depth < max_depth: + try: + for i, child in enumerate(wrapper.children()): + elem.children.append( + self._walk(child, f"{elem_id}.{i}", depth + 1, max_depth) + ) + except Exception as ce: + logger.debug(f"[WindowsAdapter:_walk:{elem_id}:children] {ce}") + + return elem + except Exception as e: + print(f"[WindowsAdapter:_walk:{elem_id}] {e}") + traceback.print_exc() + return UIElement(elem_id, "[error]", "Unknown", bounds=Bounds(0, 0, 0, 0)) + + def _synthesize_trees(self, primary: UIElement, secondary: UIElement) -> UIElement: + """Merge two accessibility trees into one richer tree. + + Uses the primary (UIA) tree as the base — it sees web content. + For each node in secondary (pywinauto) matched by bounds, enrich the + primary node with any non-empty properties the primary is missing. + Secondary nodes whose bounds don't appear anywhere in the primary tree + are injected under the deepest primary ancestor that contains them. + """ + # Build a flat index of primary nodes keyed by (x, y, w, h). + bounds_index: Dict[tuple, UIElement] = {} + + def _index(node: UIElement) -> None: + key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) + if key not in bounds_index: + bounds_index[key] = node + for c in node.children: + _index(c) + + _index(primary) + + # Enrich matched nodes; collect unmatched ones with their bounds. + unmatched: List[UIElement] = [] + + def _enrich(node: UIElement) -> None: + key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) + target = bounds_index.get(key) + if target is not None: + # Copy over properties the primary left empty. + if not target.keyboard_shortcut and node.keyboard_shortcut: + target.keyboard_shortcut = node.keyboard_shortcut + if not target.description and node.description: + target.description = node.description + if not target.value and node.value: + target.value = node.value + else: + # Not in primary — keep for injection. + w, h = node.bounds.width, node.bounds.height + if w > 0 and h > 0: # ignore zero-size ghost elements + unmatched.append(node) + for c in node.children: + _enrich(c) + + _enrich(secondary) + + # Inject unmatched secondary nodes under the deepest primary ancestor + # whose bounds contain them (largest-area match wins → most specific). + def _contains(outer: Bounds, inner: Bounds) -> bool: + return (outer.x <= inner.x and outer.y <= inner.y and + outer.x + outer.width >= inner.x + inner.width and + outer.y + outer.height >= inner.y + inner.height) + + for node in unmatched: + best: Optional[UIElement] = None + best_area = float("inf") + for pnode in bounds_index.values(): + if _contains(pnode.bounds, node.bounds): + area = pnode.bounds.width * pnode.bounds.height + if area < best_area: + best_area = area + best = pnode + if best is None: + best = primary + # Renumber the injected element_id to avoid collisions. + node.element_id = f"{best.element_id}.x{len(best.children)}" + best.children.append(node) + + return primary + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + import win32gui + import win32ui + from PIL import Image + import ctypes + + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + + rect = win32gui.GetWindowRect(hwnd) + width = rect[2] - rect[0] + height = rect[3] - rect[1] + + if width <= 0 or height <= 0: + return None + + hwnd_dc = win32gui.GetWindowDC(hwnd) + mfc_dc = win32ui.CreateDCFromHandle(hwnd_dc) + save_dc = mfc_dc.CreateCompatibleDC() + save_bmp = win32ui.CreateBitmap() + save_bmp.CreateCompatibleBitmap(mfc_dc, width, height) + save_dc.SelectObject(save_bmp) + + # PW_RENDERFULLCONTENT (0x2) renders hardware-accelerated content too + ok = ctypes.windll.user32.PrintWindow( # type: ignore[attr-defined] + hwnd, save_dc.GetSafeHdc(), 2) + + bmpinfo = save_bmp.GetInfo() + bmpstr = save_bmp.GetBitmapBits(True) + + win32gui.DeleteObject(save_bmp.GetHandle()) + save_dc.DeleteDC() + mfc_dc.DeleteDC() + win32gui.ReleaseDC(hwnd, hwnd_dc) + + if ok: + img = Image.frombuffer( + "RGB", + (bmpinfo["bmWidth"], bmpinfo["bmHeight"]), + bmpstr, "raw", "BGRX", 0, 1, + ) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + # PrintWindow failed — fall back to screen-region capture + logger.warning("[WindowsAdapter:get_screenshot] PrintWindow failed; falling back to mss") + raise RuntimeError("PrintWindow returned 0") + + except Exception as e: + logger.debug(f"[WindowsAdapter:get_screenshot] PrintWindow path failed ({e}); trying mss") + try: + import mss + from PIL import Image + import win32gui + + with mss.MSS() as sct: + if hwnd: + rect = win32gui.GetWindowRect(hwnd) + region = {"left": rect[0], "top": rect[1], + "width": rect[2] - rect[0], "height": rect[3] - rect[1]} + else: + region = sct.monitors[1] + raw = sct.grab(region) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e2: + print(f"[WindowsAdapter:get_screenshot] {e2}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True, "action": "type", "text": value} + + elif action == "key" and value: + keys = str(value).lower().split("+") + pyautogui.hotkey(*keys) + return {"success": True, "action": "key", "keys": value} + + elif action == "click_at" and isinstance(value, dict): + pyautogui.click(value["x"], value["y"]) + return {"success": True, "action": "click_at", **value} + + elif action == "scroll" and isinstance(value, dict): + pyautogui.scroll(value.get("clicks", 3), x=value.get("x"), y=value.get("y")) + return {"success": True, "action": "scroll"} + + else: + return {"success": False, "error": f"Unsupported action: {action}"} + except Exception as e: + print(f"[WindowsAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/wsl.py b/observer/adapters/wsl.py new file mode 100644 index 0000000..c4141ed --- /dev/null +++ b/observer/adapters/wsl.py @@ -0,0 +1,119 @@ +""" +WSL adapter (WSL 1 + WSL 2: X11 when DISPLAY is set, +PowerShell interop fallback). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import logging +import os +from typing import List, Optional + +from observer.adapters.linux import LinuxAdapter +from observer.models import Bounds, WindowInfo + +logger = logging.getLogger(__name__) + + +class WSLAdapter(LinuxAdapter): + """Adapter for Windows Subsystem for Linux. + + Prefers X11-based tools (wmctrl, mss) when DISPLAY is set. Falls back to + PowerShell / cmd.exe interop, which is always available in both WSL 1 and + WSL 2 via the Windows binary execution layer. + """ + + def __init__(self, config: dict) -> None: + self.config = config + self._has_display = bool(os.environ.get("DISPLAY")) + logger.info( + "[WSLAdapter:__init__] WSL detected; " + "DISPLAY=%s", "set" if self._has_display else "not set (PowerShell fallback active)", + ) + + # ── Window listing ──────────────────────────────────────────────────────── + + def list_windows(self) -> List[WindowInfo]: + if self._has_display: + result = LinuxAdapter.list_windows(self) + if result: + return result + return self._list_windows_ps() + + def _list_windows_ps(self) -> List[WindowInfo]: + """Enumerate visible Windows windows via PowerShell ConvertTo-Json.""" + try: + import json + import subprocess + ps = ( + "Get-Process " + "| Where-Object { $_.MainWindowTitle -ne '' } " + "| Select-Object Id,ProcessName,MainWindowTitle " + "| ConvertTo-Json -Compress" + ) + r = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], + capture_output=True, text=True, timeout=10, + ) + if r.returncode != 0 or not r.stdout.strip(): + return [] + data = json.loads(r.stdout) + if isinstance(data, dict): + data = [data] + results: List[WindowInfo] = [] + for i, item in enumerate(data or []): + pid = int(item.get("Id", 0)) + name = str(item.get("ProcessName", "unknown")) + title = str(item.get("MainWindowTitle", "")) + if not title: + continue + results.append(WindowInfo( + handle=pid, title=title, process_name=name, pid=pid, + bounds=Bounds(0, 0, 1920, 1080), is_focused=(i == 0), + window_uid=f"wsl:{pid}", + )) + return results + except Exception as e: + logger.debug("[WSLAdapter:_list_windows_ps] %s", e) + return [] + + # ── Screenshot ──────────────────────────────────────────────────────────── + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + if self._has_display: + result = LinuxAdapter.get_screenshot(self, hwnd) + if result: + return result + return self._screenshot_ps() + + def _screenshot_ps(self) -> Optional[bytes]: + """Capture the primary screen via PowerShell, returning PNG bytes.""" + try: + import base64 + import subprocess + # Capture screen to a MemoryStream and emit as base64 — avoids + # WSL↔Windows path translation issues entirely. + ps = ( + "Add-Type -AssemblyName System.Windows.Forms,System.Drawing;" + "$b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds;" + "$bmp=New-Object System.Drawing.Bitmap $b.Width,$b.Height;" + "$g=[System.Drawing.Graphics]::FromImage($bmp);" + "$g.CopyFromScreen($b.Location,[System.Drawing.Point]::Empty,$b.Size);" + "$ms=New-Object System.IO.MemoryStream;" + "$bmp.Save($ms,[System.Drawing.Imaging.ImageFormat]::Png);" + "$g.Dispose();$bmp.Dispose();" + "[Convert]::ToBase64String($ms.ToArray())" + ) + r = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], + capture_output=True, text=True, timeout=20, + ) + if r.returncode == 0 and r.stdout.strip(): + return base64.b64decode(r.stdout.strip()) + except Exception as e: + logger.debug("[WSLAdapter:_screenshot_ps] %s", e) + return None + + # ── get_windows_above_bounds: returns [] (inherited from LinuxAdapter) ──── + # ── get_element_tree: upgraded by linux_adapter.install_into if pyatspi ── + # ── perform_action: inherited (pyautogui; needs DISPLAY) ───────────────── diff --git a/observer/core.py b/observer/core.py new file mode 100644 index 0000000..823c63d --- /dev/null +++ b/observer/core.py @@ -0,0 +1,367 @@ +""" +ScreenObserver — public interface; selects a platform adapter +and wires the session tree cache. + +Split out of observer.py (P3); behavior is unchanged. +""" + +import logging +import io +import time +import traceback +from typing import Any, Dict, List, Optional, Tuple + +from observer.activation import ActivationMixin +from observer.adapters.linux import LinuxAdapter +from observer.adapters.macos import MacOSAdapter +from observer.adapters.mock import MockAdapter +from observer.adapters.windows import WindowsAdapter +from observer.adapters.wsl import WSLAdapter +from observer.models import ( + UIElement, WindowInfo, WindowResolution, find_element_by_path, + prune_tree_depth, +) +from observer.occlusion import OcclusionMixin +from observer.platform_info import EFFECTIVE_PLATFORM + +logger = logging.getLogger(__name__) + + +class ScreenObserver(ActivationMixin, OcclusionMixin): + """ + Platform-aware screen observer. All consumers should program against + this class rather than the platform adapters directly. + """ + + def __init__(self, config: dict): + self.config = config + self._adapter = self._select_adapter() + # Try to upgrade stub adapters to real AX implementations. + try: + if isinstance(self._adapter, MacOSAdapter): + import mac_adapter + if mac_adapter.install_into(self): + logger.info("[ScreenObserver] mac_adapter installed (pyobjc)") + elif isinstance(self._adapter, LinuxAdapter): + import linux_adapter + if linux_adapter.install_into(self): + logger.info("[ScreenObserver] linux_adapter installed (pyatspi)") + except Exception: + logger.exception("real adapter upgrade failed") + + def _select_adapter(self): + if self.config.get("mock", False): + logger.info("[ScreenObserver] Using MockAdapter") + return MockAdapter() + + target = self.config.get("platform", "auto") + # EFFECTIVE_PLATFORM is "WSL" when running inside WSL, otherwise same + # as platform.system(). Explicit config overrides auto-detection. + sys_plat = EFFECTIVE_PLATFORM if target == "auto" else target + + adapters = { + "Windows": WindowsAdapter, + "Darwin": MacOSAdapter, + "Linux": LinuxAdapter, + "WSL": WSLAdapter, + } + + cls = adapters.get(sys_plat) + if cls is None: + logger.warning("[ScreenObserver] Unknown platform '%s'; using MockAdapter", sys_plat) + return MockAdapter() + + try: + return cls(self.config) + except Exception as e: + print(f"[ScreenObserver:_select_adapter] Platform adapter failed: {e}; falling back to Mock") + traceback.print_exc() + return MockAdapter() + + @property + def is_mock(self) -> bool: + return isinstance(self._adapter, MockAdapter) + + def list_windows(self) -> List[WindowInfo]: + return self._adapter.list_windows() + + def get_element_tree(self, hwnd=None, window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return the accessibility tree for a window. + + When *window_uid* is supplied the per-window tree cache is consulted: + a fresh cached capture (within ``tree.cache_ttl_s``) is returned + without walking the adapter; a miss walks and stores. Pass + ``use_cache=False`` to force a fresh walk (post-action re-reads); + the fresh capture still refreshes the cache. Calls without a + *window_uid* always walk and are never cached. + """ + tree, _meta = self.get_element_tree_with_meta( + hwnd, window_uid=window_uid, use_cache=use_cache) + return tree + + def get_element_tree_with_meta( + self, hwnd=None, *, window_uid: Optional[str] = None, + use_cache: bool = True, + ) -> Tuple[Optional[UIElement], Dict[str, Any]]: + """get_element_tree plus capture metadata. + + Returns (tree, meta) where meta is + ``{"cache": "hit"|"miss"|"bypass", "capture_ms": int, + "node_count": int}``. + """ + tree_cfg = self.config.get("tree", {}) or {} + ttl = float(tree_cfg.get("cache_ttl_s", 2.0)) + cache = self._tree_cache() + + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, ttl_s=ttl) + if entry is not None: + return entry.tree, { + "cache": "hit", + "capture_ms": 0, + "node_count": entry.node_count, + } + + started = time.time() + tree = self._adapter.get_element_tree(hwnd) + capture_ms = int((time.time() - started) * 1000) + node_count = len(tree.flat_list()) if tree is not None else 0 + + if tree is not None and window_uid and cache is not None: + from hashing import tree_hash as _tree_hash + named = sum( + 1 for e in tree.flat_list()[1:] if (e.name or "").strip() + ) + cache.put( + window_uid, + tree=tree, + serialized=tree.to_dict(), + tree_hash=_tree_hash(tree), + max_depth=int(tree_cfg.get("max_depth", 8)), + capture_ms=capture_ms, + node_count=node_count, + named_node_count=named, + ) + + return tree, { + "cache": "bypass" if not use_cache else "miss", + "capture_ms": capture_ms, + "node_count": node_count, + } + + @staticmethod + def _tree_cache(): + """The session-scoped TreeCache (lazy import avoids cycles).""" + try: + from session import get_session + return get_session().tree_cache + except Exception: + return None + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None, + window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return only the subtree rooted at *element_path* ('root.3.2'). + + Resolution order: + 1. a fresh cached full capture (no walk at all), + 2. the adapter's native get_element_subtree (walks just the branch), + 3. full walk + extraction. + The result is depth-limited to *max_depth* levels below the subtree + root and safe to mutate (cache extraction returns copies).""" + tree_cfg = self.config.get("tree", {}) or {} + if max_depth is None: + max_depth = int(tree_cfg.get("max_depth", 8)) + + # 1. Serve from a fresh cached capture. + cache = self._tree_cache() + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, + ttl_s=float(tree_cfg.get("cache_ttl_s", 2.0))) + if entry is not None: + sub = find_element_by_path(entry.tree, element_path) + if sub is not None: + return prune_tree_depth(sub, max_depth) + + # 2. Adapter-native scoped walk. + native = getattr(self._adapter, "get_element_subtree", None) + if native is not None: + try: + sub = native(hwnd, element_path, max_depth) + if sub is not None: + return sub + except Exception: + logger.exception("[ScreenObserver:get_element_subtree] " + "adapter subtree walk failed; falling back") + + # 3. Full walk + extraction. + tree = self.get_element_tree(hwnd, window_uid=window_uid, + use_cache=use_cache) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + return self._adapter.get_screenshot(hwnd) + + def get_full_display_screenshot(self) -> Optional[bytes]: + """Capture the entire virtual desktop (all monitors combined) as a PNG.""" + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[0]) # 0 = union of all monitors + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + logger.warning(f"[ScreenObserver:get_full_display_screenshot] {e}; falling back") + return self._adapter.get_screenshot() + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + return self._adapter.perform_action(action, element_id, value, hwnd) + + def window_by_index(self, windows: List[WindowInfo], + index: Optional[int]) -> Optional[WindowInfo]: + """Convenience: return a WindowInfo by list index, or None.""" + if index is None or not windows: + return None + if 0 <= index < len(windows): + return windows[index] + return None + + def window_by_uid(self, windows: List[WindowInfo], + uid: Optional[str]) -> Optional[WindowInfo]: + """Resolve a window by stable uid; returns None if not found.""" + if not uid or not windows: + return None + for w in windows: + if w.window_uid == uid: + return w + return None + + def resolve_window(self, windows: List[WindowInfo], + window_uid: Optional[str], + window_index: Optional[int], + window_title: Optional[str] = None) -> "WindowResolution": + """Resolve a window by uid (preferred), index, or title substring.""" + if window_uid: + info = self.window_by_uid(windows, window_uid) + warning = ("both window_index and window_uid given; window_uid used" + if window_index is not None else None) + return WindowResolution(info=info, warning=warning, + used_uid=True, requested_uid=window_uid) + if window_index is not None: + info = self.window_by_index(windows, window_index) + resolved_uid = info.window_uid if info else None + return WindowResolution(info=info, warning=None, + used_uid=bool(resolved_uid), + requested_uid=resolved_uid) + if window_title: + needle = window_title.lower() + info = next((w for w in windows if needle in (w.title or "").lower()), None) + resolved_uid = info.window_uid if info else None + return WindowResolution(info=info, warning=None, + used_uid=bool(resolved_uid), + requested_uid=resolved_uid) + return WindowResolution(info=None, warning=None, + used_uid=False, requested_uid=None) + + # ── Monitors / DPI (design doc §6.3) ────────────────────────────────────── + + def get_monitors(self) -> List[Dict[str, Any]]: + """Return per-monitor metadata via mss.""" + try: + import mss + with mss.MSS() as sct: + mons = sct.monitors # [0] is union; [1..] are individual + out: List[Dict[str, Any]] = [] + for i, m in enumerate(mons[1:]): + out.append({ + "index": i, + "primary": (i == 0), + "bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + "scale_factor": 1.0, + "logical_bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + "physical_bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + }) + return out + except Exception: + return [] + + # ── Capability discovery (design doc §6.4) ──────────────────────────────── + + def get_capabilities(self) -> Dict[str, Any]: + adapter_name = type(self._adapter).__name__ + is_windows = adapter_name == "WindowsAdapter" + is_macos = adapter_name == "MacOSAdapter" + is_wsl = adapter_name == "WSLAdapter" + is_linux = adapter_name in ("LinuxAdapter", "WSLAdapter") + is_mock = adapter_name == "MockAdapter" + + # Probe optional libs. + def _has(mod: str) -> bool: + try: + __import__(mod) + return True + except Exception: + return False + + if is_macos: + ax_tree = _has("AppKit") or _has("ApplicationServices") or _has("Cocoa") + elif is_linux: + ax_tree = _has("pyatspi") + else: + ax_tree = is_windows or is_mock + + return { + "ok": True, + "platform": EFFECTIVE_PLATFORM, + "adapter": adapter_name, + "version": (self.config.get("mcp", {}) or {}).get("version", "0.2.0"), + "protocol_version": 2, + "supports": { + "accessibility_tree": bool(ax_tree), + "uia_invoke": is_windows, + "occlusion_detection": is_windows or is_mock or _has("Quartz") or _has("Xlib"), + "drag": True, + "ocr": _has("pytesseract"), + "vlm": bool((self.config.get("vlm") or {}).get("enabled") + and (self.config.get("vlm") or {}).get("model")), + "redaction": True, + "scenarios": is_mock, + "tracing": True, + "replay": True, + "image_blur": _has("PIL"), + "wsl_powershell": is_wsl, + # Action capabilities always present via REST + MCP. + "bring_to_foreground": True, + "element_targeting": bool(ax_tree), # click/focus/invoke/set_value via element_id + "observe_with_diff": True, # /api/observe returns diff token + }, + "config": { + "tree_max_depth": (self.config.get("tree", {}) or {}).get("max_depth", 8), + "tree_default_depth": (self.config.get("tree", {}) or {}).get("default_depth", 5), + "ascii_grid": { + "width": (self.config.get("ascii_sketch", {}) or {}).get("grid_width", 110), + "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), + }, + }, + # Per-window last-capture statistics (node_count, + # named_node_count, capture_ms, captured_at) so agents can spot + # accessibility-dark windows without another walk. + "tree_stats": self._last_capture_stats(), + } + + def _last_capture_stats(self) -> Dict[str, Dict[str, Any]]: + cache = self._tree_cache() + try: + return cache.stats() if cache is not None else {} + except Exception: + return {} diff --git a/observer/models.py b/observer/models.py new file mode 100644 index 0000000..4bfc39d --- /dev/null +++ b/observer/models.py @@ -0,0 +1,197 @@ +""" +Data model: Bounds, UIElement, WindowInfo, WindowResolution +plus subtree helpers (find_element_by_path, prune_tree_depth). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from dataclasses import dataclass, field, replace as _dc_replace +from typing import Any, Dict, List, Optional + + +@dataclass +class Bounds: + x: int + y: int + width: int + height: int + + @property + def right(self) -> int: + return self.x + self.width + + @property + def bottom(self) -> int: + return self.y + self.height + + @property + def center_x(self) -> int: + return self.x + self.width // 2 + + @property + def center_y(self) -> int: + return self.y + self.height // 2 + + def to_dict(self) -> Dict: + return {"x": self.x, "y": self.y, "width": self.width, "height": self.height} + + def __bool__(self) -> bool: + return self.width > 0 and self.height > 0 + + +@dataclass +class UIElement: + element_id: str + name: str + role: str + value: Optional[str] = None + bounds: Bounds = field(default_factory=lambda: Bounds(0, 0, 0, 0)) + enabled: bool = True + focused: bool = False + keyboard_shortcut: Optional[str] = None + description: Optional[str] = None + children: List["UIElement"] = field(default_factory=list) + # Extended a11y signals — None means "adapter could not determine". + # Populated where the platform exposes the pattern (UIA SelectionItem / + # ExpandCollapse / RangeValue, AX AXValue/AXMinValue/AXMaxValue, AT-SPI + # STATE_SELECTED / STATE_EXPANDED / IValue) and consumed by the ASCII + # renderer for role-aware glyphs and the structured sidecar. + selected: Optional[bool] = None + expanded: Optional[bool] = None + value_now: Optional[float] = None + value_min: Optional[float] = None + value_max: Optional[float] = None + identifier: Optional[str] = None + + def to_dict(self) -> Dict: + d: Dict[str, Any] = { + "id": self.element_id, + "name": self.name, + "role": self.role, + "value": self.value, + "bounds": self.bounds.to_dict(), + "enabled": self.enabled, + "focused": self.focused, + "keyboard_shortcut": self.keyboard_shortcut, + "description": self.description, + "children": [c.to_dict() for c in self.children], + } + # Omit extended fields when unset so existing API consumers do not + # see a flood of nulls; include them when populated. + for k in ("selected", "expanded", "value_now", "value_min", + "value_max", "identifier"): + v = getattr(self, k) + if v is not None: + d[k] = v + return d + + def flat_list(self) -> List["UIElement"]: + """Return all elements in this subtree as a flat list (DFS order).""" + result = [self] + for child in self.children: + result.extend(child.flat_list()) + return result + + +@dataclass +class WindowInfo: + handle: Any # platform-specific: HWND (int) on Windows; int index elsewhere + title: str + process_name: str + pid: int + bounds: Bounds + is_focused: bool + # Stable cross-call identifier; populated by adapters (design doc §6.1). + window_uid: str = "" + # Optional multi-monitor metadata (design doc §6.3). Populated when the + # adapter knows; left None on adapters that do not. + monitor_index: Optional[int] = None + scale_factor: Optional[float] = None + logical_bounds: Optional[Bounds] = None + physical_bounds: Optional[Bounds] = None + + def to_dict(self) -> Dict: + d: Dict[str, Any] = { + "handle": str(self.handle), + "title": self.title, + "process": self.process_name, + "pid": self.pid, + "bounds": self.bounds.to_dict(), + "focused": self.is_focused, + "window_uid": self.window_uid, + } + if self.monitor_index is not None: + d["monitor_index"] = self.monitor_index + if self.scale_factor is not None: + d["scale_factor"] = self.scale_factor + if self.logical_bounds is not None: + d["logical_bounds"] = self.logical_bounds.to_dict() + if self.physical_bounds is not None: + d["physical_bounds"] = self.physical_bounds.to_dict() + return d + + +# ─── Window resolution result ──────────────────────────────────────────────── + +@dataclass +class WindowResolution: + info: Optional[WindowInfo] + warning: Optional[str] + used_uid: bool + requested_uid: Optional[str] + + +# ─── Subtree helpers (P1 perf: scoped drill-in) ────────────────────────────── + +def find_element_by_path(root: Optional[UIElement], + element_path: str) -> Optional[UIElement]: + """Locate an element by its positional element-id path (e.g. 'root.3.2'). + + Prefers walking the id prefixes level by level (cheap); falls back to a + full DFS by exact id for trees whose ids are not strictly positional + (e.g. nodes injected by tree synthesis get ids like 'root.2.x1').""" + if root is None or not element_path: + return None + if root.element_id == element_path: + return root + # Fast path: navigate children whose ids extend the current prefix. + if element_path.startswith(root.element_id + "."): + node = root + prefix = root.element_id + rest = element_path[len(prefix) + 1:] + found = True + for seg in rest.split("."): + prefix = f"{prefix}.{seg}" + nxt = next((c for c in node.children if c.element_id == prefix), + None) + if nxt is None: + found = False + break + node = nxt + if found: + return node + # Fallback: exhaustive search by exact id. + stack = [root] + while stack: + e = stack.pop() + if e.element_id == element_path: + return e + stack.extend(e.children) + return None + + +def prune_tree_depth(elem: Optional[UIElement], + max_depth: Optional[int]) -> Optional[UIElement]: + """Return a copy of *elem* limited to *max_depth* levels below it. + + Nodes are shallow-copied (bounds objects are shared) so the original — + possibly cache-resident — tree is never mutated.""" + if elem is None or max_depth is None: + return elem + + def _copy(e: UIElement, depth: int) -> UIElement: + kids = ([] if depth >= max_depth + else [_copy(c, depth + 1) for c in e.children]) + return _dc_replace(e, children=kids) + + return _copy(elem, 0) diff --git a/observer/occlusion.py b/observer/occlusion.py new file mode 100644 index 0000000..4ba954d --- /dev/null +++ b/observer/occlusion.py @@ -0,0 +1,116 @@ +""" +Occlusion / visibility: visible areas, screen bounds and +rectangle geometry helpers (mixin consumed by ScreenObserver). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, WindowInfo + + +class OcclusionMixin: + """Occlusion / visibility methods of ScreenObserver.""" + + # Provided by the concrete ScreenObserver. + _adapter: Any + # ── Element occlusion (design doc D14) ──────────────────────────────────── + + def is_element_occluded(self, element_bounds: Bounds, target_hwnd: Any, + all_windows: List[WindowInfo]) -> bool: + """ + True iff every pixel of *element_bounds* is covered by another window + above the target in Z-order, or the element lies entirely off-screen. + + On platforms without Z-order info, returns False (assumed visible). + """ + screen = self.get_screen_bounds() + clipped = _intersect_bounds(element_bounds, screen) + if not clipped: + return True + regions = [clipped] + try: + occluders = self._adapter.get_windows_above_bounds(target_hwnd) + except Exception: + occluders = [] + for occ in occluders: + regions = _subtract_rect(regions, occ) + return not regions + + # ── Visibility helpers ──────────────────────────────────────────────────── + + def get_screen_bounds(self) -> Bounds: + """Return the bounding rect of the combined virtual screen (all monitors).""" + try: + import mss + with mss.MSS() as sct: + m = sct.monitors[0] # index 0 = union of all monitors + return Bounds(m["left"], m["top"], m["width"], m["height"]) + except Exception: + return Bounds(0, 0, 65535, 65535) + + def get_visible_areas(self, target_hwnd: Any, + all_windows: List[WindowInfo]) -> List[Dict]: + """ + Return a list of {x, y, width, height} dicts for the portions of the + window identified by *target_hwnd* that are on-screen and not covered + by windows above it in Z-order. + + On Windows the Z-order is queried precisely via win32gui. + On macOS/Linux Z-order is unavailable, so the full clipped-to-screen + bounds are returned (assuming the window is on top). + """ + target = next((w for w in all_windows if w.handle == target_hwnd), None) + if target is None: + return [] + + screen = self.get_screen_bounds() + clipped = _intersect_bounds(target.bounds, screen) + if not clipped: + return [] + + visible: List[Bounds] = [clipped] + occluders = self._adapter.get_windows_above_bounds(target_hwnd) + for occ in occluders: + visible = _subtract_rect(visible, occ) + + return [b.to_dict() for b in visible] + + +def _intersect_bounds(a: Bounds, b: Bounds) -> Optional[Bounds]: + x1 = max(a.x, b.x) + y1 = max(a.y, b.y) + x2 = min(a.right, b.right) + y2 = min(a.bottom, b.bottom) + if x2 <= x1 or y2 <= y1: + return None + return Bounds(x1, y1, x2 - x1, y2 - y1) + + +def _subtract_rect(rects: List[Bounds], occluder: Bounds) -> List[Bounds]: + """Subtract occluder from each rect, splitting into up to 4 sub-rects.""" + result: List[Bounds] = [] + for r in rects: + ix1 = max(r.x, occluder.x) + iy1 = max(r.y, occluder.y) + ix2 = min(r.right, occluder.right) + iy2 = min(r.bottom, occluder.bottom) + + if ix2 <= ix1 or iy2 <= iy1: + result.append(r) + continue + + # Top strip + if iy1 > r.y: + result.append(Bounds(r.x, r.y, r.width, iy1 - r.y)) + # Bottom strip + if iy2 < r.bottom: + result.append(Bounds(r.x, iy2, r.width, r.bottom - iy2)) + # Left strip (height = intersection height) + if ix1 > r.x: + result.append(Bounds(r.x, iy1, ix1 - r.x, iy2 - iy1)) + # Right strip (height = intersection height) + if ix2 < r.right: + result.append(Bounds(ix2, iy1, r.right - ix2, iy2 - iy1)) + return result diff --git a/observer/platform_info.py b/observer/platform_info.py new file mode 100644 index 0000000..e47a073 --- /dev/null +++ b/observer/platform_info.py @@ -0,0 +1,24 @@ +""" +Platform detection (Windows / macOS / Linux / WSL). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import platform + +PLATFORM = platform.system() + + +def _is_wsl() -> bool: + """True when running inside Windows Subsystem for Linux (WSL 1 or WSL 2).""" + if PLATFORM != "Linux": + return False + try: + with open("/proc/version") as _f: + return "microsoft" in _f.read().lower() + except Exception: + return False + + +IS_WSL = _is_wsl() +EFFECTIVE_PLATFORM = "WSL" if IS_WSL else PLATFORM From 57489c5692761185e6b207c6f90e1945577a2e89 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 20:12:56 +0000 Subject: [PATCH 12/14] [P3] decompose window_agent.py + web_inspector.py + mcp_server.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical split of the three remaining god-files into packages, each with an __init__.py that re-exports the pre-split surface: window_agent/ (was 1,978 lines; run with 'python -m window_agent') client.py ANSI colours, stdlib HTTP, REST wrappers, LLMClient dispatch.py LLM tool-name → REST endpoint dispatcher tool_schemas.py SCREEN_TOOLS catalogue, tiers, keyword groups, meta-tools prompts.py SYSTEM_PROMPT loop.py run_agent agentic loop + tool-result printing cli.py banner, window picker, interactive prompts, main() __main__.py entry point (preserves the KeyboardInterrupt handler) web_inspector/ (was 1,770 lines) assets.py inline single-page UI HTML template (_HTML) views.py register_routes() — every Flask route, moved verbatim server.py create_web_app() factory (Flask + CORS + ToolContext) mcp_server/ (was 1,237 lines) tool_schemas.py _TOOLS (tools/list schema payload) server.py MCPServer (JSON-RPC transport + dispatch + handlers) (named mcp_server, not mcp, to avoid colliding with the external MCP SDK distribution name) No behavior changes; from web_inspector import create_web_app, from mcp_server import MCPServer, mcp_server._TOOLS and all other existing import paths keep working; all tests pass unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- mcp_server/__init__.py | 23 + mcp_server/server.py | 369 ++++ mcp_server.py => mcp_server/tool_schemas.py | 388 +--- web_inspector/__init__.py | 32 + web_inspector.py => web_inspector/assets.py | 636 +----- web_inspector/server.py | 55 + web_inspector/views.py | 598 ++++++ window_agent.py | 1978 ------------------- window_agent/__init__.py | 102 + window_agent/__main__.py | 10 + window_agent/cli.py | 282 +++ window_agent/client.py | 186 ++ window_agent/dispatch.py | 287 +++ window_agent/loop.py | 245 +++ window_agent/prompts.py | 66 + window_agent/tool_schemas.py | 963 +++++++++ 16 files changed, 3225 insertions(+), 2995 deletions(-) create mode 100644 mcp_server/__init__.py create mode 100644 mcp_server/server.py rename mcp_server.py => mcp_server/tool_schemas.py (66%) mode change 100755 => 100644 create mode 100644 web_inspector/__init__.py rename web_inspector.py => web_inspector/assets.py (67%) mode change 100755 => 100644 create mode 100644 web_inspector/server.py create mode 100644 web_inspector/views.py delete mode 100644 window_agent.py create mode 100644 window_agent/__init__.py create mode 100644 window_agent/__main__.py create mode 100644 window_agent/cli.py create mode 100644 window_agent/client.py create mode 100644 window_agent/dispatch.py create mode 100644 window_agent/loop.py create mode 100644 window_agent/prompts.py create mode 100644 window_agent/tool_schemas.py diff --git a/mcp_server/__init__.py b/mcp_server/__init__.py new file mode 100644 index 0000000..3835228 --- /dev/null +++ b/mcp_server/__init__.py @@ -0,0 +1,23 @@ +""" +mcp_server — MCP stdio server (package form of the former mcp_server.py). + +Implements the Model Context Protocol (2024-11-05) as JSON-RPC 2.0 over +stdin/stdout so that any MCP-capable client (Claude Desktop, Claude Code, +etc.) can use this server as a tool provider. + +ALL output to stdout is MCP protocol JSON. All logging goes to stderr so +that the MCP framing on stdout is never polluted. + +P3 decomposition: the implementation now lives in submodules +(tool_schemas, server). This __init__ re-exports the pre-split surface +so `from mcp_server import MCPServer` / `mcp_server._TOOLS` keep working +unchanged. (The package is deliberately named mcp_server — not mcp — to +avoid colliding with the external MCP SDK distribution name.) +""" + +from __future__ import annotations + +from mcp_server.server import MCPServer +from mcp_server.tool_schemas import _TOOLS + +__all__ = ["MCPServer", "_TOOLS"] diff --git a/mcp_server/server.py b/mcp_server/server.py new file mode 100644 index 0000000..e0d76f0 --- /dev/null +++ b/mcp_server/server.py @@ -0,0 +1,369 @@ +""" +MCP stdio server: JSON-RPC 2.0 transport, message +dispatch and legacy composite tool handlers. + +Split out of mcp_server.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import logging +import sys +import traceback +from typing import Any, Dict + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from observer import ScreenObserver +import tools as _tools + +from mcp_server.tool_schemas import _TOOLS + +logger = logging.getLogger(__name__) + + +class MCPServer: + """ + MCP stdio server. + + Reads newline-delimited JSON-RPC 2.0 messages from stdin, dispatches + to tool handlers, and writes responses to stdout. All log output + is directed to stderr to preserve the integrity of the MCP framing. + """ + + PROTOCOL_VERSION = "2024-11-05" + + def __init__( + self, + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: Dict, + ): + self.observer = observer + self.renderer = renderer + self.describer = describer + self.config = config + + # ── Transport ───────────────────────────────────────────────────────────── + + def _emit(self, payload: Dict) -> None: + """Write a JSON-RPC message to stdout.""" + sys.stdout.write(json.dumps(payload) + "\n") + sys.stdout.flush() + + def _error(self, request_id: Any, code: int, message: str) -> None: + self._emit({ + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": code, "message": message}, + }) + + def _result(self, request_id: Any, result: Any) -> None: + self._emit({"jsonrpc": "2.0", "id": request_id, "result": result}) + + # ── Message dispatch ────────────────────────────────────────────────────── + + def _handle(self, msg: Dict) -> None: + method = msg.get("method", "") + params = msg.get("params") or {} + rid = msg.get("id") # None for notifications + + try: + if method == "initialize": + self._result(rid, { + "protocolVersion": self.PROTOCOL_VERSION, + "serverInfo": { + "name": self.config["mcp"]["server_name"], + "version": self.config["mcp"]["version"], + }, + "capabilities": {"tools": {}}, + }) + + elif method in ("notifications/initialized", "ping"): + if rid is not None: + self._result(rid, {}) + + elif method == "tools/list": + self._result(rid, {"tools": _TOOLS}) + + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments") or {} + result = self._dispatch(tool_name, arguments) + self._result(rid, { + "content": [{"type": "text", "text": json.dumps(result, indent=2)}] + }) + + else: + if rid is not None: + self._error(rid, -32601, f"Method not found: {method}") + + except Exception as e: + print(f"[MCPServer:_handle] {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + if rid is not None: + self._error(rid, -32603, str(e)) + + # ── Tool dispatcher ─────────────────────────────────────────────────────── + + def _dispatch(self, name: str, args: Dict) -> Any: + """Route a tools/call to the appropriate handler.""" + # New centralised tools (P1+) live in tools.py. + if name in _tools.REGISTRY: + ctx = _tools.ToolContext( + observer=self.observer, renderer=self.renderer, + describer=self.describer, config=self.config, + ) + return _tools.dispatch(ctx, name, args) + + try: + windows = self.observer.list_windows() + idx = args.get("window_index") + info = self.observer.window_by_index(windows, idx) + hwnd = info.handle if info else None + + if name == "list_windows": + return self._t_list_windows(windows) + + elif name == "get_window_structure": + return self._t_structure(hwnd, info, args) + + elif name == "get_screen_description": + return self._t_description(hwnd, info, args) + + elif name == "get_screen_sketch": + return self._t_sketch(hwnd, info, args) + + elif name == "get_screenshot": + return self._t_screenshot(hwnd, info) + + elif name == "click_at": + return self._t_click_at(args) + + elif name == "type_text": + return self.observer.perform_action("type", value=args.get("text", "")) + + elif name == "press_key": + return self.observer.perform_action("key", value=args.get("keys", "")) + + elif name == "scroll": + return self.observer.perform_action("scroll", value=args) + + elif name == "get_full_screenshot": + return self._t_full_screenshot(hwnd, info, args) + + elif name == "get_visible_areas": + return self._t_visible_areas(hwnd, info, windows) + + elif name == "bring_to_foreground": + return self._t_bring_to_foreground(hwnd, info, windows) + + else: + return {"error": f"Unknown tool: {name}"} + + except Exception as e: + print(f"[MCPServer:_dispatch:{name}] {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + return {"error": str(e)} + + # ── Individual tool handlers ────────────────────────────────────────────── + + def _t_list_windows(self, windows) -> Dict: + return { + "count": len(windows), + "windows": [ + {"index": i, **w.to_dict()} for i, w in enumerate(windows) + ], + } + + def _t_structure(self, hwnd, info, args) -> Dict: + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree for this window"} + return { + "window": info.title if info else "(focused)", + "element_count": len(tree.flat_list()), + "tree": tree.to_dict(), + } + + def _t_description(self, hwnd, info, args) -> Dict: + mode = args.get("mode", "accessibility") + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree"} + + shot = self.observer.get_screenshot(hwnd) + + if mode == "accessibility": + return {"mode": mode, "description": self.describer.from_tree(tree, info)} + elif mode == "ocr": + if shot is None: + return {"error": "Screenshot unavailable for OCR"} + return {"mode": mode, "description": self.describer.from_ocr(shot)} + elif mode == "vlm": + if shot is None: + return {"error": "Screenshot unavailable for VLM"} + vlm_mode = (self.describer.vlm_cfg.get("mode") or "single").lower() + if vlm_mode == "multipass": + env = self.describer.from_vlm_multipass( + shot, root=tree, window=info, + ) + if env is None: + return {"mode": mode, + "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} + return {"mode": mode, + "description": json.dumps(env, indent=2, ensure_ascii=False), + "vlm_structured": env} + vlm_out = self.describer.from_vlm(shot, root=tree, window=info) + if vlm_out is None: + return {"mode": mode, "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} + return {"mode": mode, "description": vlm_out} + elif mode == "combined": + return {"mode": mode, **self.describer.combined(tree, shot, info)} + else: + return {"error": f"Unknown mode: {mode}"} + + def _t_sketch(self, hwnd, info, args) -> Dict: + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree"} + + ref = info.bounds if info else tree.bounds + result = self.renderer.render_structured( + root = tree, + screen_bounds = ref, + grid_width = args.get("grid_width"), + grid_height = args.get("grid_height"), + ) + out = { + "window": info.title if info else "(focused)", + "grid_width": args.get("grid_width", self.renderer.default_width), + "grid_height": args.get("grid_height", self.renderer.default_height), + "sketch": result["sketch"], + } + if args.get("structured"): + out["elements"] = result["elements"] + out["legend"] = result["legend"] + return out + + def _t_screenshot(self, hwnd, info) -> Dict: + import base64 + shot = self.observer.get_screenshot(hwnd) + if shot is None: + return {"error": "Screenshot capture failed"} + return { + "window": info.title if info else "(full screen)", + "format": "png", + "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + + def _t_full_screenshot(self, hwnd, info, args) -> Dict: + import base64 + # Always capture all monitors combined + shot = self.observer.get_full_display_screenshot() + if shot is None: + return {"error": "Screenshot capture failed"} + + sketch = None + tree = self.observer.get_element_tree(hwnd) if hwnd is not None else None + if tree is not None: + ref = info.bounds if info else self.observer.get_screen_bounds() + # Crop the full-display PNG to window bounds for accurate OCR overlay. + ocr_bytes = shot + if info is not None: + try: + import io as _io2 + from PIL import Image as _Image2 + full_img = _Image2.open(_io2.BytesIO(shot)) + screen_b = self.observer.get_screen_bounds() + crop_box = ( + max(0, info.bounds.x - screen_b.x), + max(0, info.bounds.y - screen_b.y), + min(full_img.width, info.bounds.right - screen_b.x), + min(full_img.height, info.bounds.bottom - screen_b.y), + ) + buf2 = _io2.BytesIO() + full_img.crop(crop_box).save(buf2, format="PNG") + ocr_bytes = buf2.getvalue() + except Exception as e: + logger.debug( + f"[get_full_screenshot] window crop for OCR overlay " + f"failed; using full image: {e}") + sketch = self.renderer.render( + root = tree, + screen_bounds = ref, + grid_width = args.get("grid_width"), + grid_height = args.get("grid_height"), + screenshot_bytes = ocr_bytes, + ) + + img_w = img_h = None + try: + import io as _io + from PIL import Image as _Image + _img = _Image.open(_io.BytesIO(shot)) + img_w, img_h = _img.size + except Exception as e: + logger.debug(f"[get_full_screenshot] size probe failed: {e}") + + return { + "window": info.title if info else "(full screen)", + "screenshot_scope": "full_display", + "format": "png", + "encoding": "base64", + "width": img_w, + "height": img_h, + "data": base64.b64encode(shot).decode(), + "sketch": sketch, + } + + def _t_visible_areas(self, hwnd, info, windows) -> Dict: + if hwnd is None: + return {"error": "window_index is required for get_visible_areas"} + areas = self.observer.get_visible_areas(hwnd, windows) + return { + "window": info.title if info else "(unknown)", + "visible_regions": areas, + } + + def _t_bring_to_foreground(self, hwnd, info, windows) -> Dict: + if hwnd is None: + return {"success": False, + "error": "window_index is required for bring_to_foreground"} + result = self.observer.bring_to_foreground(hwnd, windows) + result["window"] = info.title if info else "(unknown)" + return result + + def _t_click_at(self, args) -> Dict: + return self.observer.perform_action( + "click_at", + value={ + "x": args.get("x", 0), + "y": args.get("y", 0), + "button": args.get("button", "left"), + "double": args.get("double", False), + }, + ) + + # ── Main loop ───────────────────────────────────────────────────────────── + + def run(self) -> None: + """Block on stdin, reading and processing JSON-RPC messages.""" + logger.info("[MCPServer:run] Listening on stdin (MCP mode)") + print("[MCPServer] Ready — listening on stdin", file=sys.stderr) + + for raw_line in sys.stdin: + raw_line = raw_line.strip() + if not raw_line: + continue + try: + msg = json.loads(raw_line) + self._handle(msg) + except json.JSONDecodeError as e: + print(f"[MCPServer:run] JSON parse error: {e}", file=sys.stderr) + except Exception as e: + print(f"[MCPServer:run] Unhandled error: {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) diff --git a/mcp_server.py b/mcp_server/tool_schemas.py old mode 100755 new mode 100644 similarity index 66% rename from mcp_server.py rename to mcp_server/tool_schemas.py index c56343b..810aed2 --- a/mcp_server.py +++ b/mcp_server/tool_schemas.py @@ -1,42 +1,12 @@ """ -mcp_server.py — MCP stdio server (Model Context Protocol, 2024-11-05). +MCP tool schema definitions (tools/list payload). -Implements the MCP protocol as JSON-RPC 2.0 over stdin/stdout so that any -MCP-capable client (Claude Desktop, Claude Code, etc.) can use this server -as a tool provider. - -ALL output to stdout is MCP protocol JSON. All logging goes to stderr so -that the MCP framing on stdout is never polluted. - -Exposed tools -───────────── - list_windows Enumerate visible top-level windows. - get_window_structure Accessibility element tree (JSON). - get_screen_description Prose description (accessibility / OCR / VLM / combined). - get_screen_sketch ASCII spatial layout diagram. - get_screenshot Screenshot as base64 PNG. - click_at Click at pixel coordinates. - type_text Type text into the focused element. - press_key Press a key or key combination. +Split out of mcp_server.py (P3); behavior is unchanged. """ -import json -import logging -import sys -import traceback -from typing import Any, Dict, List - -from observer import ScreenObserver -from ascii_renderer import ASCIIRenderer -from description import DescriptionGenerator -import tools as _tools - -logger = logging.getLogger(__name__) +from __future__ import annotations - -# ───────────────────────────────────────────────────────────────────────────── -# Tool schema definitions -# ───────────────────────────────────────────────────────────────────────────── +from typing import Dict, List _TOOLS: List[Dict] = [ { @@ -885,353 +855,3 @@ }, }, ] - - -# ───────────────────────────────────────────────────────────────────────────── -# MCPServer -# ───────────────────────────────────────────────────────────────────────────── - -class MCPServer: - """ - MCP stdio server. - - Reads newline-delimited JSON-RPC 2.0 messages from stdin, dispatches - to tool handlers, and writes responses to stdout. All log output - is directed to stderr to preserve the integrity of the MCP framing. - """ - - PROTOCOL_VERSION = "2024-11-05" - - def __init__( - self, - observer: ScreenObserver, - renderer: ASCIIRenderer, - describer: DescriptionGenerator, - config: Dict, - ): - self.observer = observer - self.renderer = renderer - self.describer = describer - self.config = config - - # ── Transport ───────────────────────────────────────────────────────────── - - def _emit(self, payload: Dict) -> None: - """Write a JSON-RPC message to stdout.""" - sys.stdout.write(json.dumps(payload) + "\n") - sys.stdout.flush() - - def _error(self, request_id: Any, code: int, message: str) -> None: - self._emit({ - "jsonrpc": "2.0", - "id": request_id, - "error": {"code": code, "message": message}, - }) - - def _result(self, request_id: Any, result: Any) -> None: - self._emit({"jsonrpc": "2.0", "id": request_id, "result": result}) - - # ── Message dispatch ────────────────────────────────────────────────────── - - def _handle(self, msg: Dict) -> None: - method = msg.get("method", "") - params = msg.get("params") or {} - rid = msg.get("id") # None for notifications - - try: - if method == "initialize": - self._result(rid, { - "protocolVersion": self.PROTOCOL_VERSION, - "serverInfo": { - "name": self.config["mcp"]["server_name"], - "version": self.config["mcp"]["version"], - }, - "capabilities": {"tools": {}}, - }) - - elif method in ("notifications/initialized", "ping"): - if rid is not None: - self._result(rid, {}) - - elif method == "tools/list": - self._result(rid, {"tools": _TOOLS}) - - elif method == "tools/call": - tool_name = params.get("name", "") - arguments = params.get("arguments") or {} - result = self._dispatch(tool_name, arguments) - self._result(rid, { - "content": [{"type": "text", "text": json.dumps(result, indent=2)}] - }) - - else: - if rid is not None: - self._error(rid, -32601, f"Method not found: {method}") - - except Exception as e: - print(f"[MCPServer:_handle] {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - if rid is not None: - self._error(rid, -32603, str(e)) - - # ── Tool dispatcher ─────────────────────────────────────────────────────── - - def _dispatch(self, name: str, args: Dict) -> Any: - """Route a tools/call to the appropriate handler.""" - # New centralised tools (P1+) live in tools.py. - if name in _tools.REGISTRY: - ctx = _tools.ToolContext( - observer=self.observer, renderer=self.renderer, - describer=self.describer, config=self.config, - ) - return _tools.dispatch(ctx, name, args) - - try: - windows = self.observer.list_windows() - idx = args.get("window_index") - info = self.observer.window_by_index(windows, idx) - hwnd = info.handle if info else None - - if name == "list_windows": - return self._t_list_windows(windows) - - elif name == "get_window_structure": - return self._t_structure(hwnd, info, args) - - elif name == "get_screen_description": - return self._t_description(hwnd, info, args) - - elif name == "get_screen_sketch": - return self._t_sketch(hwnd, info, args) - - elif name == "get_screenshot": - return self._t_screenshot(hwnd, info) - - elif name == "click_at": - return self._t_click_at(args) - - elif name == "type_text": - return self.observer.perform_action("type", value=args.get("text", "")) - - elif name == "press_key": - return self.observer.perform_action("key", value=args.get("keys", "")) - - elif name == "scroll": - return self.observer.perform_action("scroll", value=args) - - elif name == "get_full_screenshot": - return self._t_full_screenshot(hwnd, info, args) - - elif name == "get_visible_areas": - return self._t_visible_areas(hwnd, info, windows) - - elif name == "bring_to_foreground": - return self._t_bring_to_foreground(hwnd, info, windows) - - else: - return {"error": f"Unknown tool: {name}"} - - except Exception as e: - print(f"[MCPServer:_dispatch:{name}] {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - return {"error": str(e)} - - # ── Individual tool handlers ────────────────────────────────────────────── - - def _t_list_windows(self, windows) -> Dict: - return { - "count": len(windows), - "windows": [ - {"index": i, **w.to_dict()} for i, w in enumerate(windows) - ], - } - - def _t_structure(self, hwnd, info, args) -> Dict: - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree for this window"} - return { - "window": info.title if info else "(focused)", - "element_count": len(tree.flat_list()), - "tree": tree.to_dict(), - } - - def _t_description(self, hwnd, info, args) -> Dict: - mode = args.get("mode", "accessibility") - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree"} - - shot = self.observer.get_screenshot(hwnd) - - if mode == "accessibility": - return {"mode": mode, "description": self.describer.from_tree(tree, info)} - elif mode == "ocr": - if shot is None: - return {"error": "Screenshot unavailable for OCR"} - return {"mode": mode, "description": self.describer.from_ocr(shot)} - elif mode == "vlm": - if shot is None: - return {"error": "Screenshot unavailable for VLM"} - vlm_mode = (self.describer.vlm_cfg.get("mode") or "single").lower() - if vlm_mode == "multipass": - env = self.describer.from_vlm_multipass( - shot, root=tree, window=info, - ) - if env is None: - return {"mode": mode, - "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} - return {"mode": mode, - "description": json.dumps(env, indent=2, ensure_ascii=False), - "vlm_structured": env} - vlm_out = self.describer.from_vlm(shot, root=tree, window=info) - if vlm_out is None: - return {"mode": mode, "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} - return {"mode": mode, "description": vlm_out} - elif mode == "combined": - return {"mode": mode, **self.describer.combined(tree, shot, info)} - else: - return {"error": f"Unknown mode: {mode}"} - - def _t_sketch(self, hwnd, info, args) -> Dict: - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree"} - - ref = info.bounds if info else tree.bounds - result = self.renderer.render_structured( - root = tree, - screen_bounds = ref, - grid_width = args.get("grid_width"), - grid_height = args.get("grid_height"), - ) - out = { - "window": info.title if info else "(focused)", - "grid_width": args.get("grid_width", self.renderer.default_width), - "grid_height": args.get("grid_height", self.renderer.default_height), - "sketch": result["sketch"], - } - if args.get("structured"): - out["elements"] = result["elements"] - out["legend"] = result["legend"] - return out - - def _t_screenshot(self, hwnd, info) -> Dict: - import base64 - shot = self.observer.get_screenshot(hwnd) - if shot is None: - return {"error": "Screenshot capture failed"} - return { - "window": info.title if info else "(full screen)", - "format": "png", - "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - - def _t_full_screenshot(self, hwnd, info, args) -> Dict: - import base64 - # Always capture all monitors combined - shot = self.observer.get_full_display_screenshot() - if shot is None: - return {"error": "Screenshot capture failed"} - - sketch = None - tree = self.observer.get_element_tree(hwnd) if hwnd is not None else None - if tree is not None: - ref = info.bounds if info else self.observer.get_screen_bounds() - # Crop the full-display PNG to window bounds for accurate OCR overlay. - ocr_bytes = shot - if info is not None: - try: - import io as _io2 - from PIL import Image as _Image2 - full_img = _Image2.open(_io2.BytesIO(shot)) - screen_b = self.observer.get_screen_bounds() - crop_box = ( - max(0, info.bounds.x - screen_b.x), - max(0, info.bounds.y - screen_b.y), - min(full_img.width, info.bounds.right - screen_b.x), - min(full_img.height, info.bounds.bottom - screen_b.y), - ) - buf2 = _io2.BytesIO() - full_img.crop(crop_box).save(buf2, format="PNG") - ocr_bytes = buf2.getvalue() - except Exception as e: - logger.debug( - f"[get_full_screenshot] window crop for OCR overlay " - f"failed; using full image: {e}") - sketch = self.renderer.render( - root = tree, - screen_bounds = ref, - grid_width = args.get("grid_width"), - grid_height = args.get("grid_height"), - screenshot_bytes = ocr_bytes, - ) - - img_w = img_h = None - try: - import io as _io - from PIL import Image as _Image - _img = _Image.open(_io.BytesIO(shot)) - img_w, img_h = _img.size - except Exception as e: - logger.debug(f"[get_full_screenshot] size probe failed: {e}") - - return { - "window": info.title if info else "(full screen)", - "screenshot_scope": "full_display", - "format": "png", - "encoding": "base64", - "width": img_w, - "height": img_h, - "data": base64.b64encode(shot).decode(), - "sketch": sketch, - } - - def _t_visible_areas(self, hwnd, info, windows) -> Dict: - if hwnd is None: - return {"error": "window_index is required for get_visible_areas"} - areas = self.observer.get_visible_areas(hwnd, windows) - return { - "window": info.title if info else "(unknown)", - "visible_regions": areas, - } - - def _t_bring_to_foreground(self, hwnd, info, windows) -> Dict: - if hwnd is None: - return {"success": False, - "error": "window_index is required for bring_to_foreground"} - result = self.observer.bring_to_foreground(hwnd, windows) - result["window"] = info.title if info else "(unknown)" - return result - - def _t_click_at(self, args) -> Dict: - return self.observer.perform_action( - "click_at", - value={ - "x": args.get("x", 0), - "y": args.get("y", 0), - "button": args.get("button", "left"), - "double": args.get("double", False), - }, - ) - - # ── Main loop ───────────────────────────────────────────────────────────── - - def run(self) -> None: - """Block on stdin, reading and processing JSON-RPC messages.""" - logger.info("[MCPServer:run] Listening on stdin (MCP mode)") - print("[MCPServer] Ready — listening on stdin", file=sys.stderr) - - for raw_line in sys.stdin: - raw_line = raw_line.strip() - if not raw_line: - continue - try: - msg = json.loads(raw_line) - self._handle(msg) - except json.JSONDecodeError as e: - print(f"[MCPServer:run] JSON parse error: {e}", file=sys.stderr) - except Exception as e: - print(f"[MCPServer:run] Unhandled error: {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) diff --git a/web_inspector/__init__.py b/web_inspector/__init__.py new file mode 100644 index 0000000..c2c0abf --- /dev/null +++ b/web_inspector/__init__.py @@ -0,0 +1,32 @@ +""" +web_inspector — Human-facing inspection interface (package form of the +former web_inspector.py). + +Exposes a Flask HTTP server on localhost:5001 (configurable) with: + + GET / — Single-page inspection UI + GET /api/windows — List windows + GET /api/structure — Accessibility tree JSON + GET /api/description — Textual description (combined: accessibility + OCR + VLM) + GET /api/sketch — ASCII layout sketch + GET /api/screenshot — Screenshot as base64 PNG + POST /api/action — Execute an input action + +The HTML/CSS/JS is inlined as a template string (assets.py) so the entire +server is importable with no external static files. + +CORS policy: no CORS headers are emitted unless web_ui.cors_origins is set +in config (default [] = same-origin only). See create_web_app(). + +P3 decomposition: the implementation now lives in submodules (assets, +views, server). This __init__ re-exports the pre-split surface so +`from web_inspector import create_web_app` keeps working unchanged. +""" + +from __future__ import annotations + +from web_inspector.assets import _HTML +from web_inspector.server import create_web_app +from web_inspector.views import register_routes + +__all__ = ["_HTML", "create_web_app", "register_routes"] diff --git a/web_inspector.py b/web_inspector/assets.py old mode 100755 new mode 100644 similarity index 67% rename from web_inspector.py rename to web_inspector/assets.py index 131bedd..42a796e --- a/web_inspector.py +++ b/web_inspector/assets.py @@ -1,43 +1,10 @@ """ -web_inspector.py — Human-facing inspection interface. +Inline single-page inspection UI (HTML/CSS/JS template). -Exposes a Flask HTTP server on localhost:5001 (configurable) with: - - GET / — Single-page inspection UI - GET /api/windows — List windows - GET /api/structure — Accessibility tree JSON - GET /api/description — Textual description (combined: accessibility + OCR + VLM) - GET /api/sketch — ASCII layout sketch - GET /api/screenshot — Screenshot as base64 PNG - POST /api/action — Execute an input action - -The HTML/CSS/JS is inlined as a template string so the entire server is a -single importable Python module with no external static files. - -CORS policy: no CORS headers are emitted unless web_ui.cors_origins is set -in config (default [] = same-origin only). See create_web_app(). +Split out of web_inspector.py (P3); behavior is unchanged. """ -import base64 -import logging -import traceback -from typing import Optional - -from flask import Flask, jsonify, request -from flask_cors import CORS - -from ascii_renderer import ASCIIRenderer -from description import DescriptionGenerator -from observer import ScreenObserver -import tools as _tools -from errors import http_status_for - -logger = logging.getLogger(__name__) - - -# ───────────────────────────────────────────────────────────────────────────── -# HTML template (single-page application, no external dependencies) -# ───────────────────────────────────────────────────────────────────────────── +from __future__ import annotations _HTML = r""" @@ -1171,600 +1138,3 @@ """ - - -# ───────────────────────────────────────────────────────────────────────────── -# Flask application factory -# ───────────────────────────────────────────────────────────────────────────── - -def create_web_app( - observer: ScreenObserver, - renderer: ASCIIRenderer, - describer: DescriptionGenerator, - config: dict, -) -> Flask: - """ - Create and configure the Flask inspection application. - - All routes are defined inside this factory so they close over the - shared observer/renderer/describer instances. - """ - app = Flask(__name__) - - # CORS is opt-in (web_ui.cors_origins, default []). With the default no - # Access-Control-Allow-Origin header is ever sent, so browsers enforce - # same-origin — the bundled inspector UI at "/" is served same-origin and - # keeps working. Operators can list explicit origins, or ["*"] for - # Docker/testing scenarios where cross-origin dashboards need access. - # Never enable "*" together with a non-loopback bind outside an isolated - # environment: /api/action is unauthenticated and destructive. - cors_origins = list((config.get("web_ui") or {}).get("cors_origins") or []) - if cors_origins: - CORS(app, origins=cors_origins) - logger.warning(f"CORS enabled for origins: {cors_origins}") - - ctx = _tools.ToolContext(observer=observer, renderer=renderer, - describer=describer, config=config) - - def _tool_response(name: str, args: dict): - result = _tools.dispatch(ctx, name, args) - if not result.get("ok", True): - code = (result.get("error") or {}).get("code", "Internal") - return jsonify(result), http_status_for(code) - return jsonify(result) - - def _merge_query(extra: Optional[dict] = None) -> dict: - out: dict = {k: v for k, v in request.args.items()} - if "window_index" in out: - try: - out["window_index"] = int(out["window_index"]) - except (TypeError, ValueError): - pass - if extra: - out.update(extra) - return out - - # ── UI ──────────────────────────────────────────────────────────────────── - - @app.route("/") - def index(): - return _HTML, 200, {"Content-Type": "text/html; charset=utf-8"} - - # ── API helpers ─────────────────────────────────────────────────────────── - - def _window_from_args(): - """Resolve window_uid, window_index, or window_title → (WindowInfo, hwnd, windows). - - Priority: window_uid > window_index > window_title (substring match). - """ - windows = observer.list_windows() - res = observer.resolve_window( - windows, - window_uid=request.args.get("window_uid"), - window_index=int(request.args["window_index"]) if "window_index" in request.args else None, - window_title=request.args.get("window_title"), - ) - info = res.info - hwnd = info.handle if info else None - return info, hwnd, windows - - # ── /api/windows ────────────────────────────────────────────────────────── - - @app.route("/api/windows") - def api_windows(): - return _tool_response("list_windows", {}) - - # ── /api/structure ──────────────────────────────────────────────────────── - - @app.route("/api/structure") - def api_structure(): - # Forwards through tools.dispatch so callers can use roles=, - # name_regex=, prune_empty=, max_nodes=, page_cursor= filters. - args = _merge_query() - for key in ("roles", "exclude_roles"): - if key in args and isinstance(args[key], str): - args[key] = [s for s in args[key].split(",") if s] - for bool_key in ("visible_only", "prune_empty"): - if bool_key in args: - args[bool_key] = str(args[bool_key]).lower() in ("1", "true", "yes") - for int_key in ("max_text_len", "max_nodes", "depth"): - if int_key in args: - try: - args[int_key] = int(args[int_key]) - except (TypeError, ValueError): - args.pop(int_key, None) - return _tool_response("get_window_structure", args) - - # ── /api/description ────────────────────────────────────────────────────── - - @app.route("/api/description") - def api_description(): - args = _merge_query() - if "max_tokens" in args: - try: - args["max_tokens"] = int(args["max_tokens"]) - except (TypeError, ValueError): - args.pop("max_tokens", None) - return _tool_response("get_screen_description", args) - - # ── /api/sketch ─────────────────────────────────────────────────────────── - - @app.route("/api/sketch") - def api_sketch(): - try: - info, hwnd, _ = _window_from_args() - tree = observer.get_element_tree(hwnd) - if tree is None: - return jsonify({"error": "Could not retrieve element tree"}), 500 - - gw = request.args.get("grid_width", type=int) - gh = request.args.get("grid_height", type=int) - ref = info.bounds if info else tree.bounds - - # Optional OCR overlay: pass ?ocr=1 to enable Tesseract text overlay. - # Requires pytesseract + tesseract on PATH; silently skipped otherwise. - shot_bytes: Optional[bytes] = None - if request.args.get("ocr", "").strip() in ("1", "true", "yes"): - shot_bytes = observer.get_screenshot(hwnd) - - want_structured = request.args.get("structured", "").strip() in ( - "1", "true", "yes", - ) - result = renderer.render_structured( - root = tree, - screen_bounds = ref, - grid_width = gw, - grid_height = gh, - screenshot_bytes = shot_bytes, - ) - payload = { - "window": info.title if info else "(focused)", - "grid_width": gw or renderer.default_width, - "grid_height": gh or renderer.default_height, - "ocr_overlay": shot_bytes is not None, - "sketch": result["sketch"], - } - if want_structured: - payload["elements"] = result["elements"] - payload["legend"] = result["legend"] - return jsonify(payload) - except Exception as e: - print(f"[web_inspector:/api/sketch] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/screenshot ─────────────────────────────────────────────────────── - - @app.route("/api/screenshot") - def api_screenshot(): - try: - info, hwnd, _ = _window_from_args() - shot = observer.get_screenshot(hwnd) - if shot is None: - return jsonify({"error": "Screenshot capture failed"}), 500 - return jsonify({ - "window": info.title if info else "(full screen)", - "format": "png", - "encoding": "base64", - "data": base64.b64encode(shot).decode(), - }) - except Exception as e: - print(f"[web_inspector:/api/screenshot] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/full_screenshot ────────────────────────────────────────────────── - - @app.route("/api/full_screenshot") - def api_full_screenshot(): - """All-monitor screenshot + optional ASCII sketch in one call.""" - try: - info, hwnd, _ = _window_from_args() - # Always capture the full virtual desktop (all monitors combined) - shot = observer.get_full_display_screenshot() - if shot is None: - return jsonify({"error": "Screenshot capture failed"}), 500 - - sketch: Optional[str] = None - tree = observer.get_element_tree(hwnd) if hwnd is not None else None - if tree is not None: - gw = request.args.get("grid_width", type=int) - gh = request.args.get("grid_height", type=int) - ref = info.bounds if info else observer.get_screen_bounds() - # Crop the full-display PNG to the window's bounds so that OCR - # word coordinates (which are window-relative in ascii_renderer) - # align correctly with the sketch grid. - ocr_bytes = shot - if info is not None: - try: - import io as _io2 - from PIL import Image as _Image2 - full_img = _Image2.open(_io2.BytesIO(shot)) - screen_b = observer.get_screen_bounds() - crop_box = ( - info.bounds.x - screen_b.x, - info.bounds.y - screen_b.y, - info.bounds.right - screen_b.x, - info.bounds.bottom - screen_b.y, - ) - crop_box = ( - max(0, crop_box[0]), - max(0, crop_box[1]), - min(full_img.width, crop_box[2]), - min(full_img.height, crop_box[3]), - ) - buf2 = _io2.BytesIO() - full_img.crop(crop_box).save(buf2, format="PNG") - ocr_bytes = buf2.getvalue() - except Exception as e: - logger.debug( - f"[/api/full_screenshot] window crop for OCR " - f"overlay failed; using full image: {e}") - sketch = renderer.render( - root = tree, - screen_bounds = ref, - grid_width = gw, - grid_height = gh, - screenshot_bytes = ocr_bytes, - ) - - try: - import io as _io - from PIL import Image as _Image - _img = _Image.open(_io.BytesIO(shot)) - img_w, img_h = _img.size - except Exception: - img_w = img_h = None - - return jsonify({ - "window": info.title if info else "(full screen)", - "screenshot_scope": "full_display", - "format": "png", - "encoding": "base64", - "width": img_w, - "height": img_h, - "data": base64.b64encode(shot).decode(), - "sketch": sketch, - }) - except Exception as e: - print(f"[web_inspector:/api/full_screenshot] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/visible_areas ──────────────────────────────────────────────────── - - @app.route("/api/visible_areas") - def api_visible_areas(): - """Visible (non-occluded, on-screen) bounding boxes for a window.""" - try: - info, hwnd, windows = _window_from_args() - if hwnd is None: - return jsonify({"error": "window_uid or window_index is required"}), 400 - areas = observer.get_visible_areas(hwnd, windows) - return jsonify({ - "window": info.title if info else "(unknown)", - "visible_regions": areas, - }) - except Exception as e: - print(f"[web_inspector:/api/visible_areas] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/bring_to_foreground ────────────────────────────────────────────── - - @app.route("/api/bring_to_foreground") - def api_bring_to_foreground(): - """Click the title bar of a window to bring it to the foreground.""" - try: - info, hwnd, windows = _window_from_args() - if hwnd is None: - return jsonify({"success": False, - "error": "window_uid or window_index is required"}), 400 - result = observer.bring_to_foreground(hwnd, windows) - result["window"] = info.title if info else "(unknown)" - result["window_uid"] = info.window_uid if info else None - return jsonify(result) - except Exception as e: - print(f"[web_inspector:/api/bring_to_foreground] {e}") - traceback.print_exc() - return jsonify({"success": False, "error": str(e)}), 500 - - # ── /api/action ─────────────────────────────────────────────────────────── - - @app.route("/api/action", methods=["POST"]) - def api_action(): - body = request.get_json(force=True) or {} - action = body.get("action", "") - if action == "click_at": - return _tool_response("click_at", { - "x": body.get("x", 0), "y": body.get("y", 0), - "button": body.get("button", "left"), - "double": body.get("double", False), - }) - if action == "type": - return _tool_response("type_text", {"text": body.get("value", "")}) - if action == "key": - return _tool_response("press_key", {"keys": body.get("value", "")}) - if action == "scroll": - return _tool_response("scroll", body) - return jsonify({"success": False, "ok": False, - "error": f"Unknown action: {action}"}), 400 - - # ── P1: identity, capabilities, element-targeted actions ───────────────── - - @app.route("/api/capabilities") - def api_capabilities(): - return _tool_response("get_capabilities", {}) - - @app.route("/api/monitors") - def api_monitors(): - return _tool_response("get_monitors", {}) - - @app.route("/api/find_element") - def api_find_element(): - return _tool_response("find_element", _merge_query()) - - @app.route("/api/element/click", methods=["POST"]) - def api_element_click(): - return _tool_response("click_element", request.get_json(force=True) or {}) - - @app.route("/api/element/focus", methods=["POST"]) - def api_element_focus(): - return _tool_response("focus_element", request.get_json(force=True) or {}) - - @app.route("/api/element/set_value", methods=["POST"]) - def api_element_set_value(): - return _tool_response("set_value", request.get_json(force=True) or {}) - - @app.route("/api/element/invoke", methods=["POST"]) - def api_element_invoke(): - return _tool_response("invoke_element", request.get_json(force=True) or {}) - - @app.route("/api/element/select", methods=["POST"]) - def api_element_select(): - return _tool_response("select_option", request.get_json(force=True) or {}) - - # ── P2: observe-with-diff, snapshots, wait_for ────────────────────────── - - @app.route("/api/observe") - def api_observe(): - args = _merge_query() - if "depth" in args: - try: - args["depth"] = int(args["depth"]) - except (TypeError, ValueError): - args.pop("depth", None) - if "changed_only" in args: - args["changed_only"] = str(args["changed_only"]).lower() in ( - "1", "true", "yes") - return _tool_response("observe_window", args) - - @app.route("/api/snapshot", methods=["POST"]) - def api_snapshot(): - return _tool_response("snapshot", request.get_json(silent=True) or {}) - - @app.route("/api/snapshot/") - def api_snapshot_get(sid: str): - return _tool_response("snapshot_get", {"snapshot_id": sid}) - - @app.route("/api/snapshot/diff", methods=["POST"]) - def api_snapshot_diff(): - return _tool_response("snapshot_diff", request.get_json(force=True) or {}) - - @app.route("/api/snapshot/", methods=["DELETE"]) - def api_snapshot_drop(sid: str): - return _tool_response("snapshot_drop", {"snapshot_id": sid}) - - @app.route("/api/wait_for", methods=["POST"]) - def api_wait_for(): - return _tool_response("wait_for", request.get_json(force=True) or {}) - - @app.route("/api/wait_idle", methods=["POST"]) - def api_wait_idle(): - return _tool_response("wait_idle", request.get_json(force=True) or {}) - - @app.route("/api/element/click_and_observe", methods=["POST"]) - def api_element_click_observe(): - return _tool_response("click_element_and_observe", - request.get_json(force=True) or {}) - - @app.route("/api/type_and_observe", methods=["POST"]) - def api_type_observe(): - return _tool_response("type_and_observe", request.get_json(force=True) or {}) - - @app.route("/api/key_and_observe", methods=["POST"]) - def api_key_observe(): - return _tool_response("press_key_and_observe", request.get_json(force=True) or {}) - - # ── P3: filtering, cropping, region OCR, budgeted description ─────────── - - @app.route("/api/screenshot/cropped") - def api_screenshot_cropped(): - return _tool_response("get_screenshot_cropped", _merge_query()) - - @app.route("/api/ocr") - def api_ocr(): - return _tool_response("get_ocr", _merge_query()) - - # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── - - @app.route("/api/trace/start", methods=["POST"]) - def api_trace_start(): - return _tool_response("trace_start", request.get_json(silent=True) or {}) - - @app.route("/api/trace/stop", methods=["POST"]) - def api_trace_stop(): - return _tool_response("trace_stop", {}) - - @app.route("/api/trace/status") - def api_trace_status(): - return _tool_response("trace_status", {}) - - @app.route("/api/replay/start", methods=["POST"]) - def api_replay_start(): - return _tool_response("replay_start", request.get_json(force=True) or {}) - - @app.route("/api/replay/step", methods=["POST"]) - def api_replay_step(): - return _tool_response("replay_step", request.get_json(force=True) or {}) - - @app.route("/api/replay/status", methods=["POST"]) - def api_replay_status(): - return _tool_response("replay_status", request.get_json(force=True) or {}) - - @app.route("/api/replay/stop", methods=["POST"]) - def api_replay_stop(): - return _tool_response("replay_stop", request.get_json(force=True) or {}) - - @app.route("/api/scenario/load", methods=["POST"]) - def api_scenario_load(): - return _tool_response("load_scenario", request.get_json(force=True) or {}) - - @app.route("/api/assert_state", methods=["POST"]) - def api_assert_state(): - return _tool_response("assert_state", request.get_json(force=True) or {}) - - # ── P5: budgets / redaction status / propose ──────────────────────────── - - @app.route("/api/budget_status") - def api_budget_status(): - return _tool_response("get_budget_status", {}) - - @app.route("/api/redaction_status") - def api_redaction_status(): - return _tool_response("get_redaction_status", {}) - - @app.route("/api/propose_action", methods=["POST"]) - def api_propose(): - return _tool_response("propose_action", request.get_json(force=True) or {}) - - # ── P6: extra input verbs ──────────────────────────────────────────────── - - @app.route("/api/hover", methods=["POST"]) - def api_hover(): - body = request.get_json(force=True) or {} - if "x" in body and "y" in body and not body.get("selector") and not body.get("element_id"): - return _tool_response("hover_at", body) - return _tool_response("hover_element", body) - - @app.route("/api/element/right_click", methods=["POST"]) - def api_right_click(): - return _tool_response("right_click_element", request.get_json(force=True) or {}) - - @app.route("/api/element/double_click", methods=["POST"]) - def api_double_click(): - return _tool_response("double_click_element", request.get_json(force=True) or {}) - - @app.route("/api/drag", methods=["POST"]) - def api_drag(): - return _tool_response("drag", request.get_json(force=True) or {}) - - @app.route("/api/element/key", methods=["POST"]) - def api_key_into(): - return _tool_response("key_into_element", request.get_json(force=True) or {}) - - @app.route("/api/element/clear_text", methods=["POST"]) - def api_clear_text(): - return _tool_response("clear_text", request.get_json(force=True) or {}) - - # ── Telemetry: metrics in Prometheus format ───────────────────────────── - - @app.route("/api/metrics") - def api_metrics(): - from session import get_session - s = get_session() - tc = s.tree_cache.counters() - lines = [ - "# HELP oso_step_count Total tool calls processed", - "# TYPE oso_step_count counter", - f"oso_step_count {s.steps.count}", - "# HELP oso_uptime_seconds Process uptime", - "# TYPE oso_uptime_seconds gauge", - f"oso_uptime_seconds {int(s.steps.uptime_s)}", - # ── Tree-cache effectiveness (P2) ───────────────────────────── - "# HELP oso_tree_cache_hits_total Tree-cache lookups served " - "from a fresh cached capture", - "# TYPE oso_tree_cache_hits_total counter", - f"oso_tree_cache_hits_total {tc['hits']}", - "# HELP oso_tree_cache_misses_total Tree-cache lookups that " - "required a fresh accessibility-tree walk", - "# TYPE oso_tree_cache_misses_total counter", - f"oso_tree_cache_misses_total {tc['misses']}", - "# HELP oso_tree_cache_entries Windows currently cached", - "# TYPE oso_tree_cache_entries gauge", - f"oso_tree_cache_entries {tc['entries']}", - # ── Capture-latency summary (P2) ────────────────────────────── - "# HELP oso_tree_capture_ms Accessibility-tree capture latency " - "summary (milliseconds)", - "# TYPE oso_tree_capture_ms summary", - f"oso_tree_capture_ms_sum {tc['capture_ms_total']}", - f"oso_tree_capture_ms_count {tc['capture_count']}", - "# HELP oso_tree_capture_ms_max Slowest tree capture this " - "session (milliseconds)", - "# TYPE oso_tree_capture_ms_max gauge", - f"oso_tree_capture_ms_max {tc['capture_ms_max']}", - ] - if s.budgets is not None: - st = s.budgets.status() - lines += [ - "# TYPE oso_actions_used counter", - f"oso_actions_used {st['actions']['used']}", - "# TYPE oso_screenshots_used counter", - f"oso_screenshots_used {st['screenshots']['used']}", - ] - if s.active_trace is not None: - lines.append("oso_active_trace 1") - else: - lines.append("oso_active_trace 0") - body = "\n".join(lines) + "\n" - return body, 200, {"Content-Type": "text/plain; version=0.0.4"} - - # ── Generic tool console ──────────────────────────────────────────────── - - @app.route("/api/tools") - def api_tools_list(): - return jsonify({"ok": True, - "tools": sorted(_tools.REGISTRY.keys())}) - - @app.route("/api/tool/", methods=["GET", "POST"]) - def api_tool_run(name: str): - if request.method == "POST": - args = request.get_json(silent=True) or {} - else: - args = _merge_query() - return _tool_response(name, args) - - # /api/healthz must stay cheap enough to poll: the OCR diagnostic - # spawns a `tesseract --version` subprocess, so it is computed once - # per process and reused (binary availability doesn't change mid-run). - _healthz_ocr_cache: dict = {} - - @app.route("/api/healthz") - def api_healthz(): - from session import get_session - s = get_session() - out = { - "ok": True, - "uptime_s": int(s.steps.uptime_s), - "step_count": s.steps.count, - "adapter": type(observer._adapter).__name__, - "version": (config.get("mcp", {}) or {}).get("version", "0.2.0"), - # P2 telemetry: cache effectiveness + capture-latency summary. - "tree_cache": s.tree_cache.counters(), - } - # Surface common misconfigurations. - try: - from main import config_load_status - out.update(config_load_status()) - except Exception as e: - logger.debug(f"healthz: config_load_status unavailable: {e}") - try: - if "ocr" not in _healthz_ocr_cache: - from ocr_util import diagnose as _ocr_diag - _healthz_ocr_cache["ocr"] = _ocr_diag(config) - out["ocr"] = _healthz_ocr_cache["ocr"] - except Exception as e: - logger.debug(f"healthz: OCR diagnose failed: {e}") - return jsonify(out) - - return app diff --git a/web_inspector/server.py b/web_inspector/server.py new file mode 100644 index 0000000..fb9b3b9 --- /dev/null +++ b/web_inspector/server.py @@ -0,0 +1,55 @@ +""" +Flask application factory for the inspection server. + +Split out of web_inspector.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging + +from flask import Flask +from flask_cors import CORS + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from observer import ScreenObserver +import tools as _tools + +from web_inspector.views import register_routes + +logger = logging.getLogger(__name__) + + +def create_web_app( + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: dict, +) -> Flask: + """ + Create and configure the Flask inspection application. + + All routes are defined inside this factory so they close over the + shared observer/renderer/describer instances. + """ + app = Flask(__name__) + + # CORS is opt-in (web_ui.cors_origins, default []). With the default no + # Access-Control-Allow-Origin header is ever sent, so browsers enforce + # same-origin — the bundled inspector UI at "/" is served same-origin and + # keeps working. Operators can list explicit origins, or ["*"] for + # Docker/testing scenarios where cross-origin dashboards need access. + # Never enable "*" together with a non-loopback bind outside an isolated + # environment: /api/action is unauthenticated and destructive. + cors_origins = list((config.get("web_ui") or {}).get("cors_origins") or []) + if cors_origins: + CORS(app, origins=cors_origins) + logger.warning(f"CORS enabled for origins: {cors_origins}") + + ctx = _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + register_routes(app, observer=observer, renderer=renderer, + describer=describer, config=config, ctx=ctx) + return app diff --git a/web_inspector/views.py b/web_inspector/views.py new file mode 100644 index 0000000..b2dda9c --- /dev/null +++ b/web_inspector/views.py @@ -0,0 +1,598 @@ +""" +Flask route registration for the inspection API + UI. + +Split out of web_inspector.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +import traceback +from typing import Optional + +from flask import Flask, jsonify, request + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from errors import http_status_for +from observer import ScreenObserver +import tools as _tools + +from web_inspector.assets import _HTML + +logger = logging.getLogger(__name__) + + +def register_routes( + app: Flask, + *, + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: dict, + ctx: _tools.ToolContext, +) -> None: + """Define all routes on *app*, closing over the shared + observer/renderer/describer instances (moved verbatim out of + create_web_app).""" + def _tool_response(name: str, args: dict): + result = _tools.dispatch(ctx, name, args) + if not result.get("ok", True): + code = (result.get("error") or {}).get("code", "Internal") + return jsonify(result), http_status_for(code) + return jsonify(result) + + def _merge_query(extra: Optional[dict] = None) -> dict: + out: dict = {k: v for k, v in request.args.items()} + if "window_index" in out: + try: + out["window_index"] = int(out["window_index"]) + except (TypeError, ValueError): + pass + if extra: + out.update(extra) + return out + + # ── UI ──────────────────────────────────────────────────────────────────── + + @app.route("/") + def index(): + return _HTML, 200, {"Content-Type": "text/html; charset=utf-8"} + + # ── API helpers ─────────────────────────────────────────────────────────── + + def _window_from_args(): + """Resolve window_uid, window_index, or window_title → (WindowInfo, hwnd, windows). + + Priority: window_uid > window_index > window_title (substring match). + """ + windows = observer.list_windows() + res = observer.resolve_window( + windows, + window_uid=request.args.get("window_uid"), + window_index=int(request.args["window_index"]) if "window_index" in request.args else None, + window_title=request.args.get("window_title"), + ) + info = res.info + hwnd = info.handle if info else None + return info, hwnd, windows + + # ── /api/windows ────────────────────────────────────────────────────────── + + @app.route("/api/windows") + def api_windows(): + return _tool_response("list_windows", {}) + + # ── /api/structure ──────────────────────────────────────────────────────── + + @app.route("/api/structure") + def api_structure(): + # Forwards through tools.dispatch so callers can use roles=, + # name_regex=, prune_empty=, max_nodes=, page_cursor= filters. + args = _merge_query() + for key in ("roles", "exclude_roles"): + if key in args and isinstance(args[key], str): + args[key] = [s for s in args[key].split(",") if s] + for bool_key in ("visible_only", "prune_empty"): + if bool_key in args: + args[bool_key] = str(args[bool_key]).lower() in ("1", "true", "yes") + for int_key in ("max_text_len", "max_nodes", "depth"): + if int_key in args: + try: + args[int_key] = int(args[int_key]) + except (TypeError, ValueError): + args.pop(int_key, None) + return _tool_response("get_window_structure", args) + + # ── /api/description ────────────────────────────────────────────────────── + + @app.route("/api/description") + def api_description(): + args = _merge_query() + if "max_tokens" in args: + try: + args["max_tokens"] = int(args["max_tokens"]) + except (TypeError, ValueError): + args.pop("max_tokens", None) + return _tool_response("get_screen_description", args) + + # ── /api/sketch ─────────────────────────────────────────────────────────── + + @app.route("/api/sketch") + def api_sketch(): + try: + info, hwnd, _ = _window_from_args() + tree = observer.get_element_tree(hwnd) + if tree is None: + return jsonify({"error": "Could not retrieve element tree"}), 500 + + gw = request.args.get("grid_width", type=int) + gh = request.args.get("grid_height", type=int) + ref = info.bounds if info else tree.bounds + + # Optional OCR overlay: pass ?ocr=1 to enable Tesseract text overlay. + # Requires pytesseract + tesseract on PATH; silently skipped otherwise. + shot_bytes: Optional[bytes] = None + if request.args.get("ocr", "").strip() in ("1", "true", "yes"): + shot_bytes = observer.get_screenshot(hwnd) + + want_structured = request.args.get("structured", "").strip() in ( + "1", "true", "yes", + ) + result = renderer.render_structured( + root = tree, + screen_bounds = ref, + grid_width = gw, + grid_height = gh, + screenshot_bytes = shot_bytes, + ) + payload = { + "window": info.title if info else "(focused)", + "grid_width": gw or renderer.default_width, + "grid_height": gh or renderer.default_height, + "ocr_overlay": shot_bytes is not None, + "sketch": result["sketch"], + } + if want_structured: + payload["elements"] = result["elements"] + payload["legend"] = result["legend"] + return jsonify(payload) + except Exception as e: + print(f"[web_inspector:/api/sketch] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/screenshot ─────────────────────────────────────────────────────── + + @app.route("/api/screenshot") + def api_screenshot(): + try: + info, hwnd, _ = _window_from_args() + shot = observer.get_screenshot(hwnd) + if shot is None: + return jsonify({"error": "Screenshot capture failed"}), 500 + return jsonify({ + "window": info.title if info else "(full screen)", + "format": "png", + "encoding": "base64", + "data": base64.b64encode(shot).decode(), + }) + except Exception as e: + print(f"[web_inspector:/api/screenshot] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/full_screenshot ────────────────────────────────────────────────── + + @app.route("/api/full_screenshot") + def api_full_screenshot(): + """All-monitor screenshot + optional ASCII sketch in one call.""" + try: + info, hwnd, _ = _window_from_args() + # Always capture the full virtual desktop (all monitors combined) + shot = observer.get_full_display_screenshot() + if shot is None: + return jsonify({"error": "Screenshot capture failed"}), 500 + + sketch: Optional[str] = None + tree = observer.get_element_tree(hwnd) if hwnd is not None else None + if tree is not None: + gw = request.args.get("grid_width", type=int) + gh = request.args.get("grid_height", type=int) + ref = info.bounds if info else observer.get_screen_bounds() + # Crop the full-display PNG to the window's bounds so that OCR + # word coordinates (which are window-relative in ascii_renderer) + # align correctly with the sketch grid. + ocr_bytes = shot + if info is not None: + try: + import io as _io2 + from PIL import Image as _Image2 + full_img = _Image2.open(_io2.BytesIO(shot)) + screen_b = observer.get_screen_bounds() + crop_box = ( + info.bounds.x - screen_b.x, + info.bounds.y - screen_b.y, + info.bounds.right - screen_b.x, + info.bounds.bottom - screen_b.y, + ) + crop_box = ( + max(0, crop_box[0]), + max(0, crop_box[1]), + min(full_img.width, crop_box[2]), + min(full_img.height, crop_box[3]), + ) + buf2 = _io2.BytesIO() + full_img.crop(crop_box).save(buf2, format="PNG") + ocr_bytes = buf2.getvalue() + except Exception as e: + logger.debug( + f"[/api/full_screenshot] window crop for OCR " + f"overlay failed; using full image: {e}") + sketch = renderer.render( + root = tree, + screen_bounds = ref, + grid_width = gw, + grid_height = gh, + screenshot_bytes = ocr_bytes, + ) + + try: + import io as _io + from PIL import Image as _Image + _img = _Image.open(_io.BytesIO(shot)) + img_w, img_h = _img.size + except Exception: + img_w = img_h = None + + return jsonify({ + "window": info.title if info else "(full screen)", + "screenshot_scope": "full_display", + "format": "png", + "encoding": "base64", + "width": img_w, + "height": img_h, + "data": base64.b64encode(shot).decode(), + "sketch": sketch, + }) + except Exception as e: + print(f"[web_inspector:/api/full_screenshot] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/visible_areas ──────────────────────────────────────────────────── + + @app.route("/api/visible_areas") + def api_visible_areas(): + """Visible (non-occluded, on-screen) bounding boxes for a window.""" + try: + info, hwnd, windows = _window_from_args() + if hwnd is None: + return jsonify({"error": "window_uid or window_index is required"}), 400 + areas = observer.get_visible_areas(hwnd, windows) + return jsonify({ + "window": info.title if info else "(unknown)", + "visible_regions": areas, + }) + except Exception as e: + print(f"[web_inspector:/api/visible_areas] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/bring_to_foreground ────────────────────────────────────────────── + + @app.route("/api/bring_to_foreground") + def api_bring_to_foreground(): + """Click the title bar of a window to bring it to the foreground.""" + try: + info, hwnd, windows = _window_from_args() + if hwnd is None: + return jsonify({"success": False, + "error": "window_uid or window_index is required"}), 400 + result = observer.bring_to_foreground(hwnd, windows) + result["window"] = info.title if info else "(unknown)" + result["window_uid"] = info.window_uid if info else None + return jsonify(result) + except Exception as e: + print(f"[web_inspector:/api/bring_to_foreground] {e}") + traceback.print_exc() + return jsonify({"success": False, "error": str(e)}), 500 + + # ── /api/action ─────────────────────────────────────────────────────────── + + @app.route("/api/action", methods=["POST"]) + def api_action(): + body = request.get_json(force=True) or {} + action = body.get("action", "") + if action == "click_at": + return _tool_response("click_at", { + "x": body.get("x", 0), "y": body.get("y", 0), + "button": body.get("button", "left"), + "double": body.get("double", False), + }) + if action == "type": + return _tool_response("type_text", {"text": body.get("value", "")}) + if action == "key": + return _tool_response("press_key", {"keys": body.get("value", "")}) + if action == "scroll": + return _tool_response("scroll", body) + return jsonify({"success": False, "ok": False, + "error": f"Unknown action: {action}"}), 400 + + # ── P1: identity, capabilities, element-targeted actions ───────────────── + + @app.route("/api/capabilities") + def api_capabilities(): + return _tool_response("get_capabilities", {}) + + @app.route("/api/monitors") + def api_monitors(): + return _tool_response("get_monitors", {}) + + @app.route("/api/find_element") + def api_find_element(): + return _tool_response("find_element", _merge_query()) + + @app.route("/api/element/click", methods=["POST"]) + def api_element_click(): + return _tool_response("click_element", request.get_json(force=True) or {}) + + @app.route("/api/element/focus", methods=["POST"]) + def api_element_focus(): + return _tool_response("focus_element", request.get_json(force=True) or {}) + + @app.route("/api/element/set_value", methods=["POST"]) + def api_element_set_value(): + return _tool_response("set_value", request.get_json(force=True) or {}) + + @app.route("/api/element/invoke", methods=["POST"]) + def api_element_invoke(): + return _tool_response("invoke_element", request.get_json(force=True) or {}) + + @app.route("/api/element/select", methods=["POST"]) + def api_element_select(): + return _tool_response("select_option", request.get_json(force=True) or {}) + + # ── P2: observe-with-diff, snapshots, wait_for ────────────────────────── + + @app.route("/api/observe") + def api_observe(): + args = _merge_query() + if "depth" in args: + try: + args["depth"] = int(args["depth"]) + except (TypeError, ValueError): + args.pop("depth", None) + if "changed_only" in args: + args["changed_only"] = str(args["changed_only"]).lower() in ( + "1", "true", "yes") + return _tool_response("observe_window", args) + + @app.route("/api/snapshot", methods=["POST"]) + def api_snapshot(): + return _tool_response("snapshot", request.get_json(silent=True) or {}) + + @app.route("/api/snapshot/") + def api_snapshot_get(sid: str): + return _tool_response("snapshot_get", {"snapshot_id": sid}) + + @app.route("/api/snapshot/diff", methods=["POST"]) + def api_snapshot_diff(): + return _tool_response("snapshot_diff", request.get_json(force=True) or {}) + + @app.route("/api/snapshot/", methods=["DELETE"]) + def api_snapshot_drop(sid: str): + return _tool_response("snapshot_drop", {"snapshot_id": sid}) + + @app.route("/api/wait_for", methods=["POST"]) + def api_wait_for(): + return _tool_response("wait_for", request.get_json(force=True) or {}) + + @app.route("/api/wait_idle", methods=["POST"]) + def api_wait_idle(): + return _tool_response("wait_idle", request.get_json(force=True) or {}) + + @app.route("/api/element/click_and_observe", methods=["POST"]) + def api_element_click_observe(): + return _tool_response("click_element_and_observe", + request.get_json(force=True) or {}) + + @app.route("/api/type_and_observe", methods=["POST"]) + def api_type_observe(): + return _tool_response("type_and_observe", request.get_json(force=True) or {}) + + @app.route("/api/key_and_observe", methods=["POST"]) + def api_key_observe(): + return _tool_response("press_key_and_observe", request.get_json(force=True) or {}) + + # ── P3: filtering, cropping, region OCR, budgeted description ─────────── + + @app.route("/api/screenshot/cropped") + def api_screenshot_cropped(): + return _tool_response("get_screenshot_cropped", _merge_query()) + + @app.route("/api/ocr") + def api_ocr(): + return _tool_response("get_ocr", _merge_query()) + + # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── + + @app.route("/api/trace/start", methods=["POST"]) + def api_trace_start(): + return _tool_response("trace_start", request.get_json(silent=True) or {}) + + @app.route("/api/trace/stop", methods=["POST"]) + def api_trace_stop(): + return _tool_response("trace_stop", {}) + + @app.route("/api/trace/status") + def api_trace_status(): + return _tool_response("trace_status", {}) + + @app.route("/api/replay/start", methods=["POST"]) + def api_replay_start(): + return _tool_response("replay_start", request.get_json(force=True) or {}) + + @app.route("/api/replay/step", methods=["POST"]) + def api_replay_step(): + return _tool_response("replay_step", request.get_json(force=True) or {}) + + @app.route("/api/replay/status", methods=["POST"]) + def api_replay_status(): + return _tool_response("replay_status", request.get_json(force=True) or {}) + + @app.route("/api/replay/stop", methods=["POST"]) + def api_replay_stop(): + return _tool_response("replay_stop", request.get_json(force=True) or {}) + + @app.route("/api/scenario/load", methods=["POST"]) + def api_scenario_load(): + return _tool_response("load_scenario", request.get_json(force=True) or {}) + + @app.route("/api/assert_state", methods=["POST"]) + def api_assert_state(): + return _tool_response("assert_state", request.get_json(force=True) or {}) + + # ── P5: budgets / redaction status / propose ──────────────────────────── + + @app.route("/api/budget_status") + def api_budget_status(): + return _tool_response("get_budget_status", {}) + + @app.route("/api/redaction_status") + def api_redaction_status(): + return _tool_response("get_redaction_status", {}) + + @app.route("/api/propose_action", methods=["POST"]) + def api_propose(): + return _tool_response("propose_action", request.get_json(force=True) or {}) + + # ── P6: extra input verbs ──────────────────────────────────────────────── + + @app.route("/api/hover", methods=["POST"]) + def api_hover(): + body = request.get_json(force=True) or {} + if "x" in body and "y" in body and not body.get("selector") and not body.get("element_id"): + return _tool_response("hover_at", body) + return _tool_response("hover_element", body) + + @app.route("/api/element/right_click", methods=["POST"]) + def api_right_click(): + return _tool_response("right_click_element", request.get_json(force=True) or {}) + + @app.route("/api/element/double_click", methods=["POST"]) + def api_double_click(): + return _tool_response("double_click_element", request.get_json(force=True) or {}) + + @app.route("/api/drag", methods=["POST"]) + def api_drag(): + return _tool_response("drag", request.get_json(force=True) or {}) + + @app.route("/api/element/key", methods=["POST"]) + def api_key_into(): + return _tool_response("key_into_element", request.get_json(force=True) or {}) + + @app.route("/api/element/clear_text", methods=["POST"]) + def api_clear_text(): + return _tool_response("clear_text", request.get_json(force=True) or {}) + + # ── Telemetry: metrics in Prometheus format ───────────────────────────── + + @app.route("/api/metrics") + def api_metrics(): + from session import get_session + s = get_session() + tc = s.tree_cache.counters() + lines = [ + "# HELP oso_step_count Total tool calls processed", + "# TYPE oso_step_count counter", + f"oso_step_count {s.steps.count}", + "# HELP oso_uptime_seconds Process uptime", + "# TYPE oso_uptime_seconds gauge", + f"oso_uptime_seconds {int(s.steps.uptime_s)}", + # ── Tree-cache effectiveness (P2) ───────────────────────────── + "# HELP oso_tree_cache_hits_total Tree-cache lookups served " + "from a fresh cached capture", + "# TYPE oso_tree_cache_hits_total counter", + f"oso_tree_cache_hits_total {tc['hits']}", + "# HELP oso_tree_cache_misses_total Tree-cache lookups that " + "required a fresh accessibility-tree walk", + "# TYPE oso_tree_cache_misses_total counter", + f"oso_tree_cache_misses_total {tc['misses']}", + "# HELP oso_tree_cache_entries Windows currently cached", + "# TYPE oso_tree_cache_entries gauge", + f"oso_tree_cache_entries {tc['entries']}", + # ── Capture-latency summary (P2) ────────────────────────────── + "# HELP oso_tree_capture_ms Accessibility-tree capture latency " + "summary (milliseconds)", + "# TYPE oso_tree_capture_ms summary", + f"oso_tree_capture_ms_sum {tc['capture_ms_total']}", + f"oso_tree_capture_ms_count {tc['capture_count']}", + "# HELP oso_tree_capture_ms_max Slowest tree capture this " + "session (milliseconds)", + "# TYPE oso_tree_capture_ms_max gauge", + f"oso_tree_capture_ms_max {tc['capture_ms_max']}", + ] + if s.budgets is not None: + st = s.budgets.status() + lines += [ + "# TYPE oso_actions_used counter", + f"oso_actions_used {st['actions']['used']}", + "# TYPE oso_screenshots_used counter", + f"oso_screenshots_used {st['screenshots']['used']}", + ] + if s.active_trace is not None: + lines.append("oso_active_trace 1") + else: + lines.append("oso_active_trace 0") + body = "\n".join(lines) + "\n" + return body, 200, {"Content-Type": "text/plain; version=0.0.4"} + + # ── Generic tool console ──────────────────────────────────────────────── + + @app.route("/api/tools") + def api_tools_list(): + return jsonify({"ok": True, + "tools": sorted(_tools.REGISTRY.keys())}) + + @app.route("/api/tool/", methods=["GET", "POST"]) + def api_tool_run(name: str): + if request.method == "POST": + args = request.get_json(silent=True) or {} + else: + args = _merge_query() + return _tool_response(name, args) + + # /api/healthz must stay cheap enough to poll: the OCR diagnostic + # spawns a `tesseract --version` subprocess, so it is computed once + # per process and reused (binary availability doesn't change mid-run). + _healthz_ocr_cache: dict = {} + + @app.route("/api/healthz") + def api_healthz(): + from session import get_session + s = get_session() + out = { + "ok": True, + "uptime_s": int(s.steps.uptime_s), + "step_count": s.steps.count, + "adapter": type(observer._adapter).__name__, + "version": (config.get("mcp", {}) or {}).get("version", "0.2.0"), + # P2 telemetry: cache effectiveness + capture-latency summary. + "tree_cache": s.tree_cache.counters(), + } + # Surface common misconfigurations. + try: + from main import config_load_status + out.update(config_load_status()) + except Exception as e: + logger.debug(f"healthz: config_load_status unavailable: {e}") + try: + if "ocr" not in _healthz_ocr_cache: + from ocr_util import diagnose as _ocr_diag + _healthz_ocr_cache["ocr"] = _ocr_diag(config) + out["ocr"] = _healthz_ocr_cache["ocr"] + except Exception as e: + logger.debug(f"healthz: OCR diagnose failed: {e}") + return jsonify(out) diff --git a/window_agent.py b/window_agent.py deleted file mode 100644 index edcf2fd..0000000 --- a/window_agent.py +++ /dev/null @@ -1,1978 +0,0 @@ -#!/usr/bin/env python3 -""" -window_agent.py — Interactive window inspection and LLM agent for OSScreenObserver. - -Usage: - python window_agent.py [--rest http://127.0.0.1:5001] - -At startup you are prompted for: - OpenWebUI base URL (e.g. http://localhost:3000) - OpenWebUI API key - Model name (e.g. llama3.2:3b or mistral) - -The program then: - 1. Polls the REST server until it responds. - 2. Lists all open windows; you choose one. - 3. Displays the ASCII sketch + accessibility description. - 4. Opens an interactive prompt where you can type a task for the LLM. - 5. The LLM runs an agentic loop, calling screen tools (observe, click, - type, press keys, get OCR, etc.) until the task is done or it stops. - -No API credentials are saved to disk. -""" - -import argparse -import json -import sys -import time -import traceback -import urllib.error -import urllib.parse -import urllib.request -from typing import Any, Dict, List, Optional, Tuple - -# ─── ANSI colour helpers ────────────────────────────────────────────────────── - -_NO_COLOR = not sys.stdout.isatty() - -def _c(text: str, *codes: str) -> str: - if _NO_COLOR: - return text - _MAP = { - "bold": "\033[1m", "dim": "\033[2m", "reset": "\033[0m", - "red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", - "blue": "\033[34m", "magenta": "\033[35m", "cyan": "\033[36m", - "white": "\033[37m", "bright_white": "\033[97m", - } - return "".join(_MAP.get(c, "") for c in codes) + text + "\033[0m" - -# ─── HTTP helpers (stdlib only, no extra deps) ──────────────────────────────── - -def _get(base: str, path: str, params: Optional[Dict] = None, timeout: int = 30) -> Any: - url = base.rstrip("/") + path - if params: - filtered = {k: v for k, v in params.items() if v is not None} - if filtered: - url += "?" + urllib.parse.urlencode(filtered) - with urllib.request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Raise on any redirect so a POST is never silently converted to GET.""" - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise urllib.error.HTTPError(newurl, code, - f"Redirect to {newurl} — update your base URL", - headers, fp) - -_NO_REDIRECT_OPENER = urllib.request.build_opener(_NoRedirectHandler) - - -def _post(base: str, path: str, data: Any, headers: Optional[Dict] = None, - timeout: int = 60) -> Any: - body = json.dumps(data).encode() - req = urllib.request.Request( - base.rstrip("/") + path, - data=body, - headers={"Content-Type": "application/json", **(headers or {})}, - method="POST", - ) - with _NO_REDIRECT_OPENER.open(req, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - -# ─── REST server poller ─────────────────────────────────────────────────────── - -def wait_for_server(rest_base: str, retries: int = 40, delay: float = 0.5) -> bool: - print(_c(" Waiting for REST server …", "dim"), end="", flush=True) - for _ in range(retries): - try: - _get(rest_base, "/api/windows", timeout=2) - print(_c(" ready.", "green")) - return True - except Exception: - print(".", end="", flush=True) - time.sleep(delay) - print(_c(" timed out.", "red")) - return False - -# ─── REST API wrappers ──────────────────────────────────────────────────────── - -def _win_params(uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict[str, Any]: - """Return the minimal window-selector dict: uid > index > title substring.""" - if uid: - return {"window_uid": uid} - if index is not None: - return {"window_index": index} - if title: - return {"window_title": title} - return {} - - -def api_list_windows(rest: str) -> Dict: - return _get(rest, "/api/windows") - - -def api_observe(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - params = _win_params(uid, index, title) - sketch = _get(rest, "/api/sketch", params) - desc = _get(rest, "/api/description", params) - return { - "window": sketch.get("window", "unknown"), - "sketch": sketch.get("sketch", ""), - "description": desc.get("description", ""), - } - - -def api_element_tree(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/structure", _win_params(uid, index, title)) - - -def api_description(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/description", _win_params(uid, index, title)) - - -def api_sketch(rest: str, uid: Optional[str], index: Optional[int], - grid_width: Optional[int] = None, grid_height: Optional[int] = None, - ocr: bool = False, title: Optional[str] = None) -> Dict: - params: Dict[str, Any] = _win_params(uid, index, title) - if grid_width is not None: - params["grid_width"] = grid_width - if grid_height is not None: - params["grid_height"] = grid_height - if ocr: - params["ocr"] = "1" - return _get(rest, "/api/sketch", params) - - -def api_screenshot(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/screenshot", _win_params(uid, index, title)) - - -def api_full_screenshot(rest: str, uid: Optional[str], index: Optional[int], - grid_width: Optional[int] = None, - grid_height: Optional[int] = None, - title: Optional[str] = None) -> Dict: - params: Dict[str, Any] = _win_params(uid, index, title) - if grid_width is not None: - params["grid_width"] = grid_width - if grid_height is not None: - params["grid_height"] = grid_height - return _get(rest, "/api/full_screenshot", params) - - -def api_visible_areas(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/visible_areas", _win_params(uid, index, title)) - - -def api_bring_to_foreground(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/bring_to_foreground", _win_params(uid, index, title)) - - -def api_action(rest: str, payload: Dict) -> Dict: - return _post(rest, "/api/action", payload) - -# ─── Tool dispatcher (maps LLM tool names → REST calls) ────────────────────── - -def dispatch_tool(tool_name: str, args: Dict, rest: str, - default_uid: Optional[str] = None, - default_index: Optional[int] = None) -> Any: - """ - Route a tool call from the LLM to the appropriate REST endpoint. - Returns a Python object (will be JSON-serialised before sending back to LLM). - """ - uid: Optional[str] = args.get("window_uid") - wi: Optional[int] = args.get("window_index") if "window_index" in args else None - title: Optional[str] = args.get("window_title") - # Apply defaults only when the LLM specified none of uid / index / title. - if uid is None and wi is None and title is None: - uid = default_uid - wi = default_index - - if tool_name == "list_windows": - return api_list_windows(rest) - - elif tool_name == "observe_window": - return api_observe(rest, uid, wi, title) - - elif tool_name == "get_element_tree": - return api_element_tree(rest, uid, wi, title) - - elif tool_name == "get_screen_description": - return api_description(rest, uid, wi, title) - - elif tool_name == "get_screen_sketch": - raw_ocr = args.get("ocr", False) - if isinstance(raw_ocr, str): - ocr = raw_ocr.strip().lower() not in ("", "false", "0", "no") - else: - ocr = bool(raw_ocr) - return api_sketch(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), ocr=ocr, title=title) - - elif tool_name == "get_screenshot": - result = api_screenshot(rest, uid, wi, title) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - result["note"] = ( - "Screenshot captured (base64 data omitted from tool result). " - "Use get_screen_description for text content (OCR + VLM when available)." - ) - return result - - elif tool_name == "get_full_screenshot": - result = api_full_screenshot(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), title=title) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - note = "Screenshot captured (base64 data omitted)." - if result.get("sketch"): - note += " Sketch included above." - result["note"] = note - return result - - elif tool_name == "get_visible_areas": - if uid is None and wi is None and title is None: - return {"error": "get_visible_areas requires a selected window (window_uid, window_index, or window_title)"} - return api_visible_areas(rest, uid, wi, title) - - elif tool_name == "bring_to_foreground": - if uid is None and wi is None and title is None: - return {"error": "bring_to_foreground requires a selected window (window_uid, window_index, or window_title)"} - return api_bring_to_foreground(rest, uid, wi, title) - - elif tool_name == "click_at": - payload = { - "action": "click_at", - "x": args["x"], - "y": args["y"], - } - if "button" in args: - payload["button"] = args["button"] - if "double" in args: - payload["double"] = args["double"] - return api_action(rest, payload) - - elif tool_name == "type_text": - return api_action(rest, {"action": "type", "value": args["text"]}) - - elif tool_name == "press_key": - return api_action(rest, {"action": "key", "value": args["keys"]}) - - elif tool_name == "scroll": - payload = {"action": "scroll"} - for k in ("x", "y", "clicks"): - if k in args: - payload[k] = args[k] - return api_action(rest, payload) - - # ── P1: identity / discovery ──────────────────────────────────────────── - elif tool_name == "get_capabilities": - return _get(rest, "/api/capabilities") - - elif tool_name == "get_monitors": - return _get(rest, "/api/monitors") - - elif tool_name == "find_element": - params: Dict[str, Any] = {"selector": args["selector"]} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - return _get(rest, "/api/find_element", params) - - # ── P1 / P6: element-targeted actions ─────────────────────────────────── - elif tool_name in ("click_element", "focus_element", "invoke_element", - "set_value", "select_option", - "right_click_element", "double_click_element", - "hover_element", "clear_text", "key_into_element"): - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - path = { - "click_element": "/api/element/click", - "focus_element": "/api/element/focus", - "invoke_element": "/api/element/invoke", - "set_value": "/api/element/set_value", - "select_option": "/api/element/select", - "right_click_element": "/api/element/right_click", - "double_click_element": "/api/element/double_click", - "hover_element": "/api/hover", - "clear_text": "/api/element/clear_text", - "key_into_element": "/api/element/key", - }[tool_name] - return _post(rest, path, body) - - elif tool_name == "hover_at": - return _post(rest, "/api/hover", {k: args[k] for k in args - if k in ("x", "y", "hover_ms")}) - - elif tool_name == "drag": - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, "/api/drag", body) - - # ── P2: synchronisation and observation ───────────────────────────────── - elif tool_name == "wait_for": - body = dict(args) - return _post(rest, "/api/wait_for", body) - - elif tool_name == "wait_idle": - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, "/api/wait_idle", body) - - elif tool_name == "observe_window_diff": - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - if "since" in args: - params["since"] = args["since"] - if "format" in args: - params["format"] = args["format"] - return _get(rest, "/api/observe", params) - - elif tool_name in ("click_element_and_observe", "type_and_observe", - "press_key_and_observe"): - path = { - "click_element_and_observe": "/api/element/click_and_observe", - "type_and_observe": "/api/type_and_observe", - "press_key_and_observe": "/api/key_and_observe", - }[tool_name] - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, path, body) - - # ── P2: snapshots ─────────────────────────────────────────────────────── - elif tool_name == "snapshot": - return _post(rest, "/api/snapshot", {}) - - elif tool_name == "snapshot_get": - return _get(rest, f"/api/snapshot/{args['snapshot_id']}") - - elif tool_name == "snapshot_diff": - return _post(rest, "/api/snapshot/diff", - {k: args[k] for k in args if k in ("a", "b", "format")}) - - elif tool_name == "snapshot_drop": - sid = args["snapshot_id"] - url = rest.rstrip("/") + f"/api/snapshot/{sid}" - req = urllib.request.Request(url, method="DELETE") - with _NO_REDIRECT_OPENER.open(req, timeout=10) as resp: - return json.loads(resp.read().decode()) - - # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── - elif tool_name == "trace_start": - return _post(rest, "/api/trace/start", - {"label": args.get("label", "")}) - elif tool_name == "trace_stop": - return _post(rest, "/api/trace/stop", {}) - elif tool_name == "trace_status": - return _get(rest, "/api/trace/status") - - elif tool_name == "replay_start": - return _post(rest, "/api/replay/start", - {k: args[k] for k in args - if k in ("path", "mode", "on_divergence")}) - elif tool_name == "replay_step": - return _post(rest, "/api/replay/step", - {"replay_id": args["replay_id"]}) - elif tool_name == "replay_status": - return _post(rest, "/api/replay/status", - {"replay_id": args["replay_id"]}) - elif tool_name == "replay_stop": - return _post(rest, "/api/replay/stop", - {"replay_id": args["replay_id"]}) - - elif tool_name == "load_scenario": - return _post(rest, "/api/scenario/load", {"path": args["path"]}) - - elif tool_name == "assert_state": - return _post(rest, "/api/assert_state", - {"predicate": args.get("predicate", args.get("predicates", []))}) - - # ── P5: safety & status ───────────────────────────────────────────────── - elif tool_name == "get_budget_status": - return _get(rest, "/api/budget_status") - - elif tool_name == "get_redaction_status": - return _get(rest, "/api/redaction_status") - - elif tool_name == "propose_action": - return _post(rest, "/api/propose_action", - {"action": args["action"], "args": args.get("args", {})}) - - # ── P3 extras ─────────────────────────────────────────────────────────── - elif tool_name == "get_screenshot_cropped": - # Pixel data is huge — same omission policy as get_screenshot. - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - for k in ("element_id", "padding_px", "max_width"): - if k in args: - params[k] = args[k] - result = _get(rest, "/api/screenshot/cropped", params) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - result["note"] = "Screenshot captured (base64 data omitted)." - return result - - elif tool_name == "get_ocr": - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - if "element_id" in args: - params["element_id"] = args["element_id"] - return _get(rest, "/api/ocr", params) - - # ── Catch-all: drive any registered tool by name ──────────────────────── - elif tool_name == "call_tool": - name = args["name"] - body = args.get("args", {}) or {} - return _post(rest, f"/api/tool/{name}", body) - - else: - return {"error": f"Unknown tool: {tool_name}"} - -# ─── Tool definitions (OpenAI / OpenWebUI format) ───────────────────────────── - -# ─── Full tool catalogue (trimmed descriptions) ────────────────────────────── -# Each entry is the schema sent to the LLM. Tiers and keyword groups are in -# _TOOL_TIER / _KEYWORD_GROUPS below — not in the dict itself so the JSON -# payload stays clean. - -SCREEN_TOOLS: List[Dict] = [ - # ── Core observation ───────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "list_windows", - "description": "List visible desktop windows. Returns index, title, process, PID, geometry, and window_uid. Prefer window_uid over index — index changes after every focus change.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "observe_window", - "description": "ASCII sketch + accessibility description of a window. Call before and after every action.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer", "description": "From list_windows. Omit for focused window."}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_element_tree", - "description": "Accessibility tree as JSON. Each node has id, name, role, value, enabled, focused, bounds {x,y,width,height}.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screen_description", - "description": "Describe a window using all available sources: accessibility tree, OCR, and VLM (when enabled). Returns everything available in one call.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screen_sketch", - "description": "ASCII spatial layout of a window. ocr=true overlays Tesseract text.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "grid_width": {"type": "integer"}, - "grid_height": {"type": "integer"}, - "ocr": {"type": "boolean"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screenshot", - "description": "Capture a window screenshot. Pixel data omitted; use get_screen_description mode=ocr for text.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_full_screenshot", - "description": "Full-display screenshot + ASCII sketch with OCR for a window.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "grid_width": {"type": "integer"}, - "grid_height": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_visible_areas", - "description": "Non-occluded bounding boxes {x,y,width,height} for a window.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": ["window_index"], - }, - }, - }, - - # ── Core actions ───────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "click_at", - "description": "Click at absolute screen coordinates. Derive from get_element_tree: cx=x+w//2, cy=y+h//2.", - "parameters": { - "type": "object", - "properties": { - "x": {"type": "integer"}, - "y": {"type": "integer"}, - "button": {"type": "string", "enum": ["left", "right", "middle"]}, - "double": {"type": "boolean"}, - }, - "required": ["x", "y"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "type_text", - "description": "Type text into the focused element. Click the field first.", - "parameters": { - "type": "object", - "properties": {"text": {"type": "string"}}, - "required": ["text"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "press_key", - "description": "Press a key or chord. Examples: enter, tab, escape, ctrl+s, alt+f4.", - "parameters": { - "type": "object", - "properties": {"keys": {"type": "string"}}, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "scroll", - "description": "Scroll at (x,y). clicks>0 scrolls up, clicks<0 scrolls down.", - "parameters": { - "type": "object", - "properties": { - "clicks": {"type": "integer"}, - "x": {"type": "integer"}, - "y": {"type": "integer"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "bring_to_foreground", - "description": "Raise a window to the foreground by clicking its title bar. Window indices change after this call — always call list_windows again before using any index.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": ["window_index"], - }, - }, - }, - - # ── Element-targeted actions ────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "find_element", - "description": 'Resolve selector to element_id + bounds. XPath: Window/Pane/Button[name="OK"]. CSS: Window > Button.', - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": ["selector"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "click_element", - "description": "Click by selector or element_id. Returns ActionReceipt (changed, new_dialogs, before/after hashes).", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "button": {"type": "string", "enum": ["left", "right", "middle"]}, - "count": {"type": "integer"}, - "dry_run": {"type": "boolean"}, - "confirm_token": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "click_element_and_observe", - "description": "Click element then return post-click observation diff.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "button": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "type_and_observe", - "description": "type_text + observe_window in one round-trip.", - "parameters": { - "type": "object", - "properties": { - "text": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - "window_uid": {"type": "string"}, - }, - "required": ["text"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "press_key_and_observe", - "description": "press_key + observe_window in one round-trip.", - "parameters": { - "type": "object", - "properties": { - "keys": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - "window_uid": {"type": "string"}, - }, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "right_click_element", - "description": "Right-click an element by selector or element_id.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "double_click_element", - "description": "Double-click an element by selector or element_id.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "select_option", - "description": "Open a combo-box / list and select an option by name or zero-based index.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "option_name": {"type": "string"}, - "option_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "set_value", - "description": "Click an editable element, optionally clear it, then type a value.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "value": {"type": "string"}, - "clear_first": {"type": "boolean"}, - }, - "required": ["value"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "focus_element", - "description": "Give keyboard focus to an element.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "invoke_element", - "description": "Invoke an element's primary action (UIA InvokePattern or click fallback).", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "clear_text", - "description": "Click an editable element then select-all + delete.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "key_into_element", - "description": "Click an element then press a key combination atomically.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "keys": {"type": "string"}, - }, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "hover_at", - "description": "Move mouse to (x, y) and hold hover_ms ms.", - "parameters": { - "type": "object", - "properties": { - "x": {"type": "integer"}, "y": {"type": "integer"}, - "hover_ms": {"type": "integer"}, - }, - "required": ["x", "y"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "hover_element", - "description": "Move mouse to an element's centre and hold hover_ms ms.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "hover_ms": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "drag", - "description": "Drag from one point/element to another. Each endpoint: {x,y} or {selector} or {element_id}.", - "parameters": { - "type": "object", - "properties": { - "from": {"type": "object"}, - "to": {"type": "object"}, - "modifiers": {"type": "array", "items": {"type": "string"}}, - "duration_s": {"type": "number"}, - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - }, - "required": ["from", "to"], - }, - }, - }, - - # ── Synchronisation ─────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "wait_for", - "description": "Block until a condition matches or timeout. Conditions: element_appears, element_disappears, text_visible, window_appears, tree_changes.", - "parameters": { - "type": "object", - "properties": { - "any_of": {"type": "array"}, - "window_uid": {"type": "string"}, - "timeout_ms": {"type": "integer"}, - "poll_ms": {"type": "integer"}, - }, - "required": ["any_of"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "wait_idle", - "description": "Block until the accessibility tree is stable for quiet_ms.", - "parameters": { - "type": "object", - "properties": { - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "quiet_ms": {"type": "integer"}, - "timeout_ms": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "observe_window_diff", - "description": "observe_window but returns only the diff since a tree_token.", - "parameters": { - "type": "object", - "properties": { - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "since": {"type": "string"}, - "format": {"type": "string", "enum": ["custom", "json-patch"]}, - }, - "required": [], - }, - }, - }, - - # ── Snapshots ───────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "snapshot", - "description": "Capture all windows + trees. Returns snapshot_id.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_get", - "description": "Retrieve a saved snapshot.", - "parameters": { - "type": "object", - "properties": {"snapshot_id": {"type": "string"}}, - "required": ["snapshot_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_diff", - "description": "Diff two snapshots (added/removed windows + per-window tree diff).", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "string"}, "b": {"type": "string"}, - "format": {"type": "string", "enum": ["custom", "json-patch"]}, - }, - "required": ["a", "b"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_drop", - "description": "Free a snapshot before its TTL expires.", - "parameters": { - "type": "object", - "properties": {"snapshot_id": {"type": "string"}}, - "required": ["snapshot_id"], - }, - }, - }, - - # ── Tracing / replay ────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "trace_start", - "description": "Start recording tool calls to a JSONL trace file.", - "parameters": { - "type": "object", - "properties": {"label": {"type": "string"}}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "trace_stop", - "description": "Close the active trace.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "trace_status", - "description": "Report whether a trace is active.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "replay_start", - "description": "Load a trace for replay (execute or verify mode).", - "parameters": { - "type": "object", - "properties": { - "path": {"type": "string"}, - "mode": {"type": "string", "enum": ["execute", "verify"]}, - "on_divergence": {"type": "string", "enum": ["stop", "warn", "resume"]}, - }, - "required": ["path"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_step", - "description": "Advance one row of the active replay.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_status", - "description": "Report replay position, total, finished, divergences.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_stop", - "description": "Free a replay handle.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - - # ── Testing / harness ───────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "load_scenario", - "description": "Load a YAML scenario file into the mock adapter.", - "parameters": { - "type": "object", - "properties": {"path": {"type": "string"}}, - "required": ["path"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "assert_state", - "description": "Evaluate predicates against current state. Returns all_passed. Predicates: element_exists, value_equals, text_visible, window_focused, screenshot_similar, …", - "parameters": { - "type": "object", - "properties": { - "predicate": {"type": "array"}, - "predicates": {"type": "array"}, - }, - "required": [], - }, - }, - }, - - # ── Safety / budget ─────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_budget_status", - "description": "Remaining budget: actions, screenshots, vlm_tokens, …", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "get_redaction_status", - "description": "Redaction enabled state and applied count.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "propose_action", - "description": "Issue a confirm_token for a destructive action. Pass the token back to the action to proceed.", - "parameters": { - "type": "object", - "properties": { - "action": {"type": "string"}, - "args": {"type": "object"}, - }, - "required": ["action"], - }, - }, - }, - - # ── Discovery ───────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_capabilities", - "description": "Features the server supports on this host.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "get_monitors", - "description": "Monitors with bounds and scale factor.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - - # ── OCR extras ──────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_screenshot_cropped", - "description": "Cropped screenshot around an element or bbox, with optional max_width downscale.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - "element_id": {"type": "string"}, - "padding_px": {"type": "integer"}, - "max_width": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_ocr", - "description": "Region-scoped OCR. Returns [{text, confidence, bbox}].", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - "element_id": {"type": "string"}, - }, - "required": [], - }, - }, - }, - - # ── Escape hatch ────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "call_tool", - "description": "Call any server tool by name with arbitrary JSON args.", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "args": {"type": "object"}, - }, - "required": ["name"], - }, - }, - }, -] - -# ─── Tool index, tiers, and keyword groups ──────────────────────────────────── - -_TOOL_BY_NAME: Dict[str, Dict] = { - t["function"]["name"]: t for t in SCREEN_TOOLS -} - -# "always" — sent every turn regardless of task -# "usually" — sent by default (covers the vast majority of tasks) -# "on_demand"— omitted unless the task keywords or request_tools activates them -_TOOL_TIER: Dict[str, str] = { - # always - "list_windows": "always", - "observe_window": "always", - "get_element_tree": "always", - "find_element": "always", - "click_element": "always", - "type_text": "always", - "press_key": "always", - "scroll": "always", - "click_element_and_observe": "always", - # usually - "bring_to_foreground": "usually", - "right_click_element": "usually", - "double_click_element": "usually", - "select_option": "usually", - "set_value": "usually", - "wait_idle": "usually", - "type_and_observe": "usually", - "press_key_and_observe": "usually", - "get_screen_description": "usually", - # on_demand (everything else) - "get_screen_sketch": "on_demand", - "get_screenshot": "on_demand", - "get_full_screenshot": "on_demand", - "get_visible_areas": "on_demand", - "get_screenshot_cropped": "on_demand", - "get_ocr": "on_demand", - "hover_at": "on_demand", - "hover_element": "on_demand", - "drag": "on_demand", - "key_into_element": "on_demand", - "clear_text": "on_demand", - "focus_element": "on_demand", - "invoke_element": "on_demand", - "click_at": "on_demand", - "wait_for": "on_demand", - "observe_window_diff": "on_demand", - "snapshot": "on_demand", - "snapshot_get": "on_demand", - "snapshot_diff": "on_demand", - "snapshot_drop": "on_demand", - "trace_start": "on_demand", - "trace_stop": "on_demand", - "trace_status": "on_demand", - "replay_start": "on_demand", - "replay_step": "on_demand", - "replay_status": "on_demand", - "replay_stop": "on_demand", - "load_scenario": "on_demand", - "assert_state": "on_demand", - "get_budget_status": "on_demand", - "get_redaction_status": "on_demand", - "propose_action": "on_demand", - "get_capabilities": "on_demand", - "get_monitors": "on_demand", - "call_tool": "on_demand", -} - -# Task keywords that unlock specific on_demand tools without a request_tools call. -_KEYWORD_GROUPS: Dict[str, List[str]] = { - "drag": ["drag"], - "drop": ["drag"], - "move": ["drag"], - "hover": ["hover_at", "hover_element"], - "tooltip": ["hover_at", "hover_element"], - "screenshot": ["get_full_screenshot", "get_screenshot"], - "capture": ["get_full_screenshot"], - "sketch": ["get_screen_sketch"], - "ocr": ["get_ocr", "get_screen_description"], - "read text": ["get_ocr"], - "cropped": ["get_screenshot_cropped"], - "visible area": ["get_visible_areas"], - "occluded": ["get_visible_areas"], - "snapshot": ["snapshot", "snapshot_get", "snapshot_diff", "snapshot_drop"], - "compare": ["snapshot", "snapshot_diff"], - "diff": ["observe_window_diff", "snapshot_diff"], - "wait for": ["wait_for"], - "appear": ["wait_for"], - "loading": ["wait_for"], - "coordinates": ["click_at"], - "pixel": ["click_at"], - "clear": ["clear_text"], - "erase": ["clear_text"], - "focus": ["focus_element"], - "invoke": ["invoke_element"], - "trace": ["trace_start", "trace_stop", "trace_status"], - "record": ["trace_start", "trace_stop", "trace_status"], - "replay": ["replay_start", "replay_step", "replay_status", "replay_stop"], - "playback": ["replay_start", "replay_step", "replay_status", "replay_stop"], - "assert": ["assert_state"], - "verify state": ["assert_state"], - "budget": ["get_budget_status"], - "redact": ["get_redaction_status"], - "capabilities": ["get_capabilities"], - "monitors": ["get_monitors"], -} - -# Meta-tools always included — handled locally in the agent loop, never sent to REST. -_META_TOOLS: List[Dict] = [ - { - "type": "function", - "function": { - "name": "list_available_tools", - "description": ( - "List tools not yet active in this session, with one-line descriptions. " - "Call request_tools with the names you need to activate them." - ), - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "request_tools", - "description": ( - "Activate additional tools by name for the rest of this session. " - "Use list_available_tools first to see what is available." - ), - "parameters": { - "type": "object", - "properties": { - "names": { - "type": "array", - "items": {"type": "string"}, - "description": "Tool names to activate.", - } - }, - "required": ["names"], - }, - }, - }, -] - - -def _initial_active_tools(task: str) -> set: - """Return the set of tool names to send on turn 1 based on tier + keywords.""" - active = {name for name, tier in _TOOL_TIER.items() if tier in ("always", "usually")} - task_lower = task.lower() - for kw, names in _KEYWORD_GROUPS.items(): - if kw in task_lower: - active.update(names) - return active - - -def _tool_defs_for(active: set) -> List[Dict]: - """Build the tools list to send to the LLM from the active name set.""" - defs = [_TOOL_BY_NAME[n] for n in active if n in _TOOL_BY_NAME] - return defs + _META_TOOLS - - -SYSTEM_PROMPT = """\ -You are a GUI automation agent operating on a live desktop. -You observe screen state through accessibility tools and execute mouse and keyboard actions. - -COORDINATE RULE -All x, y values must come from get_element_tree bounds — never estimate or recall coordinates. -To click the centre of an element with bounds {x, y, width, height}: - click_x = x + width // 2 - click_y = y + height // 2 - -OBSERVATION RULE -You are blind to the screen unless you call observe_window or get_screen_description. -Call observe_window before deciding where to act, and after every action to confirm the result. -Always read the window title from the observe_window result and confirm it matches the window -you intend to act on before proceeding. - -FINDING ELEMENTS — IMPORTANT -The accessibility tree may be incomplete for web pages and some applications. -If a selector or element search fails with NOT FOUND: - 1. Call get_screen_description to get accessibility + OCR + visual text all at once. - 2. Use get_screenshot and inspect the image to identify element positions visually. - 3. Fall back to click_at with coordinates derived from element bounds in the sketch/screenshot. -Do not give up after one NOT FOUND — always try the screenshot/OCR path before reporting failure. - -WINDOW INDEX INSTABILITY — CRITICAL -window_index values change every time a window is raised, minimised, or closed. -• Every window tool call returns window_uid in its response — capture it immediately and use it - on all subsequent calls for that window instead of window_index. -• When you must call by window_index, the server auto-resolves it to the uid and returns - window_uid in the result — read that value and switch to uid= from that point on. -• Never assume the same index still refers to the same window between tool calls. - -BROWSER TAB SWITCHING -Browser tab bars appear as TabItem elements in the accessibility tree. -To switch to a different tab: observe_window, then click_element on the correct TabItem. -The window title updates to reflect the active tab after the click. - -TASK COMPLETION -Complete every part of the user's task before stopping. -Do not ask for clarification or next steps mid-task when the task is unambiguous. -Only report done when all sub-tasks are finished. - -WORKFLOW -1. list_windows — note window_uid for the target window; use uid on all future calls -2. bring_to_foreground(window_uid=…) — raise the window (result includes window_uid if you used index) -3. observe_window(window_uid=…) — verify window title matches; understand current state -4. get_element_tree(window_uid=…) — get exact coordinates when needed -5. Execute one action -6. observe_window — verify title still matches and the change occurred -7. Repeat until ALL sub-tasks are complete - -TOOL AVAILABILITY -Only a subset of tools is active at session start to keep context short. - • list_available_tools() — see what else exists - • request_tools(names=[…]) — activate specific tools for this session - -If an action does not produce the expected result, re-observe and try an alternative approach. -""" - -# OpenWebUI OpenAI-compatible API prefix (centralised so both endpoints stay in sync) -_OWU_PREFIX = "/api/v1" - -# ─── LLM client (OpenAI-compatible, OpenWebUI) ──────────────────────────────── - -class LLMClient: - def __init__(self, base_url: str, api_key: str, model: str): - self.base_url = base_url.rstrip("/") - self.api_key = api_key - self.model = model - - def chat(self, messages: List[Dict], tools: Optional[List[Dict]] = None) -> Dict: - payload: Dict[str, Any] = { - "model": self.model, - "messages": messages, - "temperature": 0.2, - } - if tools: - payload["tools"] = tools - headers = {} - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - return _post(self.base_url, f"{_OWU_PREFIX}/chat/completions", payload, headers, timeout=240) - -# ─── Agentic loop ───────────────────────────────────────────────────────────── - -MAX_TURNS = 30 -_LLM_MAX_RETRIES = 3 # retries for transient network errors per turn -_LLM_RETRY_DELAY = 5.0 # seconds before first retry (doubles each attempt) - -def run_agent( - llm: LLMClient, - rest: str, - user_task: str, - default_uid: Optional[str], - history: List[Dict], - default_index: Optional[int] = None, -) -> List[Dict]: - """ - Run the agentic tool-calling loop. - - Appends messages to *history* in place and returns the updated history. - Prints progress to stdout using ANSI colours. - """ - history.append({"role": "user", "content": user_task}) - print() - print(_c(f" User: {user_task}", "cyan")) - - active_tools: set = _initial_active_tools(user_task) - n_keyword = len(active_tools) - sum( - 1 for t, tier in _TOOL_TIER.items() if tier in ("always", "usually") - ) - print(_c( - f" [Tools: {len(active_tools)} active" - + (f" (+{n_keyword} from keywords)" if n_keyword > 0 else "") - + f" / {len(SCREEN_TOOLS)} total]", - "dim", - )) - - for turn in range(MAX_TURNS): - tool_defs = _tool_defs_for(active_tools) - resp = None - for attempt in range(_LLM_MAX_RETRIES + 1): - try: - resp = llm.chat(history, tools=tool_defs) - break - except (TimeoutError, ConnectionError, OSError) as e: - if attempt < _LLM_MAX_RETRIES: - delay = _LLM_RETRY_DELAY * (2 ** attempt) - print(_c(f"\n [LLM timeout/connection error — retrying in {delay:.0f}s: {e}]", "yellow")) - time.sleep(delay) - else: - print(_c(f"\n [LLM request failed after {_LLM_MAX_RETRIES + 1} attempts: {e}]", "red")) - return history - except urllib.error.URLError as e: - # URLError wraps OSError/TimeoutError — retry those too. - cause = e.reason if hasattr(e, "reason") else e - if isinstance(cause, (TimeoutError, ConnectionError, OSError)) and attempt < _LLM_MAX_RETRIES: - delay = _LLM_RETRY_DELAY * (2 ** attempt) - print(_c(f"\n [LLM network error — retrying in {delay:.0f}s: {e}]", "yellow")) - time.sleep(delay) - else: - print(_c(f"\n [LLM request failed: {e}]", "red")) - return history - except Exception as e: - print(_c(f"\n [LLM error: {e}]", "red")) - traceback.print_exc() - return history - if resp is None: - return history - - choices = resp.get("choices", []) - if not choices: - print(_c(" [Empty response from LLM]", "red")) - break - - choice = choices[0] - message = choice.get("message", {}) - reason = choice.get("finish_reason", "stop") - - # Accumulate assistant message into history - history.append({"role": "assistant", **{k: v for k, v in message.items() - if k not in ("role",)}}) - - # Print any text content the LLM produced - content = message.get("content") or "" - if content and content.strip(): - print() - print(_c(" Assistant:", "green", "bold")) - for line in content.strip().splitlines(): - print(_c(f" {line}", "white")) - - if reason != "tool_calls": - # LLM is done - break - - tool_calls = message.get("tool_calls") or [] - if not tool_calls: - print(_c(" [finish_reason=tool_calls but no tool_calls in message]", "yellow")) - break - - tool_results: List[Dict] = [] - for tc in tool_calls: - fn_name = tc.get("function", {}).get("name", "unknown") - raw_args = tc.get("function", {}).get("arguments", "{}") - try: - fn_args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - except json.JSONDecodeError: - fn_args = {} - - # Pretty-print the call - arg_str = ", ".join(f"{k}={v!r}" for k, v in fn_args.items()) if fn_args else "" - print() - print(_c(f" → {fn_name}({arg_str})", "yellow", "bold")) - - # Handle meta-tools locally; dispatch everything else to REST. - if fn_name == "list_available_tools": - inactive = [ - {"name": n, "description": _TOOL_BY_NAME[n]["function"]["description"]} - for n in _TOOL_BY_NAME - if n not in active_tools - ] - result = {"available": inactive, "count": len(inactive)} - elif fn_name == "request_tools": - requested = fn_args.get("names", []) - added, unknown = [], [] - for name in requested: - if name in _TOOL_BY_NAME and name not in active_tools: - active_tools.add(name) - added.append(name) - elif name not in _TOOL_BY_NAME: - unknown.append(name) - result = { - "ok": True, - "added": added, - "unknown": unknown, - "note": ( - f"Activated {len(added)} tool(s). " - "They are available in your tool list from the next turn." - + (f" Unknown: {unknown}" if unknown else "") - ), - } - if added: - print(_c(f" ↳ activated: {', '.join(added)}", "dim")) - else: - try: - result = dispatch_tool(fn_name, fn_args, rest, default_uid, default_index) - except Exception as e: - result = {"error": str(e)} - - # Print a short summary of the result - _print_tool_result(fn_name, result) - - tool_results.append({ - "role": "tool", - "tool_call_id": tc.get("id", ""), - "content": json.dumps(result), - }) - - # Add all tool results as a batch before the next LLM call - history.extend(tool_results) - - else: - print(_c(f"\n [Reached maximum {MAX_TURNS} turns]", "yellow")) - - return history - - -def _print_tool_result(tool_name: str, result: Any) -> None: - """Print a concise, human-readable summary of a tool result.""" - if isinstance(result, dict): - if result.get("error"): - print(_c(f" ✗ error: {result['error']}", "red")) - return - - if tool_name == "list_windows": - windows = result.get("windows", []) - print(_c(f" ← {len(windows)} window(s):", "dim")) - for w in windows[:8]: - flag = " [FOCUSED]" if w.get("focused") else "" - uid = f" uid={w['window_uid']}" if w.get("window_uid") else "" - print(_c(f" [{w['index']}] {w['title']}{flag}{uid}", "dim")) - if len(windows) > 8: - print(_c(f" … and {len(windows) - 8} more", "dim")) - - elif tool_name in ("observe_window",): - sketch = result.get("sketch", "") - if sketch: - first_lines = sketch.splitlines()[:5] - for ln in first_lines: - print(_c(f" │ {ln}", "dim")) - extra = len(sketch.splitlines()) - 5 - if extra > 0: - print(_c(f" │ … ({extra} more lines)", "dim")) - - elif tool_name == "get_screen_sketch": - sketch = result.get("sketch", "") - lines = sketch.splitlines() - for ln in lines[:6]: - print(_c(f" │ {ln}", "dim")) - if len(lines) > 6: - print(_c(f" │ … ({len(lines) - 6} more lines)", "dim")) - - elif tool_name in ("click_at", "type_text", "press_key", "scroll"): - ok = result.get("success", False) - sym = "✓" if ok else "✗" - color = "green" if ok else "red" - note = f" ({result['note']})" if "note" in result else "" - err = f" — {result['error']}" if "error" in result else "" - print(_c(f" ← {sym} {tool_name}{note}{err}", color)) - - elif tool_name == "get_element_tree": - count = result.get("element_count", "?") - window = result.get("window", "") - print(_c(f" ← {count} elements in '{window}'", "dim")) - - elif tool_name == "get_screen_description": - desc = result.get("description", "") - preview = (desc[:120] + "…") if len(desc) > 120 else desc - print(_c(f" ← {preview}", "dim")) - - else: - # Generic: print first 200 chars of JSON - raw = json.dumps(result) - print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) - else: - raw = str(result) - print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) - -# ─── Display helpers ────────────────────────────────────────────────────────── - -_BANNER = r""" - ___ ____ ____ ___ _ - / _ \/ ___/ ___| ___ _ __ ___ ___ _ __ / _ \| |__ ___ ___ _ ____ _____ _ __ -| | | \___ \___ \ / __| '__/ _ \/ _ \ '_ \| | | | '_ \/ __|/ _ \ '__\ \ / / _ \ '__| -| |_| |___) |__) | (__| | | __/ __/ | | | |_| | |_) \__ \ __/ | \ V / __/ | - \___/|____/____/ \___|_| \___|\___|_| |_|\___/|_.__/|___/\___|_| \_/ \___|_| -""" - -def print_banner(): - print(_c(_BANNER.strip("\n"), "cyan", "bold")) - print(_c(" Window Inspection + LLM Agent • OSScreenObserver REST API\n", "dim")) - - -def print_window_list(windows: List[Dict]) -> None: - print(_c(f"\n {'#':>3} {'Title':<50} {'Process':<20} {'Size'}", "bold")) - print(" " + "─" * 90) - for w in windows: - b = w.get("bounds", {}) - size = f"{b.get('width', 0)}×{b.get('height', 0)}" - focused = _c(" ◀", "green") if w.get("focused") else "" - idx_s = _c(f"{w['index']:>3}", "yellow") - title = w.get("title", "")[:50] - proc = w.get("process", "")[:20] - print(f" {idx_s} {title:<50} {proc:<20} {size}{focused}") - print() - - -def print_window_view(data: Dict) -> None: - """Print the sketch and description for a selected window.""" - window = data.get("window", "?") - sketch = data.get("sketch", "") - desc = data.get("description", "") - - width = max((len(ln) for ln in sketch.splitlines()), default=0) + 4 - width = max(width, 60) - header = f" SKETCH — {window} " - pad = max(0, width - len(header) - 2) - print() - print(_c("┌" + header + "─" * pad + "┐", "blue")) - for line in sketch.splitlines(): - print(_c("│ ", "blue") + line) - print(_c("└" + "─" * (width - 2) + "┘", "blue")) - - if desc: - print() - print(_c(" ACCESSIBILITY DESCRIPTION", "bold")) - print(" " + "─" * 50) - for line in desc.splitlines(): - print(" " + line) - print() - -# ─── Interactive prompts ────────────────────────────────────────────────────── - -def prompt(msg: str, default: str = "", secret: bool = False) -> str: - if default: - display = f"{msg} [{default}]: " - else: - display = f"{msg}: " - if secret: - import getpass - val = getpass.getpass(display) - else: - val = input(display).strip() - return val if val else default - - -def ask_connection() -> Tuple[str, str]: - """Interactively collect OpenWebUI connection parameters (model chosen after fetch).""" - print(_c("\n ── OpenWebUI / LLM Connection ──────────────────────────────\n", "bold")) - base_url = prompt(" OpenWebUI base URL", "http://localhost:3000") - api_key = prompt(" API key (leave blank if none)", secret=True) - return base_url, api_key - - -def pick_model(models: List[str]) -> str: - """Display a numbered menu of *models* and return the chosen model id.""" - default = models[0] - print(_c("\n Available models:\n", "bold")) - for i, m in enumerate(models): - print(_c(f" {i + 1:>3}. ", "yellow") + m) - print() - while True: - raw = input(_c(f" Select model (number or name) [{default}]: ", "bold", "cyan")).strip() - if not raw: - return default - if raw.isdigit(): - idx = int(raw) - 1 - if 0 <= idx < len(models): - return models[idx] - print(_c(f" Please enter a number between 1 and {len(models)}.", "red")) - elif raw in models: - return raw - else: - confirm = input(_c(f" '{raw}' not in list — use it anyway? [y/N] ", "yellow")).strip().lower() - if confirm == "y": - return raw - - -def list_models(llm_base: str, api_key: str) -> Tuple[List[str], Optional[str]]: - """Fetch model list from /api/v1/models. Returns (models, error_message).""" - try: - headers: Dict[str, str] = {} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - req = urllib.request.Request( - llm_base.rstrip("/") + f"{_OWU_PREFIX}/models", - headers=headers, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - data = json.loads(resp.read().decode()) - return [m["id"] for m in data.get("data", [])], None - except Exception as e: - return [], str(e) - -# ─── Main ───────────────────────────────────────────────────────────────────── - -def main() -> None: - parser = argparse.ArgumentParser(description="OSScreenObserver interactive agent") - parser.add_argument("--rest", default="http://127.0.0.1:5001", - help="Base URL of the OSScreenObserver REST server") - args = parser.parse_args() - rest = args.rest - - print_banner() - - # ── 1. REST server ──────────────────────────────────────────────────────── - print(_c(f" REST server: {rest}\n", "dim")) - if not wait_for_server(rest): - print(_c( - "\n Could not reach the REST server. " - "Start it with: python main.py --mode inspect\n", "red" - )) - sys.exit(1) - - # ── 2. LLM connection ───────────────────────────────────────────────────── - llm_base, api_key = ask_connection() - - print(_c(f"\n Checking connection to {llm_base} …", "dim"), end="", flush=True) - models, err = list_models(llm_base, api_key) - if models: - print(_c(f" OK ({len(models)} model(s) available)", "green")) - model = pick_model(models) - elif err: - print(_c(f" failed — {err}", "yellow")) - model = prompt(" Model name", "llama3.2:3b") - else: - print(_c(" connected, but no models found", "yellow")) - model = prompt(" Model name", "llama3.2:3b") - - llm = LLMClient(llm_base, api_key, model) - - # ── 3. Main window-selection loop ───────────────────────────────────────── - conversation: List[Dict] = [{"role": "system", "content": SYSTEM_PROMPT}] - selected_window: Optional[int] = None - selected_uid: Optional[str] = None - - print() - print(_c(" Commands: select window r refresh q quit", "dim")) - - while True: - # Fetch and display window list - try: - wdata = api_list_windows(rest) - except Exception as e: - print(_c(f"\n [Failed to list windows: {e}]", "red")) - input(" Press Enter to retry… ") - continue - - windows = wdata.get("windows", []) - if not windows: - print(_c("\n No windows found. Press Enter to refresh.", "yellow")) - input() - continue - - print() - print(_c(" ── Open Windows ─────────────────────────────────────────────", "bold")) - print_window_list(windows) - - raw = input(_c(" Select window [number / r / q]: ", "bold")).strip().lower() - - if raw in ("q", "quit", "exit"): - print(_c("\n Goodbye.\n", "dim")) - break - - if raw in ("r", "refresh", ""): - continue - - try: - chosen = int(raw) - except ValueError: - print(_c(" Invalid input.", "red")) - continue - - valid_indices = [w["index"] for w in windows] - if chosen not in valid_indices: - print(_c(f" Index {chosen} not in list.", "red")) - continue - - selected_window = chosen - chosen_win = next((w for w in windows if w["index"] == chosen), None) - win_title = chosen_win["title"] if chosen_win else str(chosen) - selected_uid = chosen_win.get("window_uid") if chosen_win else None - - # ── 4. Window inspection sub-loop ──────────────────────────────────── - print(_c(f"\n Loading window [{chosen}] {win_title} …", "dim")) - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed to observe window: {e}]", "red")) - continue - - print_window_view(view) - - print(_c(" Commands: send to LLM v view window r refresh b back q quit", - "dim")) - print() - - while True: - raw2 = input(_c(" Task / command: ", "bold", "cyan")).strip() - - if raw2.lower() in ("q", "quit", "exit"): - print(_c("\n Goodbye.\n", "dim")) - sys.exit(0) - - if raw2.lower() in ("b", "back", ""): - break - - if raw2.lower() in ("v", "view"): - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed: {e}]", "red")) - continue - print_window_view(view) - continue - - if raw2.lower() in ("r", "refresh"): - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed: {e}]", "red")) - continue - print_window_view(view) - continue - - # Treat anything else as a task for the LLM agent - print() - win_label = selected_uid or str(selected_window) - print(_c(f" ── Running agent for task on window [{win_label}] ──────────", - "magenta", "bold")) - conversation = run_agent(llm, rest, raw2, selected_uid, conversation, - default_index=selected_window) - print() - print(_c(" ── Agent finished ───────────────────────────────────────────", - "magenta")) - print(_c(" Type 'v' to view the window's current state, or enter another task.", - "dim")) - print() - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print(_c("\n\n Interrupted.\n", "dim")) diff --git a/window_agent/__init__.py b/window_agent/__init__.py new file mode 100644 index 0000000..ccfa283 --- /dev/null +++ b/window_agent/__init__.py @@ -0,0 +1,102 @@ +""" +window_agent — Interactive window inspection and LLM agent for +OSScreenObserver (package form of the former window_agent.py). + +Usage: + python -m window_agent [--rest http://127.0.0.1:5001] + +At startup you are prompted for: + OpenWebUI base URL (e.g. http://localhost:3000) + OpenWebUI API key + Model name (e.g. llama3.2:3b or mistral) + +The program then: + 1. Polls the REST server until it responds. + 2. Lists all open windows; you choose one. + 3. Displays the ASCII sketch + accessibility description. + 4. Opens an interactive prompt where you can type a task for the LLM. + 5. The LLM runs an agentic loop, calling screen tools (observe, click, + type, press keys, get OCR, etc.) until the task is done or it stops. + +No API credentials are saved to disk. + +P3 decomposition: the implementation now lives in submodules (client, +dispatch, tool_schemas, prompts, loop, cli). This __init__ re-exports +the pre-split surface so `import window_agent` keeps working unchanged. +""" + +from __future__ import annotations + +from window_agent.cli import ( + _BANNER, + ask_connection, + list_models, + main, + pick_model, + print_banner, + print_window_list, + print_window_view, + prompt, +) +from window_agent.client import ( + _NO_COLOR, + _NO_REDIRECT_OPENER, + _OWU_PREFIX, + LLMClient, + _c, + _get, + _NoRedirectHandler, + _post, + _win_params, + api_action, + api_bring_to_foreground, + api_description, + api_element_tree, + api_full_screenshot, + api_list_windows, + api_observe, + api_screenshot, + api_sketch, + api_visible_areas, + wait_for_server, +) +from window_agent.dispatch import dispatch_tool +from window_agent.loop import ( + _LLM_MAX_RETRIES, + _LLM_RETRY_DELAY, + MAX_TURNS, + _print_tool_result, + run_agent, +) +from window_agent.prompts import SYSTEM_PROMPT +from window_agent.tool_schemas import ( + _KEYWORD_GROUPS, + _META_TOOLS, + _TOOL_BY_NAME, + _TOOL_TIER, + SCREEN_TOOLS, + _initial_active_tools, + _tool_defs_for, +) + +__all__ = [ + # cli + "_BANNER", "ask_connection", "list_models", "main", "pick_model", + "print_banner", "print_window_list", "print_window_view", "prompt", + # client + "_NO_COLOR", "_NO_REDIRECT_OPENER", "_OWU_PREFIX", "LLMClient", "_c", + "_get", "_NoRedirectHandler", "_post", "_win_params", "api_action", + "api_bring_to_foreground", "api_description", "api_element_tree", + "api_full_screenshot", "api_list_windows", "api_observe", + "api_screenshot", "api_sketch", "api_visible_areas", "wait_for_server", + # dispatch + "dispatch_tool", + # loop + "_LLM_MAX_RETRIES", "_LLM_RETRY_DELAY", "MAX_TURNS", + "_print_tool_result", "run_agent", + # prompts + "SYSTEM_PROMPT", + # tool schemas + "_KEYWORD_GROUPS", "_META_TOOLS", "_TOOL_BY_NAME", "_TOOL_TIER", + "SCREEN_TOOLS", "_initial_active_tools", "_tool_defs_for", +] diff --git a/window_agent/__main__.py b/window_agent/__main__.py new file mode 100644 index 0000000..6d855ed --- /dev/null +++ b/window_agent/__main__.py @@ -0,0 +1,10 @@ +"""python -m window_agent — interactive agent entry point.""" + +from window_agent.cli import main +from window_agent.client import _c + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print(_c("\n\n Interrupted.\n", "dim")) diff --git a/window_agent/cli.py b/window_agent/cli.py new file mode 100644 index 0000000..476e2dd --- /dev/null +++ b/window_agent/cli.py @@ -0,0 +1,282 @@ +""" +Interactive CLI: banner, window picker, prompts, main(). + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Tuple + +from window_agent.client import ( + _OWU_PREFIX, LLMClient, _c, api_list_windows, api_observe, + wait_for_server, +) +from window_agent.loop import run_agent +from window_agent.prompts import SYSTEM_PROMPT + +# ─── Display helpers ────────────────────────────────────────────────────────── + +_BANNER = r""" + ___ ____ ____ ___ _ + / _ \/ ___/ ___| ___ _ __ ___ ___ _ __ / _ \| |__ ___ ___ _ ____ _____ _ __ +| | | \___ \___ \ / __| '__/ _ \/ _ \ '_ \| | | | '_ \/ __|/ _ \ '__\ \ / / _ \ '__| +| |_| |___) |__) | (__| | | __/ __/ | | | |_| | |_) \__ \ __/ | \ V / __/ | + \___/|____/____/ \___|_| \___|\___|_| |_|\___/|_.__/|___/\___|_| \_/ \___|_| +""" + +def print_banner(): + print(_c(_BANNER.strip("\n"), "cyan", "bold")) + print(_c(" Window Inspection + LLM Agent • OSScreenObserver REST API\n", "dim")) + + +def print_window_list(windows: List[Dict]) -> None: + print(_c(f"\n {'#':>3} {'Title':<50} {'Process':<20} {'Size'}", "bold")) + print(" " + "─" * 90) + for w in windows: + b = w.get("bounds", {}) + size = f"{b.get('width', 0)}×{b.get('height', 0)}" + focused = _c(" ◀", "green") if w.get("focused") else "" + idx_s = _c(f"{w['index']:>3}", "yellow") + title = w.get("title", "")[:50] + proc = w.get("process", "")[:20] + print(f" {idx_s} {title:<50} {proc:<20} {size}{focused}") + print() + + +def print_window_view(data: Dict) -> None: + """Print the sketch and description for a selected window.""" + window = data.get("window", "?") + sketch = data.get("sketch", "") + desc = data.get("description", "") + + width = max((len(ln) for ln in sketch.splitlines()), default=0) + 4 + width = max(width, 60) + header = f" SKETCH — {window} " + pad = max(0, width - len(header) - 2) + print() + print(_c("┌" + header + "─" * pad + "┐", "blue")) + for line in sketch.splitlines(): + print(_c("│ ", "blue") + line) + print(_c("└" + "─" * (width - 2) + "┘", "blue")) + + if desc: + print() + print(_c(" ACCESSIBILITY DESCRIPTION", "bold")) + print(" " + "─" * 50) + for line in desc.splitlines(): + print(" " + line) + print() + +# ─── Interactive prompts ────────────────────────────────────────────────────── + +def prompt(msg: str, default: str = "", secret: bool = False) -> str: + if default: + display = f"{msg} [{default}]: " + else: + display = f"{msg}: " + if secret: + import getpass + val = getpass.getpass(display) + else: + val = input(display).strip() + return val if val else default + + +def ask_connection() -> Tuple[str, str]: + """Interactively collect OpenWebUI connection parameters (model chosen after fetch).""" + print(_c("\n ── OpenWebUI / LLM Connection ──────────────────────────────\n", "bold")) + base_url = prompt(" OpenWebUI base URL", "http://localhost:3000") + api_key = prompt(" API key (leave blank if none)", secret=True) + return base_url, api_key + + +def pick_model(models: List[str]) -> str: + """Display a numbered menu of *models* and return the chosen model id.""" + default = models[0] + print(_c("\n Available models:\n", "bold")) + for i, m in enumerate(models): + print(_c(f" {i + 1:>3}. ", "yellow") + m) + print() + while True: + raw = input(_c(f" Select model (number or name) [{default}]: ", "bold", "cyan")).strip() + if not raw: + return default + if raw.isdigit(): + idx = int(raw) - 1 + if 0 <= idx < len(models): + return models[idx] + print(_c(f" Please enter a number between 1 and {len(models)}.", "red")) + elif raw in models: + return raw + else: + confirm = input(_c(f" '{raw}' not in list — use it anyway? [y/N] ", "yellow")).strip().lower() + if confirm == "y": + return raw + + +def list_models(llm_base: str, api_key: str) -> Tuple[List[str], Optional[str]]: + """Fetch model list from /api/v1/models. Returns (models, error_message).""" + try: + headers: Dict[str, str] = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request( + llm_base.rstrip("/") + f"{_OWU_PREFIX}/models", + headers=headers, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode()) + return [m["id"] for m in data.get("data", [])], None + except Exception as e: + return [], str(e) + +# ─── Main ───────────────────────────────────────────────────────────────────── + +def main() -> None: + parser = argparse.ArgumentParser(description="OSScreenObserver interactive agent") + parser.add_argument("--rest", default="http://127.0.0.1:5001", + help="Base URL of the OSScreenObserver REST server") + args = parser.parse_args() + rest = args.rest + + print_banner() + + # ── 1. REST server ──────────────────────────────────────────────────────── + print(_c(f" REST server: {rest}\n", "dim")) + if not wait_for_server(rest): + print(_c( + "\n Could not reach the REST server. " + "Start it with: python main.py --mode inspect\n", "red" + )) + sys.exit(1) + + # ── 2. LLM connection ───────────────────────────────────────────────────── + llm_base, api_key = ask_connection() + + print(_c(f"\n Checking connection to {llm_base} …", "dim"), end="", flush=True) + models, err = list_models(llm_base, api_key) + if models: + print(_c(f" OK ({len(models)} model(s) available)", "green")) + model = pick_model(models) + elif err: + print(_c(f" failed — {err}", "yellow")) + model = prompt(" Model name", "llama3.2:3b") + else: + print(_c(" connected, but no models found", "yellow")) + model = prompt(" Model name", "llama3.2:3b") + + llm = LLMClient(llm_base, api_key, model) + + # ── 3. Main window-selection loop ───────────────────────────────────────── + conversation: List[Dict] = [{"role": "system", "content": SYSTEM_PROMPT}] + selected_window: Optional[int] = None + selected_uid: Optional[str] = None + + print() + print(_c(" Commands: select window r refresh q quit", "dim")) + + while True: + # Fetch and display window list + try: + wdata = api_list_windows(rest) + except Exception as e: + print(_c(f"\n [Failed to list windows: {e}]", "red")) + input(" Press Enter to retry… ") + continue + + windows = wdata.get("windows", []) + if not windows: + print(_c("\n No windows found. Press Enter to refresh.", "yellow")) + input() + continue + + print() + print(_c(" ── Open Windows ─────────────────────────────────────────────", "bold")) + print_window_list(windows) + + raw = input(_c(" Select window [number / r / q]: ", "bold")).strip().lower() + + if raw in ("q", "quit", "exit"): + print(_c("\n Goodbye.\n", "dim")) + break + + if raw in ("r", "refresh", ""): + continue + + try: + chosen = int(raw) + except ValueError: + print(_c(" Invalid input.", "red")) + continue + + valid_indices = [w["index"] for w in windows] + if chosen not in valid_indices: + print(_c(f" Index {chosen} not in list.", "red")) + continue + + selected_window = chosen + chosen_win = next((w for w in windows if w["index"] == chosen), None) + win_title = chosen_win["title"] if chosen_win else str(chosen) + selected_uid = chosen_win.get("window_uid") if chosen_win else None + + # ── 4. Window inspection sub-loop ──────────────────────────────────── + print(_c(f"\n Loading window [{chosen}] {win_title} …", "dim")) + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed to observe window: {e}]", "red")) + continue + + print_window_view(view) + + print(_c(" Commands: send to LLM v view window r refresh b back q quit", + "dim")) + print() + + while True: + raw2 = input(_c(" Task / command: ", "bold", "cyan")).strip() + + if raw2.lower() in ("q", "quit", "exit"): + print(_c("\n Goodbye.\n", "dim")) + sys.exit(0) + + if raw2.lower() in ("b", "back", ""): + break + + if raw2.lower() in ("v", "view"): + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed: {e}]", "red")) + continue + print_window_view(view) + continue + + if raw2.lower() in ("r", "refresh"): + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed: {e}]", "red")) + continue + print_window_view(view) + continue + + # Treat anything else as a task for the LLM agent + print() + win_label = selected_uid or str(selected_window) + print(_c(f" ── Running agent for task on window [{win_label}] ──────────", + "magenta", "bold")) + conversation = run_agent(llm, rest, raw2, selected_uid, conversation, + default_index=selected_window) + print() + print(_c(" ── Agent finished ───────────────────────────────────────────", + "magenta")) + print(_c(" Type 'v' to view the window's current state, or enter another task.", + "dim")) + print() diff --git a/window_agent/client.py b/window_agent/client.py new file mode 100644 index 0000000..c27d096 --- /dev/null +++ b/window_agent/client.py @@ -0,0 +1,186 @@ +""" +ANSI colour, HTTP and REST/LLM client helpers. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Dict, List, Optional + +# ─── ANSI colour helpers ────────────────────────────────────────────────────── + +_NO_COLOR = not sys.stdout.isatty() + +def _c(text: str, *codes: str) -> str: + if _NO_COLOR: + return text + _MAP = { + "bold": "\033[1m", "dim": "\033[2m", "reset": "\033[0m", + "red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", + "blue": "\033[34m", "magenta": "\033[35m", "cyan": "\033[36m", + "white": "\033[37m", "bright_white": "\033[97m", + } + return "".join(_MAP.get(c, "") for c in codes) + text + "\033[0m" + +# ─── HTTP helpers (stdlib only, no extra deps) ──────────────────────────────── + +def _get(base: str, path: str, params: Optional[Dict] = None, timeout: int = 30) -> Any: + url = base.rstrip("/") + path + if params: + filtered = {k: v for k, v in params.items() if v is not None} + if filtered: + url += "?" + urllib.parse.urlencode(filtered) + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Raise on any redirect so a POST is never silently converted to GET.""" + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.HTTPError(newurl, code, + f"Redirect to {newurl} — update your base URL", + headers, fp) + +_NO_REDIRECT_OPENER = urllib.request.build_opener(_NoRedirectHandler) + + +def _post(base: str, path: str, data: Any, headers: Optional[Dict] = None, + timeout: int = 60) -> Any: + body = json.dumps(data).encode() + req = urllib.request.Request( + base.rstrip("/") + path, + data=body, + headers={"Content-Type": "application/json", **(headers or {})}, + method="POST", + ) + with _NO_REDIRECT_OPENER.open(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + +# ─── REST server poller ─────────────────────────────────────────────────────── + +def wait_for_server(rest_base: str, retries: int = 40, delay: float = 0.5) -> bool: + print(_c(" Waiting for REST server …", "dim"), end="", flush=True) + for _ in range(retries): + try: + _get(rest_base, "/api/windows", timeout=2) + print(_c(" ready.", "green")) + return True + except Exception: + print(".", end="", flush=True) + time.sleep(delay) + print(_c(" timed out.", "red")) + return False + +# ─── REST API wrappers ──────────────────────────────────────────────────────── + +def _win_params(uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict[str, Any]: + """Return the minimal window-selector dict: uid > index > title substring.""" + if uid: + return {"window_uid": uid} + if index is not None: + return {"window_index": index} + if title: + return {"window_title": title} + return {} + + +def api_list_windows(rest: str) -> Dict: + return _get(rest, "/api/windows") + + +def api_observe(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + params = _win_params(uid, index, title) + sketch = _get(rest, "/api/sketch", params) + desc = _get(rest, "/api/description", params) + return { + "window": sketch.get("window", "unknown"), + "sketch": sketch.get("sketch", ""), + "description": desc.get("description", ""), + } + + +def api_element_tree(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/structure", _win_params(uid, index, title)) + + +def api_description(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/description", _win_params(uid, index, title)) + + +def api_sketch(rest: str, uid: Optional[str], index: Optional[int], + grid_width: Optional[int] = None, grid_height: Optional[int] = None, + ocr: bool = False, title: Optional[str] = None) -> Dict: + params: Dict[str, Any] = _win_params(uid, index, title) + if grid_width is not None: + params["grid_width"] = grid_width + if grid_height is not None: + params["grid_height"] = grid_height + if ocr: + params["ocr"] = "1" + return _get(rest, "/api/sketch", params) + + +def api_screenshot(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/screenshot", _win_params(uid, index, title)) + + +def api_full_screenshot(rest: str, uid: Optional[str], index: Optional[int], + grid_width: Optional[int] = None, + grid_height: Optional[int] = None, + title: Optional[str] = None) -> Dict: + params: Dict[str, Any] = _win_params(uid, index, title) + if grid_width is not None: + params["grid_width"] = grid_width + if grid_height is not None: + params["grid_height"] = grid_height + return _get(rest, "/api/full_screenshot", params) + + +def api_visible_areas(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/visible_areas", _win_params(uid, index, title)) + + +def api_bring_to_foreground(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/bring_to_foreground", _win_params(uid, index, title)) + + +def api_action(rest: str, payload: Dict) -> Dict: + return _post(rest, "/api/action", payload) + +# OpenWebUI OpenAI-compatible API prefix (centralised so both endpoints stay in sync) +_OWU_PREFIX = "/api/v1" + +# ─── LLM client (OpenAI-compatible, OpenWebUI) ──────────────────────────────── + +class LLMClient: + def __init__(self, base_url: str, api_key: str, model: str): + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.model = model + + def chat(self, messages: List[Dict], tools: Optional[List[Dict]] = None) -> Dict: + payload: Dict[str, Any] = { + "model": self.model, + "messages": messages, + "temperature": 0.2, + } + if tools: + payload["tools"] = tools + headers = {} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + return _post(self.base_url, f"{_OWU_PREFIX}/chat/completions", payload, headers, timeout=240) diff --git a/window_agent/dispatch.py b/window_agent/dispatch.py new file mode 100644 index 0000000..cad7153 --- /dev/null +++ b/window_agent/dispatch.py @@ -0,0 +1,287 @@ +""" +Tool dispatcher: maps LLM tool names to REST calls. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import urllib.request +from typing import Any, Dict, Optional + +from window_agent.client import ( + _NO_REDIRECT_OPENER, _get, _post, + api_action, api_bring_to_foreground, api_description, + api_element_tree, api_full_screenshot, api_list_windows, + api_observe, api_screenshot, api_sketch, api_visible_areas, +) + +# ─── Tool dispatcher (maps LLM tool names → REST calls) ────────────────────── + +def dispatch_tool(tool_name: str, args: Dict, rest: str, + default_uid: Optional[str] = None, + default_index: Optional[int] = None) -> Any: + """ + Route a tool call from the LLM to the appropriate REST endpoint. + Returns a Python object (will be JSON-serialised before sending back to LLM). + """ + uid: Optional[str] = args.get("window_uid") + wi: Optional[int] = args.get("window_index") if "window_index" in args else None + title: Optional[str] = args.get("window_title") + # Apply defaults only when the LLM specified none of uid / index / title. + if uid is None and wi is None and title is None: + uid = default_uid + wi = default_index + + if tool_name == "list_windows": + return api_list_windows(rest) + + elif tool_name == "observe_window": + return api_observe(rest, uid, wi, title) + + elif tool_name == "get_element_tree": + return api_element_tree(rest, uid, wi, title) + + elif tool_name == "get_screen_description": + return api_description(rest, uid, wi, title) + + elif tool_name == "get_screen_sketch": + raw_ocr = args.get("ocr", False) + if isinstance(raw_ocr, str): + ocr = raw_ocr.strip().lower() not in ("", "false", "0", "no") + else: + ocr = bool(raw_ocr) + return api_sketch(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), ocr=ocr, title=title) + + elif tool_name == "get_screenshot": + result = api_screenshot(rest, uid, wi, title) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + result["note"] = ( + "Screenshot captured (base64 data omitted from tool result). " + "Use get_screen_description for text content (OCR + VLM when available)." + ) + return result + + elif tool_name == "get_full_screenshot": + result = api_full_screenshot(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), title=title) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + note = "Screenshot captured (base64 data omitted)." + if result.get("sketch"): + note += " Sketch included above." + result["note"] = note + return result + + elif tool_name == "get_visible_areas": + if uid is None and wi is None and title is None: + return {"error": "get_visible_areas requires a selected window (window_uid, window_index, or window_title)"} + return api_visible_areas(rest, uid, wi, title) + + elif tool_name == "bring_to_foreground": + if uid is None and wi is None and title is None: + return {"error": "bring_to_foreground requires a selected window (window_uid, window_index, or window_title)"} + return api_bring_to_foreground(rest, uid, wi, title) + + elif tool_name == "click_at": + payload = { + "action": "click_at", + "x": args["x"], + "y": args["y"], + } + if "button" in args: + payload["button"] = args["button"] + if "double" in args: + payload["double"] = args["double"] + return api_action(rest, payload) + + elif tool_name == "type_text": + return api_action(rest, {"action": "type", "value": args["text"]}) + + elif tool_name == "press_key": + return api_action(rest, {"action": "key", "value": args["keys"]}) + + elif tool_name == "scroll": + payload = {"action": "scroll"} + for k in ("x", "y", "clicks"): + if k in args: + payload[k] = args[k] + return api_action(rest, payload) + + # ── P1: identity / discovery ──────────────────────────────────────────── + elif tool_name == "get_capabilities": + return _get(rest, "/api/capabilities") + + elif tool_name == "get_monitors": + return _get(rest, "/api/monitors") + + elif tool_name == "find_element": + params: Dict[str, Any] = {"selector": args["selector"]} + if "window_uid" in args: + params["window_uid"] = args["window_uid"] + elif wi is not None: + params["window_index"] = wi + return _get(rest, "/api/find_element", params) + + # ── P1 / P6: element-targeted actions ─────────────────────────────────── + elif tool_name in ("click_element", "focus_element", "invoke_element", + "set_value", "select_option", + "right_click_element", "double_click_element", + "hover_element", "clear_text", "key_into_element"): + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + path = { + "click_element": "/api/element/click", + "focus_element": "/api/element/focus", + "invoke_element": "/api/element/invoke", + "set_value": "/api/element/set_value", + "select_option": "/api/element/select", + "right_click_element": "/api/element/right_click", + "double_click_element": "/api/element/double_click", + "hover_element": "/api/hover", + "clear_text": "/api/element/clear_text", + "key_into_element": "/api/element/key", + }[tool_name] + return _post(rest, path, body) + + elif tool_name == "hover_at": + return _post(rest, "/api/hover", {k: args[k] for k in args + if k in ("x", "y", "hover_ms")}) + + elif tool_name == "drag": + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, "/api/drag", body) + + # ── P2: synchronisation and observation ───────────────────────────────── + elif tool_name == "wait_for": + body = dict(args) + return _post(rest, "/api/wait_for", body) + + elif tool_name == "wait_idle": + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, "/api/wait_idle", body) + + elif tool_name == "observe_window_diff": + params = {} + if "window_uid" in args: + params["window_uid"] = args["window_uid"] + elif wi is not None: + params["window_index"] = wi + if "since" in args: + params["since"] = args["since"] + if "format" in args: + params["format"] = args["format"] + return _get(rest, "/api/observe", params) + + elif tool_name in ("click_element_and_observe", "type_and_observe", + "press_key_and_observe"): + path = { + "click_element_and_observe": "/api/element/click_and_observe", + "type_and_observe": "/api/type_and_observe", + "press_key_and_observe": "/api/key_and_observe", + }[tool_name] + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, path, body) + + # ── P2: snapshots ─────────────────────────────────────────────────────── + elif tool_name == "snapshot": + return _post(rest, "/api/snapshot", {}) + + elif tool_name == "snapshot_get": + return _get(rest, f"/api/snapshot/{args['snapshot_id']}") + + elif tool_name == "snapshot_diff": + return _post(rest, "/api/snapshot/diff", + {k: args[k] for k in args if k in ("a", "b", "format")}) + + elif tool_name == "snapshot_drop": + sid = args["snapshot_id"] + url = rest.rstrip("/") + f"/api/snapshot/{sid}" + req = urllib.request.Request(url, method="DELETE") + with _NO_REDIRECT_OPENER.open(req, timeout=10) as resp: + return json.loads(resp.read().decode()) + + # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── + elif tool_name == "trace_start": + return _post(rest, "/api/trace/start", + {"label": args.get("label", "")}) + elif tool_name == "trace_stop": + return _post(rest, "/api/trace/stop", {}) + elif tool_name == "trace_status": + return _get(rest, "/api/trace/status") + + elif tool_name == "replay_start": + return _post(rest, "/api/replay/start", + {k: args[k] for k in args + if k in ("path", "mode", "on_divergence")}) + elif tool_name == "replay_step": + return _post(rest, "/api/replay/step", + {"replay_id": args["replay_id"]}) + elif tool_name == "replay_status": + return _post(rest, "/api/replay/status", + {"replay_id": args["replay_id"]}) + elif tool_name == "replay_stop": + return _post(rest, "/api/replay/stop", + {"replay_id": args["replay_id"]}) + + elif tool_name == "load_scenario": + return _post(rest, "/api/scenario/load", {"path": args["path"]}) + + elif tool_name == "assert_state": + return _post(rest, "/api/assert_state", + {"predicate": args.get("predicate", args.get("predicates", []))}) + + # ── P5: safety & status ───────────────────────────────────────────────── + elif tool_name == "get_budget_status": + return _get(rest, "/api/budget_status") + + elif tool_name == "get_redaction_status": + return _get(rest, "/api/redaction_status") + + elif tool_name == "propose_action": + return _post(rest, "/api/propose_action", + {"action": args["action"], "args": args.get("args", {})}) + + # ── P3 extras ─────────────────────────────────────────────────────────── + elif tool_name == "get_screenshot_cropped": + # Pixel data is huge — same omission policy as get_screenshot. + params = {} + if "window_uid" in args: + params["window_uid"] = args["window_uid"] + elif wi is not None: + params["window_index"] = wi + for k in ("element_id", "padding_px", "max_width"): + if k in args: + params[k] = args[k] + result = _get(rest, "/api/screenshot/cropped", params) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + result["note"] = "Screenshot captured (base64 data omitted)." + return result + + elif tool_name == "get_ocr": + params = {} + if "window_uid" in args: + params["window_uid"] = args["window_uid"] + elif wi is not None: + params["window_index"] = wi + if "element_id" in args: + params["element_id"] = args["element_id"] + return _get(rest, "/api/ocr", params) + + # ── Catch-all: drive any registered tool by name ──────────────────────── + elif tool_name == "call_tool": + name = args["name"] + body = args.get("args", {}) or {} + return _post(rest, f"/api/tool/{name}", body) + + else: + return {"error": f"Unknown tool: {tool_name}"} diff --git a/window_agent/loop.py b/window_agent/loop.py new file mode 100644 index 0000000..4e25eda --- /dev/null +++ b/window_agent/loop.py @@ -0,0 +1,245 @@ +""" +Agentic tool-calling loop and tool-result printing. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import time +import traceback +import urllib.error +from typing import Any, Dict, List, Optional + +from window_agent.client import LLMClient, _c +from window_agent.dispatch import dispatch_tool +from window_agent.tool_schemas import ( + SCREEN_TOOLS, _TOOL_BY_NAME, _TOOL_TIER, _initial_active_tools, + _tool_defs_for, +) + +# ─── Agentic loop ───────────────────────────────────────────────────────────── + +MAX_TURNS = 30 +_LLM_MAX_RETRIES = 3 # retries for transient network errors per turn +_LLM_RETRY_DELAY = 5.0 # seconds before first retry (doubles each attempt) + +def run_agent( + llm: LLMClient, + rest: str, + user_task: str, + default_uid: Optional[str], + history: List[Dict], + default_index: Optional[int] = None, +) -> List[Dict]: + """ + Run the agentic tool-calling loop. + + Appends messages to *history* in place and returns the updated history. + Prints progress to stdout using ANSI colours. + """ + history.append({"role": "user", "content": user_task}) + print() + print(_c(f" User: {user_task}", "cyan")) + + active_tools: set = _initial_active_tools(user_task) + n_keyword = len(active_tools) - sum( + 1 for t, tier in _TOOL_TIER.items() if tier in ("always", "usually") + ) + print(_c( + f" [Tools: {len(active_tools)} active" + + (f" (+{n_keyword} from keywords)" if n_keyword > 0 else "") + + f" / {len(SCREEN_TOOLS)} total]", + "dim", + )) + + for turn in range(MAX_TURNS): + tool_defs = _tool_defs_for(active_tools) + resp = None + for attempt in range(_LLM_MAX_RETRIES + 1): + try: + resp = llm.chat(history, tools=tool_defs) + break + except (TimeoutError, ConnectionError, OSError) as e: + if attempt < _LLM_MAX_RETRIES: + delay = _LLM_RETRY_DELAY * (2 ** attempt) + print(_c(f"\n [LLM timeout/connection error — retrying in {delay:.0f}s: {e}]", "yellow")) + time.sleep(delay) + else: + print(_c(f"\n [LLM request failed after {_LLM_MAX_RETRIES + 1} attempts: {e}]", "red")) + return history + except urllib.error.URLError as e: + # URLError wraps OSError/TimeoutError — retry those too. + cause = e.reason if hasattr(e, "reason") else e + if isinstance(cause, (TimeoutError, ConnectionError, OSError)) and attempt < _LLM_MAX_RETRIES: + delay = _LLM_RETRY_DELAY * (2 ** attempt) + print(_c(f"\n [LLM network error — retrying in {delay:.0f}s: {e}]", "yellow")) + time.sleep(delay) + else: + print(_c(f"\n [LLM request failed: {e}]", "red")) + return history + except Exception as e: + print(_c(f"\n [LLM error: {e}]", "red")) + traceback.print_exc() + return history + if resp is None: + return history + + choices = resp.get("choices", []) + if not choices: + print(_c(" [Empty response from LLM]", "red")) + break + + choice = choices[0] + message = choice.get("message", {}) + reason = choice.get("finish_reason", "stop") + + # Accumulate assistant message into history + history.append({"role": "assistant", **{k: v for k, v in message.items() + if k not in ("role",)}}) + + # Print any text content the LLM produced + content = message.get("content") or "" + if content and content.strip(): + print() + print(_c(" Assistant:", "green", "bold")) + for line in content.strip().splitlines(): + print(_c(f" {line}", "white")) + + if reason != "tool_calls": + # LLM is done + break + + tool_calls = message.get("tool_calls") or [] + if not tool_calls: + print(_c(" [finish_reason=tool_calls but no tool_calls in message]", "yellow")) + break + + tool_results: List[Dict] = [] + for tc in tool_calls: + fn_name = tc.get("function", {}).get("name", "unknown") + raw_args = tc.get("function", {}).get("arguments", "{}") + try: + fn_args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + except json.JSONDecodeError: + fn_args = {} + + # Pretty-print the call + arg_str = ", ".join(f"{k}={v!r}" for k, v in fn_args.items()) if fn_args else "" + print() + print(_c(f" → {fn_name}({arg_str})", "yellow", "bold")) + + # Handle meta-tools locally; dispatch everything else to REST. + if fn_name == "list_available_tools": + inactive = [ + {"name": n, "description": _TOOL_BY_NAME[n]["function"]["description"]} + for n in _TOOL_BY_NAME + if n not in active_tools + ] + result = {"available": inactive, "count": len(inactive)} + elif fn_name == "request_tools": + requested = fn_args.get("names", []) + added, unknown = [], [] + for name in requested: + if name in _TOOL_BY_NAME and name not in active_tools: + active_tools.add(name) + added.append(name) + elif name not in _TOOL_BY_NAME: + unknown.append(name) + result = { + "ok": True, + "added": added, + "unknown": unknown, + "note": ( + f"Activated {len(added)} tool(s). " + "They are available in your tool list from the next turn." + + (f" Unknown: {unknown}" if unknown else "") + ), + } + if added: + print(_c(f" ↳ activated: {', '.join(added)}", "dim")) + else: + try: + result = dispatch_tool(fn_name, fn_args, rest, default_uid, default_index) + except Exception as e: + result = {"error": str(e)} + + # Print a short summary of the result + _print_tool_result(fn_name, result) + + tool_results.append({ + "role": "tool", + "tool_call_id": tc.get("id", ""), + "content": json.dumps(result), + }) + + # Add all tool results as a batch before the next LLM call + history.extend(tool_results) + + else: + print(_c(f"\n [Reached maximum {MAX_TURNS} turns]", "yellow")) + + return history + + +def _print_tool_result(tool_name: str, result: Any) -> None: + """Print a concise, human-readable summary of a tool result.""" + if isinstance(result, dict): + if result.get("error"): + print(_c(f" ✗ error: {result['error']}", "red")) + return + + if tool_name == "list_windows": + windows = result.get("windows", []) + print(_c(f" ← {len(windows)} window(s):", "dim")) + for w in windows[:8]: + flag = " [FOCUSED]" if w.get("focused") else "" + uid = f" uid={w['window_uid']}" if w.get("window_uid") else "" + print(_c(f" [{w['index']}] {w['title']}{flag}{uid}", "dim")) + if len(windows) > 8: + print(_c(f" … and {len(windows) - 8} more", "dim")) + + elif tool_name in ("observe_window",): + sketch = result.get("sketch", "") + if sketch: + first_lines = sketch.splitlines()[:5] + for ln in first_lines: + print(_c(f" │ {ln}", "dim")) + extra = len(sketch.splitlines()) - 5 + if extra > 0: + print(_c(f" │ … ({extra} more lines)", "dim")) + + elif tool_name == "get_screen_sketch": + sketch = result.get("sketch", "") + lines = sketch.splitlines() + for ln in lines[:6]: + print(_c(f" │ {ln}", "dim")) + if len(lines) > 6: + print(_c(f" │ … ({len(lines) - 6} more lines)", "dim")) + + elif tool_name in ("click_at", "type_text", "press_key", "scroll"): + ok = result.get("success", False) + sym = "✓" if ok else "✗" + color = "green" if ok else "red" + note = f" ({result['note']})" if "note" in result else "" + err = f" — {result['error']}" if "error" in result else "" + print(_c(f" ← {sym} {tool_name}{note}{err}", color)) + + elif tool_name == "get_element_tree": + count = result.get("element_count", "?") + window = result.get("window", "") + print(_c(f" ← {count} elements in '{window}'", "dim")) + + elif tool_name == "get_screen_description": + desc = result.get("description", "") + preview = (desc[:120] + "…") if len(desc) > 120 else desc + print(_c(f" ← {preview}", "dim")) + + else: + # Generic: print first 200 chars of JSON + raw = json.dumps(result) + print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) + else: + raw = str(result) + print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) diff --git a/window_agent/prompts.py b/window_agent/prompts.py new file mode 100644 index 0000000..99aa801 --- /dev/null +++ b/window_agent/prompts.py @@ -0,0 +1,66 @@ +""" +System prompt for the GUI-automation agent loop. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +SYSTEM_PROMPT = """\ +You are a GUI automation agent operating on a live desktop. +You observe screen state through accessibility tools and execute mouse and keyboard actions. + +COORDINATE RULE +All x, y values must come from get_element_tree bounds — never estimate or recall coordinates. +To click the centre of an element with bounds {x, y, width, height}: + click_x = x + width // 2 + click_y = y + height // 2 + +OBSERVATION RULE +You are blind to the screen unless you call observe_window or get_screen_description. +Call observe_window before deciding where to act, and after every action to confirm the result. +Always read the window title from the observe_window result and confirm it matches the window +you intend to act on before proceeding. + +FINDING ELEMENTS — IMPORTANT +The accessibility tree may be incomplete for web pages and some applications. +If a selector or element search fails with NOT FOUND: + 1. Call get_screen_description to get accessibility + OCR + visual text all at once. + 2. Use get_screenshot and inspect the image to identify element positions visually. + 3. Fall back to click_at with coordinates derived from element bounds in the sketch/screenshot. +Do not give up after one NOT FOUND — always try the screenshot/OCR path before reporting failure. + +WINDOW INDEX INSTABILITY — CRITICAL +window_index values change every time a window is raised, minimised, or closed. +• Every window tool call returns window_uid in its response — capture it immediately and use it + on all subsequent calls for that window instead of window_index. +• When you must call by window_index, the server auto-resolves it to the uid and returns + window_uid in the result — read that value and switch to uid= from that point on. +• Never assume the same index still refers to the same window between tool calls. + +BROWSER TAB SWITCHING +Browser tab bars appear as TabItem elements in the accessibility tree. +To switch to a different tab: observe_window, then click_element on the correct TabItem. +The window title updates to reflect the active tab after the click. + +TASK COMPLETION +Complete every part of the user's task before stopping. +Do not ask for clarification or next steps mid-task when the task is unambiguous. +Only report done when all sub-tasks are finished. + +WORKFLOW +1. list_windows — note window_uid for the target window; use uid on all future calls +2. bring_to_foreground(window_uid=…) — raise the window (result includes window_uid if you used index) +3. observe_window(window_uid=…) — verify window title matches; understand current state +4. get_element_tree(window_uid=…) — get exact coordinates when needed +5. Execute one action +6. observe_window — verify title still matches and the change occurred +7. Repeat until ALL sub-tasks are complete + +TOOL AVAILABILITY +Only a subset of tools is active at session start to keep context short. + • list_available_tools() — see what else exists + • request_tools(names=[…]) — activate specific tools for this session + +If an action does not produce the expected result, re-observe and try an alternative approach. +""" diff --git a/window_agent/tool_schemas.py b/window_agent/tool_schemas.py new file mode 100644 index 0000000..582b873 --- /dev/null +++ b/window_agent/tool_schemas.py @@ -0,0 +1,963 @@ +""" +LLM tool catalogue: schemas, tiers, keyword groups and +meta-tools for on-demand activation. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +from typing import Dict, List + +# ─── Tool definitions (OpenAI / OpenWebUI format) ───────────────────────────── + +# ─── Full tool catalogue (trimmed descriptions) ────────────────────────────── +# Each entry is the schema sent to the LLM. Tiers and keyword groups are in +# _TOOL_TIER / _KEYWORD_GROUPS below — not in the dict itself so the JSON +# payload stays clean. + +SCREEN_TOOLS: List[Dict] = [ + # ── Core observation ───────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "list_windows", + "description": "List visible desktop windows. Returns index, title, process, PID, geometry, and window_uid. Prefer window_uid over index — index changes after every focus change.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "observe_window", + "description": "ASCII sketch + accessibility description of a window. Call before and after every action.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer", "description": "From list_windows. Omit for focused window."}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_element_tree", + "description": "Accessibility tree as JSON. Each node has id, name, role, value, enabled, focused, bounds {x,y,width,height}.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screen_description", + "description": "Describe a window using all available sources: accessibility tree, OCR, and VLM (when enabled). Returns everything available in one call.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screen_sketch", + "description": "ASCII spatial layout of a window. ocr=true overlays Tesseract text.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "grid_width": {"type": "integer"}, + "grid_height": {"type": "integer"}, + "ocr": {"type": "boolean"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screenshot", + "description": "Capture a window screenshot. Pixel data omitted; use get_screen_description mode=ocr for text.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_full_screenshot", + "description": "Full-display screenshot + ASCII sketch with OCR for a window.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "grid_width": {"type": "integer"}, + "grid_height": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_visible_areas", + "description": "Non-occluded bounding boxes {x,y,width,height} for a window.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": ["window_index"], + }, + }, + }, + + # ── Core actions ───────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "click_at", + "description": "Click at absolute screen coordinates. Derive from get_element_tree: cx=x+w//2, cy=y+h//2.", + "parameters": { + "type": "object", + "properties": { + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "button": {"type": "string", "enum": ["left", "right", "middle"]}, + "double": {"type": "boolean"}, + }, + "required": ["x", "y"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "type_text", + "description": "Type text into the focused element. Click the field first.", + "parameters": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "press_key", + "description": "Press a key or chord. Examples: enter, tab, escape, ctrl+s, alt+f4.", + "parameters": { + "type": "object", + "properties": {"keys": {"type": "string"}}, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "scroll", + "description": "Scroll at (x,y). clicks>0 scrolls up, clicks<0 scrolls down.", + "parameters": { + "type": "object", + "properties": { + "clicks": {"type": "integer"}, + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "bring_to_foreground", + "description": "Raise a window to the foreground by clicking its title bar. Window indices change after this call — always call list_windows again before using any index.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": ["window_index"], + }, + }, + }, + + # ── Element-targeted actions ────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "find_element", + "description": 'Resolve selector to element_id + bounds. XPath: Window/Pane/Button[name="OK"]. CSS: Window > Button.', + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": ["selector"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "click_element", + "description": "Click by selector or element_id. Returns ActionReceipt (changed, new_dialogs, before/after hashes).", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "button": {"type": "string", "enum": ["left", "right", "middle"]}, + "count": {"type": "integer"}, + "dry_run": {"type": "boolean"}, + "confirm_token": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "click_element_and_observe", + "description": "Click element then return post-click observation diff.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "button": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "type_and_observe", + "description": "type_text + observe_window in one round-trip.", + "parameters": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + "window_uid": {"type": "string"}, + }, + "required": ["text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "press_key_and_observe", + "description": "press_key + observe_window in one round-trip.", + "parameters": { + "type": "object", + "properties": { + "keys": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + "window_uid": {"type": "string"}, + }, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "right_click_element", + "description": "Right-click an element by selector or element_id.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "double_click_element", + "description": "Double-click an element by selector or element_id.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "select_option", + "description": "Open a combo-box / list and select an option by name or zero-based index.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "option_name": {"type": "string"}, + "option_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "set_value", + "description": "Click an editable element, optionally clear it, then type a value.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "value": {"type": "string"}, + "clear_first": {"type": "boolean"}, + }, + "required": ["value"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "focus_element", + "description": "Give keyboard focus to an element.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "invoke_element", + "description": "Invoke an element's primary action (UIA InvokePattern or click fallback).", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "clear_text", + "description": "Click an editable element then select-all + delete.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "key_into_element", + "description": "Click an element then press a key combination atomically.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "keys": {"type": "string"}, + }, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "hover_at", + "description": "Move mouse to (x, y) and hold hover_ms ms.", + "parameters": { + "type": "object", + "properties": { + "x": {"type": "integer"}, "y": {"type": "integer"}, + "hover_ms": {"type": "integer"}, + }, + "required": ["x", "y"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "hover_element", + "description": "Move mouse to an element's centre and hold hover_ms ms.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "hover_ms": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "drag", + "description": "Drag from one point/element to another. Each endpoint: {x,y} or {selector} or {element_id}.", + "parameters": { + "type": "object", + "properties": { + "from": {"type": "object"}, + "to": {"type": "object"}, + "modifiers": {"type": "array", "items": {"type": "string"}}, + "duration_s": {"type": "number"}, + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + }, + "required": ["from", "to"], + }, + }, + }, + + # ── Synchronisation ─────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "wait_for", + "description": "Block until a condition matches or timeout. Conditions: element_appears, element_disappears, text_visible, window_appears, tree_changes.", + "parameters": { + "type": "object", + "properties": { + "any_of": {"type": "array"}, + "window_uid": {"type": "string"}, + "timeout_ms": {"type": "integer"}, + "poll_ms": {"type": "integer"}, + }, + "required": ["any_of"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "wait_idle", + "description": "Block until the accessibility tree is stable for quiet_ms.", + "parameters": { + "type": "object", + "properties": { + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "quiet_ms": {"type": "integer"}, + "timeout_ms": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "observe_window_diff", + "description": "observe_window but returns only the diff since a tree_token.", + "parameters": { + "type": "object", + "properties": { + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "since": {"type": "string"}, + "format": {"type": "string", "enum": ["custom", "json-patch"]}, + }, + "required": [], + }, + }, + }, + + # ── Snapshots ───────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "snapshot", + "description": "Capture all windows + trees. Returns snapshot_id.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_get", + "description": "Retrieve a saved snapshot.", + "parameters": { + "type": "object", + "properties": {"snapshot_id": {"type": "string"}}, + "required": ["snapshot_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_diff", + "description": "Diff two snapshots (added/removed windows + per-window tree diff).", + "parameters": { + "type": "object", + "properties": { + "a": {"type": "string"}, "b": {"type": "string"}, + "format": {"type": "string", "enum": ["custom", "json-patch"]}, + }, + "required": ["a", "b"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_drop", + "description": "Free a snapshot before its TTL expires.", + "parameters": { + "type": "object", + "properties": {"snapshot_id": {"type": "string"}}, + "required": ["snapshot_id"], + }, + }, + }, + + # ── Tracing / replay ────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "trace_start", + "description": "Start recording tool calls to a JSONL trace file.", + "parameters": { + "type": "object", + "properties": {"label": {"type": "string"}}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "trace_stop", + "description": "Close the active trace.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "trace_status", + "description": "Report whether a trace is active.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "replay_start", + "description": "Load a trace for replay (execute or verify mode).", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "mode": {"type": "string", "enum": ["execute", "verify"]}, + "on_divergence": {"type": "string", "enum": ["stop", "warn", "resume"]}, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_step", + "description": "Advance one row of the active replay.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_status", + "description": "Report replay position, total, finished, divergences.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_stop", + "description": "Free a replay handle.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + + # ── Testing / harness ───────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "load_scenario", + "description": "Load a YAML scenario file into the mock adapter.", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "assert_state", + "description": "Evaluate predicates against current state. Returns all_passed. Predicates: element_exists, value_equals, text_visible, window_focused, screenshot_similar, …", + "parameters": { + "type": "object", + "properties": { + "predicate": {"type": "array"}, + "predicates": {"type": "array"}, + }, + "required": [], + }, + }, + }, + + # ── Safety / budget ─────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_budget_status", + "description": "Remaining budget: actions, screenshots, vlm_tokens, …", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "get_redaction_status", + "description": "Redaction enabled state and applied count.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "propose_action", + "description": "Issue a confirm_token for a destructive action. Pass the token back to the action to proceed.", + "parameters": { + "type": "object", + "properties": { + "action": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["action"], + }, + }, + }, + + # ── Discovery ───────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_capabilities", + "description": "Features the server supports on this host.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "get_monitors", + "description": "Monitors with bounds and scale factor.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + + # ── OCR extras ──────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_screenshot_cropped", + "description": "Cropped screenshot around an element or bbox, with optional max_width downscale.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + "element_id": {"type": "string"}, + "padding_px": {"type": "integer"}, + "max_width": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_ocr", + "description": "Region-scoped OCR. Returns [{text, confidence, bbox}].", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + "element_id": {"type": "string"}, + }, + "required": [], + }, + }, + }, + + # ── Escape hatch ────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "call_tool", + "description": "Call any server tool by name with arbitrary JSON args.", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["name"], + }, + }, + }, +] + +# ─── Tool index, tiers, and keyword groups ──────────────────────────────────── + +_TOOL_BY_NAME: Dict[str, Dict] = { + t["function"]["name"]: t for t in SCREEN_TOOLS +} + +# "always" — sent every turn regardless of task +# "usually" — sent by default (covers the vast majority of tasks) +# "on_demand"— omitted unless the task keywords or request_tools activates them +_TOOL_TIER: Dict[str, str] = { + # always + "list_windows": "always", + "observe_window": "always", + "get_element_tree": "always", + "find_element": "always", + "click_element": "always", + "type_text": "always", + "press_key": "always", + "scroll": "always", + "click_element_and_observe": "always", + # usually + "bring_to_foreground": "usually", + "right_click_element": "usually", + "double_click_element": "usually", + "select_option": "usually", + "set_value": "usually", + "wait_idle": "usually", + "type_and_observe": "usually", + "press_key_and_observe": "usually", + "get_screen_description": "usually", + # on_demand (everything else) + "get_screen_sketch": "on_demand", + "get_screenshot": "on_demand", + "get_full_screenshot": "on_demand", + "get_visible_areas": "on_demand", + "get_screenshot_cropped": "on_demand", + "get_ocr": "on_demand", + "hover_at": "on_demand", + "hover_element": "on_demand", + "drag": "on_demand", + "key_into_element": "on_demand", + "clear_text": "on_demand", + "focus_element": "on_demand", + "invoke_element": "on_demand", + "click_at": "on_demand", + "wait_for": "on_demand", + "observe_window_diff": "on_demand", + "snapshot": "on_demand", + "snapshot_get": "on_demand", + "snapshot_diff": "on_demand", + "snapshot_drop": "on_demand", + "trace_start": "on_demand", + "trace_stop": "on_demand", + "trace_status": "on_demand", + "replay_start": "on_demand", + "replay_step": "on_demand", + "replay_status": "on_demand", + "replay_stop": "on_demand", + "load_scenario": "on_demand", + "assert_state": "on_demand", + "get_budget_status": "on_demand", + "get_redaction_status": "on_demand", + "propose_action": "on_demand", + "get_capabilities": "on_demand", + "get_monitors": "on_demand", + "call_tool": "on_demand", +} + +# Task keywords that unlock specific on_demand tools without a request_tools call. +_KEYWORD_GROUPS: Dict[str, List[str]] = { + "drag": ["drag"], + "drop": ["drag"], + "move": ["drag"], + "hover": ["hover_at", "hover_element"], + "tooltip": ["hover_at", "hover_element"], + "screenshot": ["get_full_screenshot", "get_screenshot"], + "capture": ["get_full_screenshot"], + "sketch": ["get_screen_sketch"], + "ocr": ["get_ocr", "get_screen_description"], + "read text": ["get_ocr"], + "cropped": ["get_screenshot_cropped"], + "visible area": ["get_visible_areas"], + "occluded": ["get_visible_areas"], + "snapshot": ["snapshot", "snapshot_get", "snapshot_diff", "snapshot_drop"], + "compare": ["snapshot", "snapshot_diff"], + "diff": ["observe_window_diff", "snapshot_diff"], + "wait for": ["wait_for"], + "appear": ["wait_for"], + "loading": ["wait_for"], + "coordinates": ["click_at"], + "pixel": ["click_at"], + "clear": ["clear_text"], + "erase": ["clear_text"], + "focus": ["focus_element"], + "invoke": ["invoke_element"], + "trace": ["trace_start", "trace_stop", "trace_status"], + "record": ["trace_start", "trace_stop", "trace_status"], + "replay": ["replay_start", "replay_step", "replay_status", "replay_stop"], + "playback": ["replay_start", "replay_step", "replay_status", "replay_stop"], + "assert": ["assert_state"], + "verify state": ["assert_state"], + "budget": ["get_budget_status"], + "redact": ["get_redaction_status"], + "capabilities": ["get_capabilities"], + "monitors": ["get_monitors"], +} + +# Meta-tools always included — handled locally in the agent loop, never sent to REST. +_META_TOOLS: List[Dict] = [ + { + "type": "function", + "function": { + "name": "list_available_tools", + "description": ( + "List tools not yet active in this session, with one-line descriptions. " + "Call request_tools with the names you need to activate them." + ), + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "request_tools", + "description": ( + "Activate additional tools by name for the rest of this session. " + "Use list_available_tools first to see what is available." + ), + "parameters": { + "type": "object", + "properties": { + "names": { + "type": "array", + "items": {"type": "string"}, + "description": "Tool names to activate.", + } + }, + "required": ["names"], + }, + }, + }, +] + + +def _initial_active_tools(task: str) -> set: + """Return the set of tool names to send on turn 1 based on tier + keywords.""" + active = {name for name, tier in _TOOL_TIER.items() if tier in ("always", "usually")} + task_lower = task.lower() + for kw, names in _KEYWORD_GROUPS.items(): + if kw in task_lower: + active.update(names) + return active + + +def _tool_defs_for(active: set) -> List[Dict]: + """Build the tools list to send to the LLM from the active name set.""" + defs = [_TOOL_BY_NAME[n] for n in active if n in _TOOL_BY_NAME] + return defs + _META_TOOLS From 52bd58dbb8edf63f67273606fc6d44a6dedf7215 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 20:15:32 +0000 Subject: [PATCH 13/14] [P3] module size guard in CI scripts/check_module_size.py fails when any git-tracked non-test .py exceeds the design doc's ~600-line module cap, with an explicit, justified allowlist for intentional remainders: web_inspector/assets.py inline SPA HTML template (data, not logic) window_agent/tool_schemas.py declarative LLM tool catalogue mcp_server/tool_schemas.py declarative MCP tools/list payload ascii_renderer.py single cohesive renderer (pre-existing) description.py single cohesive generator (pre-existing) observer/adapters/windows.py single cohesive UIA adapter (640 lines) The guard also flags stale allowlist entries (files that shrank under the cap or were deleted) so exceptions cannot silently accumulate. Wired into ci.yml right after ruff. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- .github/workflows/ci.yml | 2 + scripts/check_module_size.py | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 scripts/check_module_size.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6f0260..1fc08d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,8 @@ jobs: pip install -r requirements.txt -r requirements-dev.txt - name: Lint with ruff run: ruff check . --exclude tests + - name: Module size guard (design doc ~600 LOC cap) + run: python scripts/check_module_size.py - name: Type-check with mypy run: mypy . - name: Run regression tests (with coverage floor) diff --git a/scripts/check_module_size.py b/scripts/check_module_size.py new file mode 100644 index 0000000..29357b9 --- /dev/null +++ b/scripts/check_module_size.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +check_module_size.py — CI guard against god-files creeping back. + +The design doc (agentic_features_design.md) caps modules at ~600 LOC: +"Keep modules under ~600 LOC; split when growing past that." After the +P3 decomposition every tracked non-test source module fits under that +cap except a small set of intentional remainders listed in ALLOWLIST. + +Fails (exit 1) when any git-tracked non-test .py file exceeds MAX_LINES +and is not allowlisted, and also when an allowlisted file shrinks back +under the cap (so stale allowlist entries are removed rather than +becoming silent escape hatches). +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +MAX_LINES = 600 + +# Intentional remainders. Every entry needs a justification; new code +# modules must be split instead of added here. +ALLOWLIST: dict[str, str] = { + # Declarative data blobs, not logic: one schema/asset entry per line + # would gain nothing from splitting across files. + "web_inspector/assets.py": + "single inline HTML/CSS/JS template string for the SPA UI", + "window_agent/tool_schemas.py": + "declarative LLM tool catalogue (schemas + tiers + keyword groups)", + "mcp_server/tool_schemas.py": + "declarative MCP tools/list schema payload (_TOOLS)", + # Cohesive single-responsibility modules slightly over the cap; + # splitting them would separate tightly coupled halves. + "ascii_renderer.py": + "one renderer: grid projection + glyph rules + OCR overlay share state", + "description.py": + "one generator: accessibility/OCR/VLM description passes share helpers", + "observer/adapters/windows.py": + "one adapter: UIA COM walker + pywinauto walk + tree synthesis", +} + + +def tracked_py_files(root: Path) -> list[str]: + out = subprocess.run( + ["git", "ls-files", "*.py"], + cwd=root, capture_output=True, text=True, check=True, + ).stdout + return [p for p in out.splitlines() + if p and not p.startswith("tests/")] + + +def main() -> int: + root = Path(__file__).resolve().parent.parent + failures: list[str] = [] + stale: list[str] = [] + seen_allowlisted: set[str] = set() + + for rel in tracked_py_files(root): + path = root / rel + try: + n = sum(1 for _ in path.open("rb")) + except OSError as e: + failures.append(f"{rel}: unreadable ({e})") + continue + if rel in ALLOWLIST: + seen_allowlisted.add(rel) + if n <= MAX_LINES: + stale.append(f"{rel}: {n} lines — now under the cap; " + f"remove its ALLOWLIST entry") + continue + if n > MAX_LINES: + failures.append( + f"{rel}: {n} lines exceeds the {MAX_LINES}-line module cap " + f"(design doc: split modules growing past ~600 LOC)") + + for rel in sorted(set(ALLOWLIST) - seen_allowlisted): + stale.append(f"{rel}: allowlisted but not tracked — remove the entry") + + for msg in failures + stale: + print(f"check_module_size: {msg}", file=sys.stderr) + if failures or stale: + return 1 + print(f"check_module_size: OK — all tracked non-test modules are " + f"<= {MAX_LINES} lines ({len(ALLOWLIST)} allowlisted exceptions)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 879211371c8fac8e588f5ad479a3f9a8c0991ec3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 5 Jul 2026 20:53:35 +0000 Subject: [PATCH 14/14] Address Copilot review: fix window-selector forwarding, wait_for smoke args, sanitize signature - window_agent/dispatch.py: route find_element, observe_window_diff, get_screenshot_cropped, and get_ocr through the shared _win_params helper so resolved window_title / default selection is honored and a null window_uid key no longer suppresses index/title selectors. Also forward the bbox param to the cropped-screenshot and OCR REST routes. - tests/test_surface_parity.py: update SMOKE_ARGS[wait_for] from the stale predicate/kind shape to the current any_of/type contract (window_appears). - redaction.py: annotate sanitize_screen_text as Optional[str] -> Optional[str] to reflect the documented non-str passthrough behavior. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012jUkqbwroQmYRAdFDsp5r9 --- redaction.py | 4 ++-- tests/test_surface_parity.py | 4 ++-- window_agent/dispatch.py | 37 +++++++++++++----------------------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/redaction.py b/redaction.py index 3976c8a..9270497 100644 --- a/redaction.py +++ b/redaction.py @@ -19,7 +19,7 @@ import io import logging import re -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ }) -def sanitize_screen_text(text: str) -> str: +def sanitize_screen_text(text: Optional[str]) -> Optional[str]: """Strip ANSI escape sequences and non-whitespace control characters from screen-extracted text. Idempotent; returns non-str input as-is.""" if not isinstance(text, str) or not text: diff --git a/tests/test_surface_parity.py b/tests/test_surface_parity.py index 1b462c1..d3ed13d 100644 --- a/tests/test_surface_parity.py +++ b/tests/test_surface_parity.py @@ -102,8 +102,8 @@ # Minimal args so no tool blocks (waits) or errors on missing input in a # way unrelated to reachability. SMOKE_ARGS = { - "wait_for": {"timeout_ms": 20, "predicate": {"kind": "window_exists", - "title_regex": "."}}, + "wait_for": {"timeout_ms": 20, + "any_of": [{"type": "window_appears", "title_regex": "."}]}, "wait_idle": {"timeout_ms": 20, "quiet_ms": 5}, "hover_at": {"hover_ms": 1}, "hover_element": {"hover_ms": 1, diff --git a/window_agent/dispatch.py b/window_agent/dispatch.py index cad7153..68ce05c 100644 --- a/window_agent/dispatch.py +++ b/window_agent/dispatch.py @@ -11,7 +11,7 @@ from typing import Any, Dict, Optional from window_agent.client import ( - _NO_REDIRECT_OPENER, _get, _post, + _NO_REDIRECT_OPENER, _get, _post, _win_params, api_action, api_bring_to_foreground, api_description, api_element_tree, api_full_screenshot, api_list_windows, api_observe, api_screenshot, api_sketch, api_visible_areas, @@ -117,11 +117,11 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, return _get(rest, "/api/monitors") elif tool_name == "find_element": - params: Dict[str, Any] = {"selector": args["selector"]} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi + # Honor the resolved uid/index/title (with defaults applied above); + # _win_params drops null/empty selectors so a stray window_uid=None + # key from the LLM doesn't suppress index/title selection. + params: Dict[str, Any] = {"selector": args["selector"], + **_win_params(uid, wi, title)} return _get(rest, "/api/find_element", params) # ── P1 / P6: element-targeted actions ─────────────────────────────────── @@ -168,11 +168,7 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, return _post(rest, "/api/wait_idle", body) elif tool_name == "observe_window_diff": - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi + params = dict(_win_params(uid, wi, title)) if "since" in args: params["since"] = args["since"] if "format" in args: @@ -253,12 +249,8 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, # ── P3 extras ─────────────────────────────────────────────────────────── elif tool_name == "get_screenshot_cropped": # Pixel data is huge — same omission policy as get_screenshot. - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - for k in ("element_id", "padding_px", "max_width"): + params = dict(_win_params(uid, wi, title)) + for k in ("element_id", "padding_px", "max_width", "bbox"): if k in args: params[k] = args[k] result = _get(rest, "/api/screenshot/cropped", params) @@ -268,13 +260,10 @@ def dispatch_tool(tool_name: str, args: Dict, rest: str, return result elif tool_name == "get_ocr": - params = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - if "element_id" in args: - params["element_id"] = args["element_id"] + params = dict(_win_params(uid, wi, title)) + for k in ("element_id", "bbox"): + if k in args: + params[k] = args[k] return _get(rest, "/api/ocr", params) # ── Catch-all: drive any registered tool by name ────────────────────────