diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9b9c35..1fc08d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,16 @@ jobs: pip install -r requirements.txt -r requirements-dev.txt - name: Lint with ruff run: ruff check . --exclude tests - - name: Run regression tests - run: pytest tests/ -q -m "not user" --ignore=tests/user + - name: Module size guard (design doc ~600 LOC cap) + run: python scripts/check_module_size.py + - name: Type-check with mypy + run: mypy . + - name: Run regression tests (with coverage floor) + # Coverage floor: measured 56% source coverage on this lane + # (tests/ omitted via pyproject [tool.coverage.run]); floor set + # 3 points below so incidental drift doesn't break CI while real + # regressions in test breadth do. + run: pytest tests/ -q -m "not user" --ignore=tests/user --cov --cov-fail-under=53 - name: Run user tests (when display + tesseract present) run: pytest tests/user/ -q -m "user and not slow_llm and not slow_vlm and not needs_display" diff --git a/.gitignore b/.gitignore index fe3c460..b6724cc 100755 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,9 @@ config.json # Test harness outputs test-results/ .pytest_cache/ + +# Coverage artifacts (pytest-cov) +.coverage +.coverage.* +htmlcov/ +.mypy_cache/ diff --git a/README.md b/README.md index c0d278f..144e207 100644 --- a/README.md +++ b/README.md @@ -17,29 +17,37 @@ OSScreenObserver exposes a full REST API on port `5001` (configurable). Most `/a > ### Security & Network Bind Defaults > -> **The default bind host is now `0.0.0.0` (all network interfaces) on port `5001`.** -> This default is intended for **testing inside an isolated sandbox or container** (e.g., a disposable VM/dev container) where exposing the API to the container network is convenient and safe. +> **The default bind host is `127.0.0.1` (loopback-only) on port `5001`.** +> The REST API has **no authentication** — any client that can reach the port can call every endpoint, including `/api/action`, which can click, type, and otherwise control the host desktop. Binding to loopback by default means only processes on the same machine can reach it. > -> **The REST API has NO authentication.** Any client that can reach the port can call every endpoint, including `/api/action`, which can click, type, and otherwise control the host desktop. -> -> **For local-only use on a workstation, override the bind to loopback:** +> **Opt-in network exposure (Docker, isolated test VMs):** pass the bind host explicitly on the command line or in config — this is the deliberate opt-in path, and startup prints a prominent warning whenever the bind is non-loopback ("unauthenticated action API exposed to the network"): > > ```bash -> # Command-line override (recommended for local dev) -> python main.py --mode both --host 127.0.0.1 +> # Explicit opt-in — e.g. inside a container whose port is published +> python main.py --mode both --host 0.0.0.0 > ``` > -> Or edit `config.json`: +> Or in `config.json`: > > ```json > { -> "web_ui": { "host": "127.0.0.1", "port": 5001 } +> "web_ui": { "host": "0.0.0.0", "port": 5001 } > } > ``` > -> Do **not** expose the default `0.0.0.0` bind on a workstation connected to an untrusted network (home Wi-Fi, café, corporate LAN, public cloud VM) without a firewall, reverse proxy with authentication, or VPN in front of it. +> **Docker note:** this repo ships no Dockerfile of its own; containerized runs (e.g. the shared AutoGUI Docker harness — see "Docker harness" below) must pass `--host 0.0.0.0` (or set `web_ui.host`) explicitly so the server stays reachable through the container's published port. Nothing else changes — the opt-in flag is the only difference from a local run. +> +> Do **not** use a non-loopback bind on a workstation connected to an untrusted network (home Wi-Fi, café, corporate LAN, public cloud VM) without a firewall, reverse proxy with authentication, or VPN in front of it. +> +> **CORS:** disabled by default. The server sends no `Access-Control-Allow-Origin` headers unless `web_ui.cors_origins` is configured (default `[]`), so browsers enforce the same-origin policy — websites open in the user's browser cannot script cross-origin calls to the API, and the bundled web inspector keeps working because it is served same-origin at `/`. To allow cross-origin callers (an external dashboard, or Docker/testing scenarios), list explicit origins — or `["*"]` for anything-goes test rigs: > -> **CORS warning:** The Flask server enables permissive CORS for all routes by default (`CORS(app)`). Any website running in the user's browser can send cross-origin requests to the API — including destructive `/api/action` calls. Restrict CORS origins or add an authentication/proxy layer before exposing the server to a multi-user environment. +> ```json +> { +> "web_ui": { "host": "127.0.0.1", "port": 5001, "cors_origins": ["http://localhost:3000"] } +> } +> ``` +> +> Permissive CORS is never sent on `/api/action` (or any other route) by default; enabling `["*"]` re-opens the cross-origin attack surface described above, so combine it with a non-loopback bind only inside fully isolated environments. ### Startup modes @@ -69,8 +77,8 @@ curl http://127.0.0.1:5001/api/healthz | `GET` | `/api/screenshot` | Base64-encoded PNG screenshot | | `POST` | `/api/action` | Execute click, type, key, or scroll action | | `GET` | `/api/capabilities` | Server capabilities and platform info | -| `GET` | `/api/healthz` | Health and uptime | -| `GET` | `/api/metrics` | Prometheus-format metrics (`text/plain`) | +| `GET` | `/api/healthz` | Health, uptime, step count, tree-cache hit/miss + capture-latency telemetry, config/OCR diagnostics (cheap to poll — the OCR probe is computed once per process) | +| `GET` | `/api/metrics` | Prometheus-format metrics (`text/plain`): `oso_step_count`, `oso_uptime_seconds`, `oso_tree_cache_hits_total` / `oso_tree_cache_misses_total` / `oso_tree_cache_entries`, `oso_tree_capture_ms` summary (+`_max`), budget counters when configured, `oso_active_trace` | ### Example workflow @@ -409,6 +417,36 @@ in the tools menu. You can then ask Claude to: --- +## Trust Boundary: Screen Content Is Untrusted + +Everything OSScreenObserver reads off the screen is **attacker-influenced +input**: window titles, accessibility-tree element names and values, OCR +words, and VLM descriptions all come from whatever happens to be displayed +— a web page, email, or document on screen can deliberately contain +prompt-injection text ("ignore your previous instructions and …"). + +The server marks and sanitizes this data so clients can handle it safely: + +- Results of screen-text-carrying tools (`list_windows`, `find_element`, + `get_window_structure`, `observe_window`, `wait_for`, `snapshot_get`, + `snapshot_diff`, `get_ocr`, `get_screen_description`) carry a top-level + **`untrusted: true`** flag. +- ANSI escape sequences and non-whitespace control characters are stripped + from extracted text, so screen content cannot smuggle terminal escapes + into logs or client UIs. + +**Guidance for MCP clients / agents:** treat every `untrusted: true` field +as *data, never instructions*. Text found on screen must not change the +agent's goals, unlock destructive actions, or be executed/evaluated. Pair +this with the confirmation-token flow (`propose_action` + +`confirmation_required` rules) so that even a successfully injected "click +Delete" suggestion still requires an explicit out-of-band token before the +destructive verb executes. Note that screenshots (base64 PNGs) are equally +attacker-influenced; they are not flagged per-pixel, so the same rule +applies to anything a VLM extracts from them. + +--- + ## REST API Reference The web inspector exposes the following endpoints (all `GET` unless noted): @@ -416,7 +454,8 @@ The web inspector exposes the following endpoints (all `GET` unless noted): | Endpoint | Params | Description | |----------|--------|-------------| | `GET /api/windows` | — | List all visible windows | -| `GET /api/structure` | `window_index` | Accessibility element tree (JSON) | +| `GET /api/structure` | `window_index`, `depth`, `scope` | Accessibility element tree (JSON). Returned to `tree.default_depth` levels by default; deeper branches carry `truncated: true` + `child_count` — drill in with `scope=` and/or a larger `depth` (capped at `tree.max_depth`) | +| `GET /api/observe` | `window_index`/`window_uid`, `since`, `changed_only`, `depth` | Tree observation. `since=` returns a diff against that baseline; `changed_only=1` compares against the window's last capture and returns a tiny `{unchanged: true}` or a diff instead of the full tree. Responses include `perf` (`capture_ms`, `node_count`, `cache` hit/miss/bypass, `depth_used`) | | `GET /api/description` | `window_index` | Combined description (accessibility + OCR + VLM, whatever is available). `mode` query parameter is accepted but ignored — always returns combined output. | | `GET /api/sketch` | `window_index`, `grid_width`, `grid_height`, `ocr` | ASCII layout sketch | | `GET /api/screenshot` | `window_index` | Screenshot as base64 PNG | @@ -490,14 +529,15 @@ contains `"success": false` with an explanatory error message — the click is ## Configuration Reference (`config.json`) -The following shows the built-in defaults (when no `config.json` is provided). The shipped `config.json` overrides `web_ui.host` to `127.0.0.1` for loopback-only access. +The following shows the built-in defaults (when no `config.json` is provided). ```jsonc { "web_ui": { - "host": "0.0.0.0", // bind address; use "127.0.0.1" for loopback-only + "host": "127.0.0.1", // bind address; loopback-only by default — set "0.0.0.0" (or use --host) to opt in to network exposure (unauthenticated!) "port": 5001, // HTTP port - "debug": false + "debug": false, + "cors_origins": [] // CORS allowlist; [] = no CORS headers (same-origin only), ["*"] = permissive (isolated test rigs only) }, "mcp": { "server_name": "os-screen-observer", @@ -521,7 +561,11 @@ The following shows the built-in defaults (when no `config.json` is provided). T "unicode_box": true // false → plain ASCII +/-/| instead of ┌─┐│└┘ }, "tree": { - "max_depth": 8 // maximum depth to traverse (Windows only) + "max_depth": 8, // hard cap on traversal depth + "default_depth": 5, // depth returned when no depth= is requested (drill in via scope=) + "cache_ttl_s": 2.0, // per-window tree cache TTL; input actions invalidate automatically + "strategy": "merged", // Windows capture pipeline: "merged" (UIA + pywinauto) or "uia_only" (faster) + "sparse_threshold": 5 // fewer named nodes than this → degraded (sparse tree) hint in results }, "logging": { "level": "INFO" // DEBUG / INFO / WARNING / ERROR @@ -657,15 +701,23 @@ Default CI lane selects `not user` so the new tier is opt-in. 1. **Accessibility-dark applications** — Games, Electron apps with custom renderers, and applications that do not instrument UIA will produce sparse trees. The OCR - and VLM modalities degrade more gracefully in these cases. + and VLM modalities degrade more gracefully in these cases; `get_window_structure` + flags such captures with `degraded: {reason: "sparse_accessibility_tree", …}` + (threshold: `tree.sparse_threshold`) and `get_capabilities` reports per-window + `tree_stats` so agents can switch modality automatically. 2. **Prompt injection risk** — Screen content is included verbatim in tool results. Malicious content on-screen could attempt to influence the AI's behavior. Apply appropriate trust boundaries when using this server in production contexts. 3. **Performance** — Full tree traversal on a complex window (e.g., a browser with - many DOM-mapped UIA nodes) can take several seconds. The `tree.max_depth` - config key limits traversal depth. + many DOM-mapped UIA nodes) can take several seconds. Mitigations: captures are + cached per window for `tree.cache_ttl_s` (input actions invalidate), results + are depth-bounded to `tree.default_depth` with `scope=`/`depth=` drill-in, + `observe_window` supports `changed_only=1`, and on Windows `tree.strategy: + "uia_only"` plus UIA CacheRequest batching cut per-walk COM round trips. + Every structure/observe result reports `perf.capture_ms` so slow windows are + visible. `tree.max_depth` still hard-caps traversal depth. 4. **Action safety** — The `click_at`, `type_text`, and `press_key` tools execute real input events. Apply appropriate authorization controls before exposing diff --git a/ascii_renderer.py b/ascii_renderer.py index e8489c1..e4cb5bb 100755 --- a/ascii_renderer.py +++ b/ascii_renderer.py @@ -323,8 +323,8 @@ def _preprocess_image(img: "Image.Image", upscale: int) -> Tuple["Image.Image", img = img.convert("L") try: img = ImageOps.autocontrast(img, cutoff=2) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr preprocess] autocontrast skipped: {e}") return img, float(upscale or 1) @@ -358,8 +358,8 @@ def _ocr_lines(screenshot_bytes: bytes, try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr] tesseract_cmd configure failed: {e}") ocr_cfg = (config or {}).get("ocr") or {} min_conf = int(ocr_cfg.get("min_confidence", 30)) @@ -442,8 +442,8 @@ def _ocr_roi_text(crop: "Image.Image", psm: int, config: Optional[dict]) -> str: resample=__import__("PIL").Image.LANCZOS, ).convert("L") crop = ImageOps.autocontrast(crop, cutoff=2) - except Exception: - pass + except Exception as e: + logger.debug(f"[ocr roi] crop preprocess skipped: {e}") txt = pytesseract.image_to_string(crop, config=f"--psm {psm} --oem 3") return " ".join(txt.split()).strip() except Exception: @@ -535,8 +535,8 @@ def __init__(self, config: dict): try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[init] tesseract_cmd configure failed: {e}") # ── public string-only API (back-compat) ───────────────────────────────── diff --git a/budgets.py b/budgets.py index 9da714b..ebbecd2 100644 --- a/budgets.py +++ b/budgets.py @@ -48,12 +48,12 @@ def is_set(self) -> bool: return self.limit is not None def remaining(self) -> Optional[int]: - if not self.is_set(): + if self.limit is None: return None return max(0, self.limit - self.used) def trip(self) -> bool: - return self.is_set() and self.used >= self.limit + return self.limit is not None and self.used >= self.limit class BudgetStore: @@ -93,7 +93,8 @@ def gate(self, tool: str) -> Optional[Dict[str, Any]]: with self.lock: now = time.time() elapsed = now - self.started_at - if self.session_secs.is_set() and elapsed >= self.session_secs.limit: + session_limit = self.session_secs.limit + if session_limit is not None and elapsed >= session_limit: return error_dict(Code.BUDGET_EXCEEDED, "max_session_seconds reached", elapsed=int(elapsed)) diff --git a/config.json.example b/config.json.example index b395462..d57213e 100644 --- a/config.json.example +++ b/config.json.example @@ -1,11 +1,12 @@ { "_about": "Copy this file to config.json and edit for your machine. config.json is in .gitignore. All sections are optional; OSScreenObserver falls back to reasonable defaults when a key is missing. Keys starting with an underscore are documentation-only and ignored at load time.", - "_web_ui": "HTTP inspector server. Bind to 127.0.0.1 unless you are deliberately exposing it to your LAN. Disable debug in shared environments.", + "_web_ui": "HTTP inspector server. The API is unauthenticated, so 'host' defaults to loopback (127.0.0.1); set '0.0.0.0' (or pass --host 0.0.0.0) only to deliberately expose it — e.g. inside a Docker container with a published port — and expect a startup warning. 'cors_origins' is the CORS allowlist: [] (default) sends no CORS headers so browsers enforce same-origin (the bundled inspector UI is same-origin and unaffected); list explicit origins, or [\"*\"] for isolated Docker/testing rigs. Disable debug in shared environments.", "web_ui": { "host": "127.0.0.1", "port": 5001, - "debug": false + "debug": false, + "cors_origins": [] }, "_mcp": "Identifiers reported over the MCP framing channel. Bump 'version' when you publish a breaking change to a custom fork.", @@ -79,9 +80,13 @@ "vlm_fallback": false }, - "_tree": "Accessibility-tree walk limits. 'max_depth' caps recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval. Raise it if your target app legitimately nests deeper.", + "_tree": "Accessibility-tree walk limits. 'max_depth' is the hard cap on recursion so pathological apps (web views with deep DOMs) cannot blow up tree retrieval; raise it if your target app legitimately nests deeper. 'default_depth' is the depth get_window_structure / observe_window return when the caller passes no explicit depth= — deeper branches are marked truncated:true with a child_count so agents can drill in via the scope= parameter (requests may go deeper, up to max_depth). 'cache_ttl_s' controls the per-window tree cache: read-only tools reuse the last capture for this many seconds instead of re-walking the tree (input actions invalidate the cache automatically); set 0 to disable reuse. 'strategy' (Windows only) selects the capture pipeline: 'merged' (default) runs both the raw-UIA walk and a second pywinauto walk and synthesizes them — richest tree, slowest; 'uia_only' runs just the raw-UIA walk — roughly half the capture time, may miss a few native-control properties the merge would contribute. 'sparse_threshold': when a capture has fewer than this many named non-root nodes, get_window_structure attaches a degraded:{reason:'sparse_accessibility_tree'} hint suggesting OCR/VLM fallbacks.", "tree": { - "max_depth": 8 + "max_depth": 8, + "default_depth": 5, + "cache_ttl_s": 2.0, + "strategy": "merged", + "sparse_threshold": 5 }, "_logging": "Python logging level for the OSScreenObserver loggers. Use DEBUG to trace adapter behavior, WARNING to silence routine info.", diff --git a/description.py b/description.py index b581cb3..e6903b8 100755 --- a/description.py +++ b/description.py @@ -243,8 +243,8 @@ def __init__(self, config: dict): try: from ocr_util import configure as _ocr_configure _ocr_configure(config) - except Exception: - pass + except Exception as e: + logger.debug(f"[init] tesseract_cmd configure failed: {e}") # ── Accessibility tree → prose ──────────────────────────────────────────── @@ -537,7 +537,7 @@ def _prepare_image(screenshot_bytes: bytes, max_dim: int) -> bytes: return screenshot_bytes scale = max_dim / long_edge new_size = (max(1, int(w * scale)), max(1, int(h * scale))) - resized = img.resize(new_size, Image.LANCZOS) + resized = img.resize(new_size, Image.Resampling.LANCZOS) buf = io.BytesIO() resized.save(buf, format="PNG", optimize=True) return buf.getvalue() @@ -718,11 +718,11 @@ def from_vlm_multipass( env["_passes"]["scene_ms"] = int((time.time() - t0) * 1000) scene_obj: Dict[str, Any] = {} if raw1: - scene_obj, err = _tolerant_json_loads(raw1) - if scene_obj is None: - scene_obj = {} + parsed1, err = _tolerant_json_loads(raw1) + if parsed1 is None: env["_passes"]["scene_error"] = err else: + scene_obj = parsed1 for k in ("app", "screen_type", "primary_task"): if scene_obj.get(k) is not None: env[k] = scene_obj[k] @@ -736,11 +736,11 @@ def from_vlm_multipass( env["_passes"]["controls_ms"] = int((time.time() - t0) * 1000) controls_obj: Dict[str, Any] = {} if raw2: - controls_obj, err = _tolerant_json_loads(raw2) - if controls_obj is None: - controls_obj = {} + parsed2, err = _tolerant_json_loads(raw2) + if parsed2 is None: env["_passes"]["controls_error"] = err else: + controls_obj = parsed2 for k in ("focused", "modal_open", "controls", "sensitive_regions"): if controls_obj.get(k) is not None: @@ -770,8 +770,8 @@ def from_vlm_multipass( tree_text = "" try: tree_text = self.from_tree(root, window) - except Exception: - pass + except Exception as e: + logger.debug(f"[vlm verify] tree grounding skipped: {e}") ctxv = ( f"\n" f"{json.dumps(env.get('controls') or [], ensure_ascii=False)}\n" diff --git a/element_selectors.py b/element_selectors.py index 13dc13f..6b197e4 100644 --- a/element_selectors.py +++ b/element_selectors.py @@ -361,8 +361,8 @@ def _parse_role_and_preds(token: str) -> Tuple[str, List[Predicate]]: pos = end + 1 continue # Playwright :has-text("…") / :text("…") → name*="…" - if rest[pos] == ":" and _HASTEXT_RE.match(rest, pos): - ht = _HASTEXT_RE.match(rest, pos) + ht = _HASTEXT_RE.match(rest, pos) if rest[pos] == ":" else None + if ht is not None: text_val = ht.group("dq") if ht.group("dq") is not None else ht.group("sq") preds.append(Predicate(key="name", op="*=", value=text_val)) pos = ht.end() diff --git a/linux_adapter.py b/linux_adapter.py index ac61abe..aed6b90 100644 --- a/linux_adapter.py +++ b/linux_adapter.py @@ -70,8 +70,8 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": states = node.getState().getStates() enabled = pyatspi.STATE_ENABLED in states focused = pyatspi.STATE_FOCUSED in states - selected = (pyatspi.STATE_SELECTED in states - or pyatspi.STATE_CHECKED in states) + selected: 'bool | None' = (pyatspi.STATE_SELECTED in states + or pyatspi.STATE_CHECKED in states) expanded = pyatspi.STATE_EXPANDED in states if ( pyatspi.STATE_EXPANDABLE in states) else None # STATE_SELECTED on non-selectable widgets is meaningless; only @@ -92,8 +92,9 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": value_now = float(iv.currentValue) value_min = float(iv.minimumValue) value_max = float(iv.maximumValue) - except Exception: - pass + except Exception as e: + logger.debug(f"[atspi] value interface unavailable for " + f"{eid}: {e}") ui = UIElement( element_id=eid, name=name, role=role, value=value, @@ -108,8 +109,8 @@ def _walk(node, eid: str, depth: int, max_depth: int) -> "UIElement": for i in range(node.childCount): ui.children.append(_walk(node[i], f"{eid}.{i}", depth + 1, max_depth)) - except Exception: - pass + except Exception as e: + logger.debug(f"[atspi] child walk truncated at {eid}: {e}") return ui def get_element_tree(hwnd=None) -> Optional[UIElement]: diff --git a/main.py b/main.py index 8275423..1c6de29 100644 --- a/main.py +++ b/main.py @@ -54,7 +54,11 @@ # ───────────────────────────────────────────────────────────────────────────── _DEFAULT_CONFIG = { - "web_ui": {"host": "0.0.0.0", "port": 5001, "debug": False}, + # Loopback by default: the HTTP API is unauthenticated, so it must not + # be reachable from the network unless the operator explicitly opts in + # via --host / web_ui.host (e.g. 0.0.0.0 for Docker deployments). + "web_ui": {"host": "127.0.0.1", "port": 5001, "debug": False, + "cors_origins": []}, "mcp": {"server_name": "os-screen-observer", "version": "0.1.0"}, "ocr": {"enabled": True, "tesseract_cmd": None, "min_confidence": 30}, "vlm": {"enabled": False, @@ -63,7 +67,8 @@ "model": None, "max_tokens": 1500}, "ascii_sketch": {"grid_width": 110, "grid_height": 38, "unicode_box": True}, - "tree": {"max_depth": 8}, + "tree": {"max_depth": 8, "default_depth": 5, "cache_ttl_s": 2.0, + "strategy": "merged", "sparse_threshold": 5}, "logging": {"level": "INFO"}, "mock": False, "platform": "auto", @@ -130,10 +135,11 @@ def load_config(path: str) -> dict: print(f"\n[main:load_config] {msg}\n", file=sys.stderr) return dict(_DEFAULT_CONFIG) # Deep-merge with defaults so missing keys are always present - merged = dict(_DEFAULT_CONFIG) + merged: dict = dict(_DEFAULT_CONFIG) for k, v in cfg.items(): - if isinstance(v, dict) and isinstance(merged.get(k), dict): - merged[k] = {**merged[k], **v} + base = merged.get(k) + if isinstance(v, dict) and isinstance(base, dict): + merged[k] = {**base, **v} else: merged[k] = v return merged @@ -155,6 +161,32 @@ def config_load_status() -> dict: "config_error": _CONFIG_LOAD_ERROR or None} +_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def bind_warning(host: str) -> "str | None": + """Return a prominent warning banner when *host* exposes the HTTP API + beyond loopback, or None for loopback binds. + + The web/REST surface is unauthenticated by design (see README + "Security & Network Bind Defaults"); a non-loopback bind is an explicit + operator opt-in (Docker, isolated test VMs) and deserves a loud notice. + """ + if (host or "").strip().lower() in _LOOPBACK_HOSTS: + return None + return ( + "╔══════════════════════════════════════════════════════════════════════╗\n" + "║ SECURITY WARNING: ║\n" + "║ unauthenticated action API exposed to the network ║\n" + f"║ web_ui.host = {host!r:<54s} ║\n" + "║ Anyone who can reach this port can observe the screen and inject ║\n" + "║ clicks/keystrokes via /api/action. Only use a non-loopback bind ║\n" + "║ inside an isolated container/VM or behind a firewall or ║\n" + "║ authenticating reverse proxy. ║\n" + "╚══════════════════════════════════════════════════════════════════════╝" + ) + + def setup_logging(config: dict) -> None: level_name = config.get("logging", {}).get("level", "INFO").upper() level = getattr(logging, level_name, logging.INFO) @@ -365,6 +397,16 @@ def main() -> None: host = config["web_ui"]["host"] port = config["web_ui"]["port"] + + warning = bind_warning(host) + if warning: + print(warning, file=sys.stderr) + logger.warning( + "[main] unauthenticated action API exposed to the network " + f"(web_ui.host={host!r}); this is an explicit opt-in — see " + "README 'Security & Network Bind Defaults'" + ) + app = create_web_app(observer, renderer, describer, config) def _run_flask(): diff --git a/mcp_server/__init__.py b/mcp_server/__init__.py new file mode 100644 index 0000000..3835228 --- /dev/null +++ b/mcp_server/__init__.py @@ -0,0 +1,23 @@ +""" +mcp_server — MCP stdio server (package form of the former mcp_server.py). + +Implements the Model Context Protocol (2024-11-05) as JSON-RPC 2.0 over +stdin/stdout so that any MCP-capable client (Claude Desktop, Claude Code, +etc.) can use this server as a tool provider. + +ALL output to stdout is MCP protocol JSON. All logging goes to stderr so +that the MCP framing on stdout is never polluted. + +P3 decomposition: the implementation now lives in submodules +(tool_schemas, server). This __init__ re-exports the pre-split surface +so `from mcp_server import MCPServer` / `mcp_server._TOOLS` keep working +unchanged. (The package is deliberately named mcp_server — not mcp — to +avoid colliding with the external MCP SDK distribution name.) +""" + +from __future__ import annotations + +from mcp_server.server import MCPServer +from mcp_server.tool_schemas import _TOOLS + +__all__ = ["MCPServer", "_TOOLS"] diff --git a/mcp_server/server.py b/mcp_server/server.py new file mode 100644 index 0000000..e0d76f0 --- /dev/null +++ b/mcp_server/server.py @@ -0,0 +1,369 @@ +""" +MCP stdio server: JSON-RPC 2.0 transport, message +dispatch and legacy composite tool handlers. + +Split out of mcp_server.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import logging +import sys +import traceback +from typing import Any, Dict + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from observer import ScreenObserver +import tools as _tools + +from mcp_server.tool_schemas import _TOOLS + +logger = logging.getLogger(__name__) + + +class MCPServer: + """ + MCP stdio server. + + Reads newline-delimited JSON-RPC 2.0 messages from stdin, dispatches + to tool handlers, and writes responses to stdout. All log output + is directed to stderr to preserve the integrity of the MCP framing. + """ + + PROTOCOL_VERSION = "2024-11-05" + + def __init__( + self, + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: Dict, + ): + self.observer = observer + self.renderer = renderer + self.describer = describer + self.config = config + + # ── Transport ───────────────────────────────────────────────────────────── + + def _emit(self, payload: Dict) -> None: + """Write a JSON-RPC message to stdout.""" + sys.stdout.write(json.dumps(payload) + "\n") + sys.stdout.flush() + + def _error(self, request_id: Any, code: int, message: str) -> None: + self._emit({ + "jsonrpc": "2.0", + "id": request_id, + "error": {"code": code, "message": message}, + }) + + def _result(self, request_id: Any, result: Any) -> None: + self._emit({"jsonrpc": "2.0", "id": request_id, "result": result}) + + # ── Message dispatch ────────────────────────────────────────────────────── + + def _handle(self, msg: Dict) -> None: + method = msg.get("method", "") + params = msg.get("params") or {} + rid = msg.get("id") # None for notifications + + try: + if method == "initialize": + self._result(rid, { + "protocolVersion": self.PROTOCOL_VERSION, + "serverInfo": { + "name": self.config["mcp"]["server_name"], + "version": self.config["mcp"]["version"], + }, + "capabilities": {"tools": {}}, + }) + + elif method in ("notifications/initialized", "ping"): + if rid is not None: + self._result(rid, {}) + + elif method == "tools/list": + self._result(rid, {"tools": _TOOLS}) + + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments") or {} + result = self._dispatch(tool_name, arguments) + self._result(rid, { + "content": [{"type": "text", "text": json.dumps(result, indent=2)}] + }) + + else: + if rid is not None: + self._error(rid, -32601, f"Method not found: {method}") + + except Exception as e: + print(f"[MCPServer:_handle] {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + if rid is not None: + self._error(rid, -32603, str(e)) + + # ── Tool dispatcher ─────────────────────────────────────────────────────── + + def _dispatch(self, name: str, args: Dict) -> Any: + """Route a tools/call to the appropriate handler.""" + # New centralised tools (P1+) live in tools.py. + if name in _tools.REGISTRY: + ctx = _tools.ToolContext( + observer=self.observer, renderer=self.renderer, + describer=self.describer, config=self.config, + ) + return _tools.dispatch(ctx, name, args) + + try: + windows = self.observer.list_windows() + idx = args.get("window_index") + info = self.observer.window_by_index(windows, idx) + hwnd = info.handle if info else None + + if name == "list_windows": + return self._t_list_windows(windows) + + elif name == "get_window_structure": + return self._t_structure(hwnd, info, args) + + elif name == "get_screen_description": + return self._t_description(hwnd, info, args) + + elif name == "get_screen_sketch": + return self._t_sketch(hwnd, info, args) + + elif name == "get_screenshot": + return self._t_screenshot(hwnd, info) + + elif name == "click_at": + return self._t_click_at(args) + + elif name == "type_text": + return self.observer.perform_action("type", value=args.get("text", "")) + + elif name == "press_key": + return self.observer.perform_action("key", value=args.get("keys", "")) + + elif name == "scroll": + return self.observer.perform_action("scroll", value=args) + + elif name == "get_full_screenshot": + return self._t_full_screenshot(hwnd, info, args) + + elif name == "get_visible_areas": + return self._t_visible_areas(hwnd, info, windows) + + elif name == "bring_to_foreground": + return self._t_bring_to_foreground(hwnd, info, windows) + + else: + return {"error": f"Unknown tool: {name}"} + + except Exception as e: + print(f"[MCPServer:_dispatch:{name}] {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + return {"error": str(e)} + + # ── Individual tool handlers ────────────────────────────────────────────── + + def _t_list_windows(self, windows) -> Dict: + return { + "count": len(windows), + "windows": [ + {"index": i, **w.to_dict()} for i, w in enumerate(windows) + ], + } + + def _t_structure(self, hwnd, info, args) -> Dict: + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree for this window"} + return { + "window": info.title if info else "(focused)", + "element_count": len(tree.flat_list()), + "tree": tree.to_dict(), + } + + def _t_description(self, hwnd, info, args) -> Dict: + mode = args.get("mode", "accessibility") + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree"} + + shot = self.observer.get_screenshot(hwnd) + + if mode == "accessibility": + return {"mode": mode, "description": self.describer.from_tree(tree, info)} + elif mode == "ocr": + if shot is None: + return {"error": "Screenshot unavailable for OCR"} + return {"mode": mode, "description": self.describer.from_ocr(shot)} + elif mode == "vlm": + if shot is None: + return {"error": "Screenshot unavailable for VLM"} + vlm_mode = (self.describer.vlm_cfg.get("mode") or "single").lower() + if vlm_mode == "multipass": + env = self.describer.from_vlm_multipass( + shot, root=tree, window=info, + ) + if env is None: + return {"mode": mode, + "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} + return {"mode": mode, + "description": json.dumps(env, indent=2, ensure_ascii=False), + "vlm_structured": env} + vlm_out = self.describer.from_vlm(shot, root=tree, window=info) + if vlm_out is None: + return {"mode": mode, "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} + return {"mode": mode, "description": vlm_out} + elif mode == "combined": + return {"mode": mode, **self.describer.combined(tree, shot, info)} + else: + return {"error": f"Unknown mode: {mode}"} + + def _t_sketch(self, hwnd, info, args) -> Dict: + tree = self.observer.get_element_tree(hwnd) + if tree is None: + return {"error": "Could not retrieve element tree"} + + ref = info.bounds if info else tree.bounds + result = self.renderer.render_structured( + root = tree, + screen_bounds = ref, + grid_width = args.get("grid_width"), + grid_height = args.get("grid_height"), + ) + out = { + "window": info.title if info else "(focused)", + "grid_width": args.get("grid_width", self.renderer.default_width), + "grid_height": args.get("grid_height", self.renderer.default_height), + "sketch": result["sketch"], + } + if args.get("structured"): + out["elements"] = result["elements"] + out["legend"] = result["legend"] + return out + + def _t_screenshot(self, hwnd, info) -> Dict: + import base64 + shot = self.observer.get_screenshot(hwnd) + if shot is None: + return {"error": "Screenshot capture failed"} + return { + "window": info.title if info else "(full screen)", + "format": "png", + "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + + def _t_full_screenshot(self, hwnd, info, args) -> Dict: + import base64 + # Always capture all monitors combined + shot = self.observer.get_full_display_screenshot() + if shot is None: + return {"error": "Screenshot capture failed"} + + sketch = None + tree = self.observer.get_element_tree(hwnd) if hwnd is not None else None + if tree is not None: + ref = info.bounds if info else self.observer.get_screen_bounds() + # Crop the full-display PNG to window bounds for accurate OCR overlay. + ocr_bytes = shot + if info is not None: + try: + import io as _io2 + from PIL import Image as _Image2 + full_img = _Image2.open(_io2.BytesIO(shot)) + screen_b = self.observer.get_screen_bounds() + crop_box = ( + max(0, info.bounds.x - screen_b.x), + max(0, info.bounds.y - screen_b.y), + min(full_img.width, info.bounds.right - screen_b.x), + min(full_img.height, info.bounds.bottom - screen_b.y), + ) + buf2 = _io2.BytesIO() + full_img.crop(crop_box).save(buf2, format="PNG") + ocr_bytes = buf2.getvalue() + except Exception as e: + logger.debug( + f"[get_full_screenshot] window crop for OCR overlay " + f"failed; using full image: {e}") + sketch = self.renderer.render( + root = tree, + screen_bounds = ref, + grid_width = args.get("grid_width"), + grid_height = args.get("grid_height"), + screenshot_bytes = ocr_bytes, + ) + + img_w = img_h = None + try: + import io as _io + from PIL import Image as _Image + _img = _Image.open(_io.BytesIO(shot)) + img_w, img_h = _img.size + except Exception as e: + logger.debug(f"[get_full_screenshot] size probe failed: {e}") + + return { + "window": info.title if info else "(full screen)", + "screenshot_scope": "full_display", + "format": "png", + "encoding": "base64", + "width": img_w, + "height": img_h, + "data": base64.b64encode(shot).decode(), + "sketch": sketch, + } + + def _t_visible_areas(self, hwnd, info, windows) -> Dict: + if hwnd is None: + return {"error": "window_index is required for get_visible_areas"} + areas = self.observer.get_visible_areas(hwnd, windows) + return { + "window": info.title if info else "(unknown)", + "visible_regions": areas, + } + + def _t_bring_to_foreground(self, hwnd, info, windows) -> Dict: + if hwnd is None: + return {"success": False, + "error": "window_index is required for bring_to_foreground"} + result = self.observer.bring_to_foreground(hwnd, windows) + result["window"] = info.title if info else "(unknown)" + return result + + def _t_click_at(self, args) -> Dict: + return self.observer.perform_action( + "click_at", + value={ + "x": args.get("x", 0), + "y": args.get("y", 0), + "button": args.get("button", "left"), + "double": args.get("double", False), + }, + ) + + # ── Main loop ───────────────────────────────────────────────────────────── + + def run(self) -> None: + """Block on stdin, reading and processing JSON-RPC messages.""" + logger.info("[MCPServer:run] Listening on stdin (MCP mode)") + print("[MCPServer] Ready — listening on stdin", file=sys.stderr) + + for raw_line in sys.stdin: + raw_line = raw_line.strip() + if not raw_line: + continue + try: + msg = json.loads(raw_line) + self._handle(msg) + except json.JSONDecodeError as e: + print(f"[MCPServer:run] JSON parse error: {e}", file=sys.stderr) + except Exception as e: + print(f"[MCPServer:run] Unhandled error: {e}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) diff --git a/mcp_server.py b/mcp_server/tool_schemas.py old mode 100755 new mode 100644 similarity index 65% rename from mcp_server.py rename to mcp_server/tool_schemas.py index f528e45..810aed2 --- a/mcp_server.py +++ b/mcp_server/tool_schemas.py @@ -1,42 +1,12 @@ """ -mcp_server.py — MCP stdio server (Model Context Protocol, 2024-11-05). +MCP tool schema definitions (tools/list payload). -Implements the MCP protocol as JSON-RPC 2.0 over stdin/stdout so that any -MCP-capable client (Claude Desktop, Claude Code, etc.) can use this server -as a tool provider. - -ALL output to stdout is MCP protocol JSON. All logging goes to stderr so -that the MCP framing on stdout is never polluted. - -Exposed tools -───────────── - list_windows Enumerate visible top-level windows. - get_window_structure Accessibility element tree (JSON). - get_screen_description Prose description (accessibility / OCR / VLM / combined). - get_screen_sketch ASCII spatial layout diagram. - get_screenshot Screenshot as base64 PNG. - click_at Click at pixel coordinates. - type_text Type text into the focused element. - press_key Press a key or key combination. +Split out of mcp_server.py (P3); behavior is unchanged. """ -import json -import logging -import sys -import traceback -from typing import Any, Dict, List - -from observer import ScreenObserver -from ascii_renderer import ASCIIRenderer -from description import DescriptionGenerator -import tools as _tools - -logger = logging.getLogger(__name__) +from __future__ import annotations - -# ───────────────────────────────────────────────────────────────────────────── -# Tool schema definitions -# ───────────────────────────────────────────────────────────────────────────── +from typing import Dict, List _TOOLS: List[Dict] = [ { @@ -57,15 +27,23 @@ "description": ( "Return the accessibility element tree for a window as structured JSON. " "Each node carries id, name, role, value, bounds, enabled, focused, " - "keyboard_shortcut, and a children array. Supports server-side " - "filtering: roles, exclude_roles, name_regex, visible_only, " - "max_text_len, prune_empty, max_nodes (with page_cursor)." + "keyboard_shortcut, and a children array. Trees are returned to " + "tree.default_depth levels by default; deeper branches are marked " + "truncated:true with a child_count — pass depth (up to " + "tree.max_depth) or drill into one branch with scope=. " + "Supports server-side filtering: roles, exclude_roles, name_regex, " + "visible_only, max_text_len, prune_empty, max_nodes (with " + "page_cursor)." ), "inputSchema": { "type": "object", "properties": { "window_index": {"type": "integer"}, "window_uid": {"type": "string"}, + "depth": {"type": "integer", + "description": "Tree depth to return (default tree.default_depth, capped at tree.max_depth)."}, + "scope": {"type": "string", + "description": "Element id (e.g. 'root.3.2') — return only that subtree."}, "roles": {"type": "array", "items": {"type": "string"}}, "exclude_roles": {"type": "array", "items": {"type": "string"}}, "name_regex": {"type": "string"}, @@ -447,7 +425,11 @@ "tree_token from a previous observation as 'since' to get only " "what changed (custom diff format by default; pass " "format='json-patch' for RFC 6902). An expired token returns the " - "full tree with base_token=null." + "full tree with base_token=null. Alternatively pass " + "changed_only=true (no since needed) to compare against the " + "window's last capture: unchanged windows return a tiny " + "{unchanged: true, tree_hash} response, changed windows return a " + "diff instead of the full tree." ), "inputSchema": { "type": "object", @@ -456,6 +438,10 @@ "window_index": {"type": "integer"}, "since": {"type": "string"}, "format": {"type": "string", "enum": ["custom", "json-patch"]}, + "depth": {"type": "integer", + "description": "Tree depth for full-format responses (default tree.default_depth, capped at tree.max_depth)."}, + "changed_only": {"type": "boolean", + "description": "Compare against the window's last capture; returns {unchanged:true} or a diff instead of the full tree."}, }, "required": [], }, @@ -869,351 +855,3 @@ }, }, ] - - -# ───────────────────────────────────────────────────────────────────────────── -# MCPServer -# ───────────────────────────────────────────────────────────────────────────── - -class MCPServer: - """ - MCP stdio server. - - Reads newline-delimited JSON-RPC 2.0 messages from stdin, dispatches - to tool handlers, and writes responses to stdout. All log output - is directed to stderr to preserve the integrity of the MCP framing. - """ - - PROTOCOL_VERSION = "2024-11-05" - - def __init__( - self, - observer: ScreenObserver, - renderer: ASCIIRenderer, - describer: DescriptionGenerator, - config: Dict, - ): - self.observer = observer - self.renderer = renderer - self.describer = describer - self.config = config - - # ── Transport ───────────────────────────────────────────────────────────── - - def _emit(self, payload: Dict) -> None: - """Write a JSON-RPC message to stdout.""" - sys.stdout.write(json.dumps(payload) + "\n") - sys.stdout.flush() - - def _error(self, request_id: Any, code: int, message: str) -> None: - self._emit({ - "jsonrpc": "2.0", - "id": request_id, - "error": {"code": code, "message": message}, - }) - - def _result(self, request_id: Any, result: Any) -> None: - self._emit({"jsonrpc": "2.0", "id": request_id, "result": result}) - - # ── Message dispatch ────────────────────────────────────────────────────── - - def _handle(self, msg: Dict) -> None: - method = msg.get("method", "") - params = msg.get("params") or {} - rid = msg.get("id") # None for notifications - - try: - if method == "initialize": - self._result(rid, { - "protocolVersion": self.PROTOCOL_VERSION, - "serverInfo": { - "name": self.config["mcp"]["server_name"], - "version": self.config["mcp"]["version"], - }, - "capabilities": {"tools": {}}, - }) - - elif method in ("notifications/initialized", "ping"): - if rid is not None: - self._result(rid, {}) - - elif method == "tools/list": - self._result(rid, {"tools": _TOOLS}) - - elif method == "tools/call": - tool_name = params.get("name", "") - arguments = params.get("arguments") or {} - result = self._dispatch(tool_name, arguments) - self._result(rid, { - "content": [{"type": "text", "text": json.dumps(result, indent=2)}] - }) - - else: - if rid is not None: - self._error(rid, -32601, f"Method not found: {method}") - - except Exception as e: - print(f"[MCPServer:_handle] {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - if rid is not None: - self._error(rid, -32603, str(e)) - - # ── Tool dispatcher ─────────────────────────────────────────────────────── - - def _dispatch(self, name: str, args: Dict) -> Any: - """Route a tools/call to the appropriate handler.""" - # New centralised tools (P1+) live in tools.py. - if name in _tools.REGISTRY: - ctx = _tools.ToolContext( - observer=self.observer, renderer=self.renderer, - describer=self.describer, config=self.config, - ) - return _tools.dispatch(ctx, name, args) - - try: - windows = self.observer.list_windows() - idx = args.get("window_index") - info = self.observer.window_by_index(windows, idx) - hwnd = info.handle if info else None - - if name == "list_windows": - return self._t_list_windows(windows) - - elif name == "get_window_structure": - return self._t_structure(hwnd, info, args) - - elif name == "get_screen_description": - return self._t_description(hwnd, info, args) - - elif name == "get_screen_sketch": - return self._t_sketch(hwnd, info, args) - - elif name == "get_screenshot": - return self._t_screenshot(hwnd, info) - - elif name == "click_at": - return self._t_click_at(args) - - elif name == "type_text": - return self.observer.perform_action("type", value=args.get("text", "")) - - elif name == "press_key": - return self.observer.perform_action("key", value=args.get("keys", "")) - - elif name == "scroll": - return self.observer.perform_action("scroll", value=args) - - elif name == "get_full_screenshot": - return self._t_full_screenshot(hwnd, info, args) - - elif name == "get_visible_areas": - return self._t_visible_areas(hwnd, info, windows) - - elif name == "bring_to_foreground": - return self._t_bring_to_foreground(hwnd, info, windows) - - else: - return {"error": f"Unknown tool: {name}"} - - except Exception as e: - print(f"[MCPServer:_dispatch:{name}] {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) - return {"error": str(e)} - - # ── Individual tool handlers ────────────────────────────────────────────── - - def _t_list_windows(self, windows) -> Dict: - return { - "count": len(windows), - "windows": [ - {"index": i, **w.to_dict()} for i, w in enumerate(windows) - ], - } - - def _t_structure(self, hwnd, info, args) -> Dict: - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree for this window"} - return { - "window": info.title if info else "(focused)", - "element_count": len(tree.flat_list()), - "tree": tree.to_dict(), - } - - def _t_description(self, hwnd, info, args) -> Dict: - mode = args.get("mode", "accessibility") - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree"} - - shot = self.observer.get_screenshot(hwnd) - - if mode == "accessibility": - return {"mode": mode, "description": self.describer.from_tree(tree, info)} - elif mode == "ocr": - if shot is None: - return {"error": "Screenshot unavailable for OCR"} - return {"mode": mode, "description": self.describer.from_ocr(shot)} - elif mode == "vlm": - if shot is None: - return {"error": "Screenshot unavailable for VLM"} - vlm_mode = (self.describer.vlm_cfg.get("mode") or "single").lower() - if vlm_mode == "multipass": - env = self.describer.from_vlm_multipass( - shot, root=tree, window=info, - ) - if env is None: - return {"mode": mode, - "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} - return {"mode": mode, - "description": json.dumps(env, indent=2, ensure_ascii=False), - "vlm_structured": env} - vlm_out = self.describer.from_vlm(shot, root=tree, window=info) - if vlm_out is None: - return {"mode": mode, "description": "[VLM unavailable — check vlm.base_url and vlm.model in config.json]"} - return {"mode": mode, "description": vlm_out} - elif mode == "combined": - return {"mode": mode, **self.describer.combined(tree, shot, info)} - else: - return {"error": f"Unknown mode: {mode}"} - - def _t_sketch(self, hwnd, info, args) -> Dict: - tree = self.observer.get_element_tree(hwnd) - if tree is None: - return {"error": "Could not retrieve element tree"} - - ref = info.bounds if info else tree.bounds - result = self.renderer.render_structured( - root = tree, - screen_bounds = ref, - grid_width = args.get("grid_width"), - grid_height = args.get("grid_height"), - ) - out = { - "window": info.title if info else "(focused)", - "grid_width": args.get("grid_width", self.renderer.default_width), - "grid_height": args.get("grid_height", self.renderer.default_height), - "sketch": result["sketch"], - } - if args.get("structured"): - out["elements"] = result["elements"] - out["legend"] = result["legend"] - return out - - def _t_screenshot(self, hwnd, info) -> Dict: - import base64 - shot = self.observer.get_screenshot(hwnd) - if shot is None: - return {"error": "Screenshot capture failed"} - return { - "window": info.title if info else "(full screen)", - "format": "png", - "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - - def _t_full_screenshot(self, hwnd, info, args) -> Dict: - import base64 - # Always capture all monitors combined - shot = self.observer.get_full_display_screenshot() - if shot is None: - return {"error": "Screenshot capture failed"} - - sketch = None - tree = self.observer.get_element_tree(hwnd) if hwnd is not None else None - if tree is not None: - ref = info.bounds if info else self.observer.get_screen_bounds() - # Crop the full-display PNG to window bounds for accurate OCR overlay. - ocr_bytes = shot - if info is not None: - try: - import io as _io2 - from PIL import Image as _Image2 - full_img = _Image2.open(_io2.BytesIO(shot)) - screen_b = self.observer.get_screen_bounds() - crop_box = ( - max(0, info.bounds.x - screen_b.x), - max(0, info.bounds.y - screen_b.y), - min(full_img.width, info.bounds.right - screen_b.x), - min(full_img.height, info.bounds.bottom - screen_b.y), - ) - buf2 = _io2.BytesIO() - full_img.crop(crop_box).save(buf2, format="PNG") - ocr_bytes = buf2.getvalue() - except Exception: - pass - sketch = self.renderer.render( - root = tree, - screen_bounds = ref, - grid_width = args.get("grid_width"), - grid_height = args.get("grid_height"), - screenshot_bytes = ocr_bytes, - ) - - img_w = img_h = None - try: - import io as _io - from PIL import Image as _Image - _img = _Image.open(_io.BytesIO(shot)) - img_w, img_h = _img.size - except Exception: - pass - - return { - "window": info.title if info else "(full screen)", - "screenshot_scope": "full_display", - "format": "png", - "encoding": "base64", - "width": img_w, - "height": img_h, - "data": base64.b64encode(shot).decode(), - "sketch": sketch, - } - - def _t_visible_areas(self, hwnd, info, windows) -> Dict: - if hwnd is None: - return {"error": "window_index is required for get_visible_areas"} - areas = self.observer.get_visible_areas(hwnd, windows) - return { - "window": info.title if info else "(unknown)", - "visible_regions": areas, - } - - def _t_bring_to_foreground(self, hwnd, info, windows) -> Dict: - if hwnd is None: - return {"success": False, - "error": "window_index is required for bring_to_foreground"} - result = self.observer.bring_to_foreground(hwnd, windows) - result["window"] = info.title if info else "(unknown)" - return result - - def _t_click_at(self, args) -> Dict: - return self.observer.perform_action( - "click_at", - value={ - "x": args.get("x", 0), - "y": args.get("y", 0), - "button": args.get("button", "left"), - "double": args.get("double", False), - }, - ) - - # ── Main loop ───────────────────────────────────────────────────────────── - - def run(self) -> None: - """Block on stdin, reading and processing JSON-RPC messages.""" - logger.info("[MCPServer:run] Listening on stdin (MCP mode)") - print("[MCPServer] Ready — listening on stdin", file=sys.stderr) - - for raw_line in sys.stdin: - raw_line = raw_line.strip() - if not raw_line: - continue - try: - msg = json.loads(raw_line) - self._handle(msg) - except json.JSONDecodeError as e: - print(f"[MCPServer:run] JSON parse error: {e}", file=sys.stderr) - except Exception as e: - print(f"[MCPServer:run] Unhandled error: {e}", file=sys.stderr) - traceback.print_exc(file=sys.stderr) diff --git a/observer.py b/observer.py deleted file mode 100644 index 6ea8b3d..0000000 --- a/observer.py +++ /dev/null @@ -1,1687 +0,0 @@ -""" -observer.py — Core screen observation module. - -Provides a platform-aware ScreenObserver that exposes a uniform interface -for: enumerating windows, walking the accessibility element tree, capturing -screenshots, and dispatching input actions. Platform adapters (Windows/macOS/ -Linux/Mock) share a common base and are selected automatically at runtime. - -Data model ----------- - Bounds — screen-coordinate bounding rectangle - UIElement — one node of the accessibility tree - WindowInfo — top-level window metadata -""" - -import io -import logging -import os -import platform -import traceback -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -logger = logging.getLogger(__name__) - -PLATFORM = platform.system() - - -def _is_wsl() -> bool: - """True when running inside Windows Subsystem for Linux (WSL 1 or WSL 2).""" - if PLATFORM != "Linux": - return False - try: - with open("/proc/version") as _f: - return "microsoft" in _f.read().lower() - except Exception: - return False - - -IS_WSL = _is_wsl() -EFFECTIVE_PLATFORM = "WSL" if IS_WSL else PLATFORM - - -# ───────────────────────────────────────────────────────────────────────────── -# Data Structures -# ───────────────────────────────────────────────────────────────────────────── - -@dataclass -class Bounds: - x: int - y: int - width: int - height: int - - @property - def right(self) -> int: - return self.x + self.width - - @property - def bottom(self) -> int: - return self.y + self.height - - @property - def center_x(self) -> int: - return self.x + self.width // 2 - - @property - def center_y(self) -> int: - return self.y + self.height // 2 - - def to_dict(self) -> Dict: - return {"x": self.x, "y": self.y, "width": self.width, "height": self.height} - - def __bool__(self) -> bool: - return self.width > 0 and self.height > 0 - - -@dataclass -class UIElement: - element_id: str - name: str - role: str - value: Optional[str] = None - bounds: Bounds = field(default_factory=lambda: Bounds(0, 0, 0, 0)) - enabled: bool = True - focused: bool = False - keyboard_shortcut: Optional[str] = None - description: Optional[str] = None - children: List["UIElement"] = field(default_factory=list) - # Extended a11y signals — None means "adapter could not determine". - # Populated where the platform exposes the pattern (UIA SelectionItem / - # ExpandCollapse / RangeValue, AX AXValue/AXMinValue/AXMaxValue, AT-SPI - # STATE_SELECTED / STATE_EXPANDED / IValue) and consumed by the ASCII - # renderer for role-aware glyphs and the structured sidecar. - selected: Optional[bool] = None - expanded: Optional[bool] = None - value_now: Optional[float] = None - value_min: Optional[float] = None - value_max: Optional[float] = None - identifier: Optional[str] = None - - def to_dict(self) -> Dict: - d: Dict[str, Any] = { - "id": self.element_id, - "name": self.name, - "role": self.role, - "value": self.value, - "bounds": self.bounds.to_dict(), - "enabled": self.enabled, - "focused": self.focused, - "keyboard_shortcut": self.keyboard_shortcut, - "description": self.description, - "children": [c.to_dict() for c in self.children], - } - # Omit extended fields when unset so existing API consumers do not - # see a flood of nulls; include them when populated. - for k in ("selected", "expanded", "value_now", "value_min", - "value_max", "identifier"): - v = getattr(self, k) - if v is not None: - d[k] = v - return d - - def flat_list(self) -> List["UIElement"]: - """Return all elements in this subtree as a flat list (DFS order).""" - result = [self] - for child in self.children: - result.extend(child.flat_list()) - return result - - -@dataclass -class WindowInfo: - handle: Any # platform-specific: HWND (int) on Windows; int index elsewhere - title: str - process_name: str - pid: int - bounds: Bounds - is_focused: bool - # Stable cross-call identifier; populated by adapters (design doc §6.1). - window_uid: str = "" - # Optional multi-monitor metadata (design doc §6.3). Populated when the - # adapter knows; left None on adapters that do not. - monitor_index: Optional[int] = None - scale_factor: Optional[float] = None - logical_bounds: Optional[Bounds] = None - physical_bounds: Optional[Bounds] = None - - def to_dict(self) -> Dict: - d: Dict[str, Any] = { - "handle": str(self.handle), - "title": self.title, - "process": self.process_name, - "pid": self.pid, - "bounds": self.bounds.to_dict(), - "focused": self.is_focused, - "window_uid": self.window_uid, - } - if self.monitor_index is not None: - d["monitor_index"] = self.monitor_index - if self.scale_factor is not None: - d["scale_factor"] = self.scale_factor - if self.logical_bounds is not None: - d["logical_bounds"] = self.logical_bounds.to_dict() - if self.physical_bounds is not None: - d["physical_bounds"] = self.physical_bounds.to_dict() - return d - - -# ─── Window resolution result ──────────────────────────────────────────────── - -@dataclass -class WindowResolution: - info: Optional[WindowInfo] - warning: Optional[str] - used_uid: bool - requested_uid: Optional[str] - - -# ───────────────────────────────────────────────────────────────────────────── -# Mock Adapter (no OS dependencies; safe to run in any environment) -# ───────────────────────────────────────────────────────────────────────────── - -class MockAdapter: - """Synthetic data adapter for development and testing.""" - - def __init__(self) -> None: - import secrets as _s - self._nonce = _s.token_hex(4) - # Optional scenario hook (design doc §15.5). Set by main.py when - # --scenario is supplied; methods route through the scenario when - # active so that input actions can drive state transitions. - self.scenario: Optional[Any] = None - - def get_windows_above_bounds(self, hwnd) -> List["Bounds"]: - return [] # Mock assumes the target window is on top - - def list_windows(self) -> List[WindowInfo]: - if self.scenario is not None: - return self.scenario.list_windows(self._nonce) - return [ - WindowInfo(1001, "Untitled — Notepad", "notepad.exe", 1234, - Bounds(80, 60, 800, 600), True, - window_uid=f"mock:0:{self._nonce}"), - WindowInfo(1002, "GitHub · Where software is built — Google Chrome", - "chrome.exe", 5678, Bounds(0, 0, 1920, 1050), False, - window_uid=f"mock:1:{self._nonce}"), - WindowInfo(1003, "screen_observer.py — Visual Studio Code", - "code.exe", 9012, Bounds(960, 0, 960, 1050), False, - window_uid=f"mock:2:{self._nonce}"), - ] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - if self.scenario is not None: - return self.scenario.get_element_tree(hwnd) - root = UIElement("root", "Untitled — Notepad", "Window", - bounds=Bounds(80, 60, 800, 600)) - - # Menu bar - menubar = UIElement("root.0", "MenuBar", "MenuBar", - bounds=Bounds(80, 60, 800, 22)) - for i, lbl in enumerate(["File", "Edit", "Format", "View", "Help"]): - menubar.children.append(UIElement( - f"root.0.{i}", lbl, "MenuItem", - bounds=Bounds(80 + i * 58, 60, 56, 22) - )) - root.children.append(menubar) - - # Main editing area - editor = UIElement( - "root.1", "Text Editor", "Document", - value="Hello, world!\nThis is a test document.\nLine 3 has some content here.\n", - bounds=Bounds(80, 82, 800, 514), focused=True - ) - root.children.append(editor) - - # Horizontal scroll bar - hscroll = UIElement("root.2", "Horizontal ScrollBar", "ScrollBar", - bounds=Bounds(80, 596, 784, 18)) - root.children.append(hscroll) - - # Vertical scroll bar - vscroll = UIElement("root.3", "Vertical ScrollBar", "ScrollBar", - bounds=Bounds(864, 82, 18, 532)) - root.children.append(vscroll) - - # Progress bar — exercises value_now/min/max for the role glyph path. - root.children.append(UIElement( - "root.5", "Saving", "ProgressBar", - bounds=Bounds(560, 600, 200, 10), - value_now=40.0, value_min=0.0, value_max=100.0, - )) - - # Word-wrap toggle checkbox — exercises selected=True path. - root.children.append(UIElement( - "root.6", "Word Wrap", "CheckBox", - bounds=Bounds(80, 636, 120, 18), selected=True, - )) - - # Status bar - sb = UIElement("root.4", "Status Bar", "StatusBar", - bounds=Bounds(80, 614, 800, 22)) - for i, (lbl, val) in enumerate([ - ("Position", "Ln 1, Col 1"), - ("Zoom", "100%"), - ("Encoding", "UTF-8"), - ("EOL", "Windows (CRLF)"), - ]): - sb.children.append(UIElement( - f"root.4.{i}", lbl, "Text", value=val, - bounds=Bounds(80 + i * 190, 614, 188, 22) - )) - root.children.append(sb) - - return root - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - from PIL import Image, ImageDraw - img = Image.new("RGB", (800, 600), "#1a1e2e") - draw = ImageDraw.Draw(img) - - # Title bar - draw.rectangle([0, 0, 800, 30], fill="#2d3250") - draw.text((10, 8), "Untitled — Notepad", fill="#c8d3f5") - - # Menu bar - draw.rectangle([0, 30, 800, 52], fill="#1e2030") - for i, m in enumerate(["File", "Edit", "Format", "View", "Help"]): - draw.text((10 + i * 58, 36), m, fill="#a9b1d6") - - # Editor area - draw.rectangle([0, 52, 782, 570], fill="#1a1e2e") - for i, ln in enumerate(["Hello, world!", "This is a test document.", - "Line 3 has some content here.", ""]): - draw.text((6, 58 + i * 18), ln, fill="#c0caf5") - - # Scrollbars - draw.rectangle([782, 52, 800, 570], fill="#24283b") - draw.rectangle([0, 570, 782, 586], fill="#24283b") - - # Status bar - draw.rectangle([0, 586, 800, 600], fill="#16161e") - draw.text((6, 589), "Ln 1, Col 1 100% UTF-8 Windows (CRLF)", - fill="#565f89") - - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - print(f"[MockAdapter:get_screenshot] {e}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id: str = None, - value: Any = None, hwnd=None) -> Dict: - if self.scenario is not None: - handled = self.scenario.handle_action(action=action, - element_id=element_id, - value=value, hwnd=hwnd) - if handled is not None: - return handled - return { - "success": True, - "action": action, - "note": "Mock adapter — no real OS action performed", - } - - -# ───────────────────────────────────────────────────────────────────────────── -# UIA control-type ID → pywinauto-compatible role name. -# Values from UIAutomationClient.h; must match what selectors/descriptions expect. -_UIA_TYPE_TO_ROLE: Dict[int, str] = { - 50000: "Button", 50001: "Calendar", 50002: "CheckBox", - 50003: "ComboBox", 50004: "Edit", 50005: "Hyperlink", - 50006: "Image", 50007: "ListItem", 50008: "ListBox", - 50009: "Menu", 50010: "MenuBar", 50011: "MenuItem", - 50012: "ProgressBar", 50013: "RadioButton", 50014: "ScrollBar", - 50015: "Slider", 50016: "Spinner", 50017: "StatusBar", - 50018: "TabControl", 50019: "TabItem", 50020: "Text", - 50021: "Toolbar", 50022: "ToolTip", 50023: "Tree", - 50024: "TreeItem", 50025: "Custom", 50026: "GroupBox", - 50027: "Thumb", 50028: "DataGrid", 50029: "DataItem", - 50030: "Document", 50031: "SplitButton", 50032: "Dialog", - 50033: "Pane", 50034: "Header", 50035: "HeaderItem", - 50036: "Table", 50037: "TitleBar", 50038: "Separator", - 50039: "SemanticZoom",50040: "AppBar", -} - - -# Windows Adapter (requires: pywinauto, pywin32, psutil) -# ───────────────────────────────────────────────────────────────────────────── - -class WindowsAdapter: - """Full Windows UIA adapter using pywinauto + pywin32.""" - - def __init__(self, config: dict): - self.config = config - try: - import win32gui # noqa: F401 - import win32process # noqa: F401 - import psutil # noqa: F401 - from pywinauto import Application # noqa: F401 - self._Application = Application - logger.info("[WindowsAdapter:__init__] pywinauto/pywin32 ready") - except ImportError as e: - print(f"[WindowsAdapter:__init__] Missing dependency: {e}") - traceback.print_exc() - raise - - def list_windows(self) -> List[WindowInfo]: - try: - import win32gui - import win32process - import psutil - - results: List[WindowInfo] = [] - fg = win32gui.GetForegroundWindow() - - def _cb(hwnd, _): - try: - if not win32gui.IsWindowVisible(hwnd): - return - title = win32gui.GetWindowText(hwnd) - if not title: - return - rect = win32gui.GetWindowRect(hwnd) - w, h = rect[2] - rect[0], rect[3] - rect[1] - if w <= 0 or h <= 0: - return - try: - _, pid = win32process.GetWindowThreadProcessId(hwnd) - proc_name = psutil.Process(pid).name() - except Exception: - pid, proc_name = 0, "unknown" - results.append(WindowInfo( - handle=hwnd, title=title, process_name=proc_name, pid=pid, - bounds=Bounds(rect[0], rect[1], w, h), - is_focused=(hwnd == fg), - window_uid=f"win:{pid}:{hwnd}", - )) - except Exception as inner: - logger.debug(f"[WindowsAdapter:list_windows:_cb] {inner}") - - win32gui.EnumWindows(_cb, None) - return sorted(results, key=lambda w: (not w.is_focused, w.title.lower())) - except Exception as e: - print(f"[WindowsAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - """Return bounds of visible windows that are above hwnd in Z-order.""" - try: - import win32gui - GW_HWNDNEXT = 2 - above: List[Bounds] = [] - h = win32gui.GetTopWindow(None) - while h and h != hwnd: - try: - if win32gui.IsWindowVisible(h): - rect = win32gui.GetWindowRect(h) - w = rect[2] - rect[0] - hh = rect[3] - rect[1] - if w > 0 and hh > 0: - above.append(Bounds(rect[0], rect[1], w, hh)) - except Exception: - pass - try: - h = win32gui.GetWindow(h, GW_HWNDNEXT) - except Exception: - break - return above - except Exception as e: - logger.debug(f"[WindowsAdapter:get_windows_above_bounds] {e}") - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - try: - import win32gui - if hwnd is None: - hwnd = win32gui.GetForegroundWindow() - except Exception as e: - print(f"[WindowsAdapter:get_element_tree] win32gui: {e}") - return None - - # Run both walkers and synthesise: UIA crosses Chromium fragment boundaries - # (gets web content); pywinauto may surface native-control properties that - # UIA omits. The merged result gives the LLM everything either source sees. - uia_tree = self._uia_walk(hwnd) - pw_tree = None - try: - app = self._Application(backend="uia").connect(handle=hwnd) - window = app.window(handle=hwnd) - wrapper = window.wrapper_object() - max_depth = self.config.get("tree", {}).get("max_depth", 8) - pw_tree = self._walk(wrapper, "root", 0, max_depth) - except Exception as e: - logger.debug(f"[WindowsAdapter:get_element_tree] pywinauto: {e}") - - if uia_tree is None and pw_tree is None: - return None - if uia_tree is None: - return pw_tree - if pw_tree is None: - return uia_tree - return self._synthesize_trees(uia_tree, pw_tree) - - def _uia_walk(self, hwnd: int) -> Optional[UIElement]: - """Walk the accessibility tree via raw IUIAutomation COM calls. - - pywinauto's children() falls back to EnumChildWindows for HWND-backed - elements (e.g. Chrome_RenderWidgetHostHWND), missing all web content. - Using FindAll(TreeScope_Children) directly on the IUIAutomationElement - always uses UIA and correctly crosses Chromium fragment boundaries. - """ - try: - # pywinauto already initialised comtypes/UIA at import time. - # Retrieve the raw IUIAutomation COM interface from pywinauto's singleton. - from pywinauto.uia_defines import IUIA - _iuia_obj = IUIA() - # pywinauto ≥0.6 exposes it as .iuia; older versions expose it directly. - raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) - - root = raw_uia.ElementFromHandle(hwnd) - true_cond = raw_uia.CreateTrueCondition() - max_depth = self.config.get("tree", {}).get("max_depth", 8) - - # UIA property IDs (UIAutomationClient.h) - _NAME = 30005 - _CTRL_TYPE = 30003 - _ENABLED = 30010 - _FOCUSED = 30008 - _VALUE = 30045 - _ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" - _ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" - _HELP_TEXT = 30013 # tooltip / description - _AUTOMATION_ID = 30011 - _RANGE_VALUE = 30047 - _RANGE_MIN = 30049 - _RANGE_MAX = 30050 - _IS_SELECTED = 30079 - _EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf - _SCOPE_CHILDREN = 0x2 - - def _prop(elem, pid, default=None): - try: - v = elem.GetCurrentPropertyValue(pid) - return v if v is not None else default - except Exception: - return default - - def walk(elem, elem_id: str, depth: int) -> UIElement: - name = _prop(elem, _NAME, "") or "" - ctrl = _prop(elem, _CTRL_TYPE, 0) or 0 - role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") - try: - r = elem.CurrentBoundingRectangle - bounds = Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) - except Exception: - bounds = Bounds(0, 0, 0, 0) - enabled = bool(_prop(elem, _ENABLED, True)) - focused = bool(_prop(elem, _FOCUSED, False)) - value = _prop(elem, _VALUE) or None - # Keyboard shortcut: prefer access key, fall back to accelerator. - ks = _prop(elem, _ACCESS_KEY) or _prop(elem, _ACCEL_KEY) or None - desc = _prop(elem, _HELP_TEXT) or None - aid = _prop(elem, _AUTOMATION_ID) or None - # RangeValue pattern: slider / progress / scrollbar - vn = _prop(elem, _RANGE_VALUE) - vmin = _prop(elem, _RANGE_MIN) - vmax = _prop(elem, _RANGE_MAX) - # SelectionItem.IsSelected: checkbox / radio / tab / menuitem - sel_raw = _prop(elem, _IS_SELECTED) - sel = bool(sel_raw) if sel_raw is not None else None - # ExpandCollapse: combobox / treeitem / menuitem - exp_raw = _prop(elem, _EXPAND_STATE) - if exp_raw in (0, 1): - exp = bool(exp_raw) - else: - exp = None - try: - vn_f = float(vn) if vn is not None else None - except Exception: - vn_f = None - try: - vmin_f = float(vmin) if vmin is not None else None - except Exception: - vmin_f = None - try: - vmax_f = float(vmax) if vmax is not None else None - except Exception: - vmax_f = None - node = UIElement( - element_id=elem_id, name=name, role=role, value=value, - bounds=bounds, enabled=enabled, focused=focused, - keyboard_shortcut=ks or None, - description=desc or None, - selected=sel, expanded=exp, - value_now=vn_f, value_min=vmin_f, value_max=vmax_f, - identifier=str(aid) if aid else None, - ) - if depth < max_depth: - try: - kids = elem.FindAll(_SCOPE_CHILDREN, true_cond) - for i in range(kids.Length): - node.children.append( - walk(kids.GetElement(i), f"{elem_id}.{i}", depth + 1) - ) - except Exception: - pass - return node - - return walk(root, "root", 0) - except Exception as e: - logger.debug(f"[WindowsAdapter:_uia_walk] {e}") - return None - - def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: - try: - try: - rect = wrapper.rectangle() - bounds = Bounds(rect.left, rect.top, - rect.right - rect.left, rect.bottom - rect.top) - except Exception: - bounds = Bounds(0, 0, 0, 0) - - try: - name = wrapper.window_text() or "" - except Exception: - name = "" - - try: - role = wrapper.friendly_class_name() or "Unknown" - except Exception: - role = "Unknown" - - try: - value = wrapper.get_value() if hasattr(wrapper, "get_value") else None - except Exception: - value = None - - try: - enabled = wrapper.is_enabled() - except Exception: - enabled = True - - try: - focused = wrapper.has_keyboard_focus() - except Exception: - focused = False - - elem = UIElement( - element_id=elem_id, name=name, role=role, value=value, - bounds=bounds, enabled=enabled, focused=focused, - ) - - if depth < max_depth: - try: - for i, child in enumerate(wrapper.children()): - elem.children.append( - self._walk(child, f"{elem_id}.{i}", depth + 1, max_depth) - ) - except Exception as ce: - logger.debug(f"[WindowsAdapter:_walk:{elem_id}:children] {ce}") - - return elem - except Exception as e: - print(f"[WindowsAdapter:_walk:{elem_id}] {e}") - traceback.print_exc() - return UIElement(elem_id, "[error]", "Unknown", bounds=Bounds(0, 0, 0, 0)) - - def _synthesize_trees(self, primary: UIElement, secondary: UIElement) -> UIElement: - """Merge two accessibility trees into one richer tree. - - Uses the primary (UIA) tree as the base — it sees web content. - For each node in secondary (pywinauto) matched by bounds, enrich the - primary node with any non-empty properties the primary is missing. - Secondary nodes whose bounds don't appear anywhere in the primary tree - are injected under the deepest primary ancestor that contains them. - """ - # Build a flat index of primary nodes keyed by (x, y, w, h). - bounds_index: Dict[tuple, UIElement] = {} - - def _index(node: UIElement) -> None: - key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) - if key not in bounds_index: - bounds_index[key] = node - for c in node.children: - _index(c) - - _index(primary) - - # Enrich matched nodes; collect unmatched ones with their bounds. - unmatched: List[UIElement] = [] - - def _enrich(node: UIElement) -> None: - key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) - target = bounds_index.get(key) - if target is not None: - # Copy over properties the primary left empty. - if not target.keyboard_shortcut and node.keyboard_shortcut: - target.keyboard_shortcut = node.keyboard_shortcut - if not target.description and node.description: - target.description = node.description - if not target.value and node.value: - target.value = node.value - else: - # Not in primary — keep for injection. - w, h = node.bounds.width, node.bounds.height - if w > 0 and h > 0: # ignore zero-size ghost elements - unmatched.append(node) - for c in node.children: - _enrich(c) - - _enrich(secondary) - - # Inject unmatched secondary nodes under the deepest primary ancestor - # whose bounds contain them (largest-area match wins → most specific). - def _contains(outer: Bounds, inner: Bounds) -> bool: - return (outer.x <= inner.x and outer.y <= inner.y and - outer.x + outer.width >= inner.x + inner.width and - outer.y + outer.height >= inner.y + inner.height) - - for node in unmatched: - best: Optional[UIElement] = None - best_area = float("inf") - for pnode in bounds_index.values(): - if _contains(pnode.bounds, node.bounds): - area = pnode.bounds.width * pnode.bounds.height - if area < best_area: - best_area = area - best = pnode - if best is None: - best = primary - # Renumber the injected element_id to avoid collisions. - node.element_id = f"{best.element_id}.x{len(best.children)}" - best.children.append(node) - - return primary - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - import win32gui - import win32ui - from PIL import Image - import ctypes - - if hwnd is None: - hwnd = win32gui.GetForegroundWindow() - - rect = win32gui.GetWindowRect(hwnd) - width = rect[2] - rect[0] - height = rect[3] - rect[1] - - if width <= 0 or height <= 0: - return None - - hwnd_dc = win32gui.GetWindowDC(hwnd) - mfc_dc = win32ui.CreateDCFromHandle(hwnd_dc) - save_dc = mfc_dc.CreateCompatibleDC() - save_bmp = win32ui.CreateBitmap() - save_bmp.CreateCompatibleBitmap(mfc_dc, width, height) - save_dc.SelectObject(save_bmp) - - # PW_RENDERFULLCONTENT (0x2) renders hardware-accelerated content too - ok = ctypes.windll.user32.PrintWindow(hwnd, save_dc.GetSafeHdc(), 2) - - bmpinfo = save_bmp.GetInfo() - bmpstr = save_bmp.GetBitmapBits(True) - - win32gui.DeleteObject(save_bmp.GetHandle()) - save_dc.DeleteDC() - mfc_dc.DeleteDC() - win32gui.ReleaseDC(hwnd, hwnd_dc) - - if ok: - img = Image.frombuffer( - "RGB", - (bmpinfo["bmWidth"], bmpinfo["bmHeight"]), - bmpstr, "raw", "BGRX", 0, 1, - ) - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - - # PrintWindow failed — fall back to screen-region capture - logger.warning("[WindowsAdapter:get_screenshot] PrintWindow failed; falling back to mss") - raise RuntimeError("PrintWindow returned 0") - - except Exception as e: - logger.debug(f"[WindowsAdapter:get_screenshot] PrintWindow path failed ({e}); trying mss") - try: - import mss - from PIL import Image - import win32gui - - with mss.MSS() as sct: - if hwnd: - rect = win32gui.GetWindowRect(hwnd) - region = {"left": rect[0], "top": rect[1], - "width": rect[2] - rect[0], "height": rect[3] - rect[1]} - else: - region = sct.monitors[1] - raw = sct.grab(region) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e2: - print(f"[WindowsAdapter:get_screenshot] {e2}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id: str = None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True, "action": "type", "text": value} - - elif action == "key" and value: - keys = str(value).lower().split("+") - pyautogui.hotkey(*keys) - return {"success": True, "action": "key", "keys": value} - - elif action == "click_at" and isinstance(value, dict): - pyautogui.click(value["x"], value["y"]) - return {"success": True, "action": "click_at", **value} - - elif action == "scroll" and isinstance(value, dict): - pyautogui.scroll(value.get("clicks", 3), x=value.get("x"), y=value.get("y")) - return {"success": True, "action": "scroll"} - - else: - return {"success": False, "error": f"Unsupported action: {action}"} - except Exception as e: - print(f"[WindowsAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# macOS Adapter (screenshot works; AX tree is a stub pending pyobjc work) -# ───────────────────────────────────────────────────────────────────────────── - -class MacOSAdapter: - def __init__(self, config: dict): - self.config = config - logger.info("[MacOSAdapter:__init__] macOS adapter loaded (AX tree is stub)") - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - return [] # Z-order unavailable without Quartz CGWindowList - - def list_windows(self) -> List[WindowInfo]: - try: - import subprocess - script = ('tell application "System Events" to get name of every ' - 'process whose background only is false') - r = subprocess.run(["osascript", "-e", script], - capture_output=True, text=True, timeout=5) - apps = [a.strip() for a in r.stdout.split(",") if a.strip()] - return [WindowInfo(i, a, a, 0, Bounds(0, 0, 1920, 1080), i == 0, - window_uid=f"mac:{i}") - for i, a in enumerate(apps)] - except Exception as e: - print(f"[MacOSAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - logger.warning("[MacOSAdapter:get_element_tree] Full AX tree requires pyobjc; returning stub") - return UIElement("root", "macOS Application (AX tree stub)", "Window", - bounds=Bounds(0, 0, 1920, 1080)) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[1]) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - print(f"[MacOSAdapter:get_screenshot] {e}") - traceback.print_exc() - return None - - def perform_action(self, action: str, element_id=None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True} - if action == "key" and value: - pyautogui.hotkey(*str(value).lower().split("+")) - return {"success": True} - return {"success": False, "error": f"Unsupported: {action}"} - except Exception as e: - print(f"[MacOSAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# Linux Adapter (screenshot works; AT-SPI tree is stub pending pyatspi work) -# ───────────────────────────────────────────────────────────────────────────── - -class LinuxAdapter: - def __init__(self, config: dict): - self.config = config - logger.info("[LinuxAdapter:__init__] Linux adapter loaded (AT-SPI tree is stub)") - - def get_windows_above_bounds(self, hwnd) -> List[Bounds]: - return [] # Z-order unavailable without Xlib/wnck - - def list_windows(self) -> List[WindowInfo]: - # Prefer wmctrl when present; otherwise fall back to Xlib so the - # adapter still works on systems without the wmctrl binary installed. - import subprocess - try: - r = subprocess.run(["wmctrl", "-lG"], capture_output=True, text=True, timeout=5) - except FileNotFoundError: - logger.info("[LinuxAdapter:list_windows] wmctrl not installed; trying Xlib fallback") - return self._list_windows_xlib() - except Exception as e: - logger.warning("[LinuxAdapter:list_windows] wmctrl failed: %s; trying Xlib fallback", e) - return self._list_windows_xlib() - try: - windows = [] - for line in r.stdout.strip().split("\n"): - if not line: - continue - parts = line.split(None, 8) - if len(parts) < 8: - continue - hwnd = int(parts[0], 16) - x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]) - title = parts[8] if len(parts) > 8 else "(no title)" - windows.append(WindowInfo(hwnd, title, title, 0, - Bounds(x, y, w, h), False, - window_uid=f"x11:{hwnd:x}")) - return windows - except Exception as e: - print(f"[LinuxAdapter:list_windows] {e}") - traceback.print_exc() - return [] - - def _list_windows_xlib(self) -> List[WindowInfo]: - try: - from Xlib import display, X # noqa: F401 - from Xlib.error import XError - except ImportError: - logger.warning( - "[LinuxAdapter:list_windows] neither 'wmctrl' nor python-xlib " - "is available; install one (e.g. `apt install wmctrl` or " - "`pip install python-xlib`) to enumerate windows" - ) - return [] - try: - d = display.Display() - root = d.screen().root - NET_CLIENT_LIST = d.intern_atom("_NET_CLIENT_LIST") - NET_WM_NAME = d.intern_atom("_NET_WM_NAME") - UTF8_STRING = d.intern_atom("UTF8_STRING") - prop = root.get_full_property(NET_CLIENT_LIST, X.AnyPropertyType) - if prop is None: - return [] - windows: List[WindowInfo] = [] - for wid in prop.value: - try: - w = d.create_resource_object("window", wid) - geom = w.get_geometry() - coords = w.translate_coords(root, 0, 0) - x, y = -coords.x, -coords.y - title = "" - name_prop = w.get_full_property(NET_WM_NAME, UTF8_STRING) - if name_prop and name_prop.value: - title = name_prop.value.decode("utf-8", errors="replace") - else: - wm_name = w.get_wm_name() - if wm_name: - title = wm_name if isinstance(wm_name, str) else wm_name.decode("utf-8", errors="replace") - title = title or "(no title)" - windows.append(WindowInfo(int(wid), title, title, 0, - Bounds(x, y, geom.width, geom.height), False, - window_uid=f"x11:{int(wid):x}")) - except XError: - continue - return windows - except Exception as e: - print(f"[LinuxAdapter:_list_windows_xlib] {e}") - traceback.print_exc() - return [] - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - logger.warning("[LinuxAdapter:get_element_tree] Full tree requires pyatspi; returning stub") - return UIElement("root", "Linux Application (AT-SPI stub)", "Window", - bounds=Bounds(0, 0, 1920, 1080)) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - # mss needs a running X server (DISPLAY must be set). - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[1]) - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - logger.debug("[LinuxAdapter:get_screenshot] mss failed: %s; trying scrot", e) - # scrot can write PNG directly to stdout (-z suppresses notifications). - try: - import subprocess - r = subprocess.run(["scrot", "-z", "-"], capture_output=True, timeout=10) - if r.returncode == 0 and r.stdout: - return r.stdout - except Exception as e: - logger.debug("[LinuxAdapter:get_screenshot] scrot failed: %s", e) - return None - - def perform_action(self, action: str, element_id=None, - value: Any = None, hwnd=None) -> Dict: - try: - import pyautogui - if action == "type" and value: - pyautogui.write(str(value), interval=0.02) - return {"success": True} - if action == "key" and value: - pyautogui.hotkey(*str(value).lower().split("+")) - return {"success": True} - return {"success": False, "error": f"Unsupported: {action}"} - except Exception as e: - print(f"[LinuxAdapter:perform_action] {e}") - traceback.print_exc() - return {"success": False, "error": str(e)} - - -# ───────────────────────────────────────────────────────────────────────────── -# WSL Adapter (WSL 1 + WSL 2: X11 when DISPLAY is set, PowerShell fallback) -# ───────────────────────────────────────────────────────────────────────────── - -class WSLAdapter(LinuxAdapter): - """Adapter for Windows Subsystem for Linux. - - Prefers X11-based tools (wmctrl, mss) when DISPLAY is set. Falls back to - PowerShell / cmd.exe interop, which is always available in both WSL 1 and - WSL 2 via the Windows binary execution layer. - """ - - def __init__(self, config: dict) -> None: - self.config = config - self._has_display = bool(os.environ.get("DISPLAY")) - logger.info( - "[WSLAdapter:__init__] WSL detected; " - "DISPLAY=%s", "set" if self._has_display else "not set (PowerShell fallback active)", - ) - - # ── Window listing ──────────────────────────────────────────────────────── - - def list_windows(self) -> List[WindowInfo]: - if self._has_display: - result = LinuxAdapter.list_windows(self) - if result: - return result - return self._list_windows_ps() - - def _list_windows_ps(self) -> List[WindowInfo]: - """Enumerate visible Windows windows via PowerShell ConvertTo-Json.""" - try: - import json - import subprocess - ps = ( - "Get-Process " - "| Where-Object { $_.MainWindowTitle -ne '' } " - "| Select-Object Id,ProcessName,MainWindowTitle " - "| ConvertTo-Json -Compress" - ) - r = subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], - capture_output=True, text=True, timeout=10, - ) - if r.returncode != 0 or not r.stdout.strip(): - return [] - data = json.loads(r.stdout) - if isinstance(data, dict): - data = [data] - results: List[WindowInfo] = [] - for i, item in enumerate(data or []): - pid = int(item.get("Id", 0)) - name = str(item.get("ProcessName", "unknown")) - title = str(item.get("MainWindowTitle", "")) - if not title: - continue - results.append(WindowInfo( - handle=pid, title=title, process_name=name, pid=pid, - bounds=Bounds(0, 0, 1920, 1080), is_focused=(i == 0), - window_uid=f"wsl:{pid}", - )) - return results - except Exception as e: - logger.debug("[WSLAdapter:_list_windows_ps] %s", e) - return [] - - # ── Screenshot ──────────────────────────────────────────────────────────── - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - if self._has_display: - result = LinuxAdapter.get_screenshot(self, hwnd) - if result: - return result - return self._screenshot_ps() - - def _screenshot_ps(self) -> Optional[bytes]: - """Capture the primary screen via PowerShell, returning PNG bytes.""" - try: - import base64 - import subprocess - # Capture screen to a MemoryStream and emit as base64 — avoids - # WSL↔Windows path translation issues entirely. - ps = ( - "Add-Type -AssemblyName System.Windows.Forms,System.Drawing;" - "$b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds;" - "$bmp=New-Object System.Drawing.Bitmap $b.Width,$b.Height;" - "$g=[System.Drawing.Graphics]::FromImage($bmp);" - "$g.CopyFromScreen($b.Location,[System.Drawing.Point]::Empty,$b.Size);" - "$ms=New-Object System.IO.MemoryStream;" - "$bmp.Save($ms,[System.Drawing.Imaging.ImageFormat]::Png);" - "$g.Dispose();$bmp.Dispose();" - "[Convert]::ToBase64String($ms.ToArray())" - ) - r = subprocess.run( - ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], - capture_output=True, text=True, timeout=20, - ) - if r.returncode == 0 and r.stdout.strip(): - return base64.b64decode(r.stdout.strip()) - except Exception as e: - logger.debug("[WSLAdapter:_screenshot_ps] %s", e) - return None - - # ── get_windows_above_bounds: returns [] (inherited from LinuxAdapter) ──── - # ── get_element_tree: upgraded by linux_adapter.install_into if pyatspi ── - # ── perform_action: inherited (pyautogui; needs DISPLAY) ───────────────── - - -# ───────────────────────────────────────────────────────────────────────────── -# ScreenObserver (public interface; delegates to platform adapter) -# ───────────────────────────────────────────────────────────────────────────── - -class ScreenObserver: - """ - Platform-aware screen observer. All consumers should program against - this class rather than the platform adapters directly. - """ - - def __init__(self, config: dict): - self.config = config - self._adapter = self._select_adapter() - # Try to upgrade stub adapters to real AX implementations. - try: - if isinstance(self._adapter, MacOSAdapter): - import mac_adapter - if mac_adapter.install_into(self): - logger.info("[ScreenObserver] mac_adapter installed (pyobjc)") - elif isinstance(self._adapter, LinuxAdapter): - import linux_adapter - if linux_adapter.install_into(self): - logger.info("[ScreenObserver] linux_adapter installed (pyatspi)") - except Exception: - logger.exception("real adapter upgrade failed") - - def _select_adapter(self): - if self.config.get("mock", False): - logger.info("[ScreenObserver] Using MockAdapter") - return MockAdapter() - - target = self.config.get("platform", "auto") - # EFFECTIVE_PLATFORM is "WSL" when running inside WSL, otherwise same - # as platform.system(). Explicit config overrides auto-detection. - sys_plat = EFFECTIVE_PLATFORM if target == "auto" else target - - adapters = { - "Windows": WindowsAdapter, - "Darwin": MacOSAdapter, - "Linux": LinuxAdapter, - "WSL": WSLAdapter, - } - - cls = adapters.get(sys_plat) - if cls is None: - logger.warning("[ScreenObserver] Unknown platform '%s'; using MockAdapter", sys_plat) - return MockAdapter() - - try: - return cls(self.config) - except Exception as e: - print(f"[ScreenObserver:_select_adapter] Platform adapter failed: {e}; falling back to Mock") - traceback.print_exc() - return MockAdapter() - - @property - def is_mock(self) -> bool: - return isinstance(self._adapter, MockAdapter) - - def list_windows(self) -> List[WindowInfo]: - return self._adapter.list_windows() - - def get_element_tree(self, hwnd=None) -> Optional[UIElement]: - return self._adapter.get_element_tree(hwnd) - - def get_screenshot(self, hwnd=None) -> Optional[bytes]: - return self._adapter.get_screenshot(hwnd) - - def get_full_display_screenshot(self) -> Optional[bytes]: - """Capture the entire virtual desktop (all monitors combined) as a PNG.""" - try: - import mss - from PIL import Image - with mss.MSS() as sct: - raw = sct.grab(sct.monitors[0]) # 0 = union of all monitors - img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") - buf = io.BytesIO() - img.save(buf, "PNG") - return buf.getvalue() - except Exception as e: - logger.warning(f"[ScreenObserver:get_full_display_screenshot] {e}; falling back") - return self._adapter.get_screenshot() - - def perform_action(self, action: str, element_id: str = None, - value: Any = None, hwnd=None) -> Dict: - return self._adapter.perform_action(action, element_id, value, hwnd) - - def window_by_index(self, windows: List[WindowInfo], - index: Optional[int]) -> Optional[WindowInfo]: - """Convenience: return a WindowInfo by list index, or None.""" - if index is None or not windows: - return None - if 0 <= index < len(windows): - return windows[index] - return None - - def window_by_uid(self, windows: List[WindowInfo], - uid: Optional[str]) -> Optional[WindowInfo]: - """Resolve a window by stable uid; returns None if not found.""" - if not uid or not windows: - return None - for w in windows: - if w.window_uid == uid: - return w - return None - - def resolve_window(self, windows: List[WindowInfo], - window_uid: Optional[str], - window_index: Optional[int], - window_title: Optional[str] = None) -> "WindowResolution": - """Resolve a window by uid (preferred), index, or title substring.""" - if window_uid: - info = self.window_by_uid(windows, window_uid) - warning = ("both window_index and window_uid given; window_uid used" - if window_index is not None else None) - return WindowResolution(info=info, warning=warning, - used_uid=True, requested_uid=window_uid) - if window_index is not None: - info = self.window_by_index(windows, window_index) - resolved_uid = info.window_uid if info else None - return WindowResolution(info=info, warning=None, - used_uid=bool(resolved_uid), - requested_uid=resolved_uid) - if window_title: - needle = window_title.lower() - info = next((w for w in windows if needle in (w.title or "").lower()), None) - resolved_uid = info.window_uid if info else None - return WindowResolution(info=info, warning=None, - used_uid=bool(resolved_uid), - requested_uid=resolved_uid) - return WindowResolution(info=None, warning=None, - used_uid=False, requested_uid=None) - - # ── Monitors / DPI (design doc §6.3) ────────────────────────────────────── - - def get_monitors(self) -> List[Dict[str, Any]]: - """Return per-monitor metadata via mss.""" - try: - import mss - with mss.MSS() as sct: - mons = sct.monitors # [0] is union; [1..] are individual - out: List[Dict[str, Any]] = [] - for i, m in enumerate(mons[1:]): - out.append({ - "index": i, - "primary": (i == 0), - "bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - "scale_factor": 1.0, - "logical_bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - "physical_bounds": {"x": m["left"], "y": m["top"], - "width": m["width"], "height": m["height"]}, - }) - return out - except Exception: - return [] - - # ── Capability discovery (design doc §6.4) ──────────────────────────────── - - def get_capabilities(self) -> Dict[str, Any]: - adapter_name = type(self._adapter).__name__ - is_windows = adapter_name == "WindowsAdapter" - is_macos = adapter_name == "MacOSAdapter" - is_wsl = adapter_name == "WSLAdapter" - is_linux = adapter_name in ("LinuxAdapter", "WSLAdapter") - is_mock = adapter_name == "MockAdapter" - - # Probe optional libs. - def _has(mod: str) -> bool: - try: - __import__(mod) - return True - except Exception: - return False - - if is_macos: - ax_tree = _has("AppKit") or _has("ApplicationServices") or _has("Cocoa") - elif is_linux: - ax_tree = _has("pyatspi") - else: - ax_tree = is_windows or is_mock - - return { - "ok": True, - "platform": EFFECTIVE_PLATFORM, - "adapter": adapter_name, - "version": (self.config.get("mcp", {}) or {}).get("version", "0.2.0"), - "protocol_version": 2, - "supports": { - "accessibility_tree": bool(ax_tree), - "uia_invoke": is_windows, - "occlusion_detection": is_windows or is_mock or _has("Quartz") or _has("Xlib"), - "drag": True, - "ocr": _has("pytesseract"), - "vlm": bool((self.config.get("vlm") or {}).get("enabled") - and (self.config.get("vlm") or {}).get("model")), - "redaction": True, - "scenarios": is_mock, - "tracing": True, - "replay": True, - "image_blur": _has("PIL"), - "wsl_powershell": is_wsl, - # Action capabilities always present via REST + MCP. - "bring_to_foreground": True, - "element_targeting": bool(ax_tree), # click/focus/invoke/set_value via element_id - "observe_with_diff": True, # /api/observe returns diff token - }, - "config": { - "tree_max_depth": (self.config.get("tree", {}) or {}).get("max_depth", 8), - "ascii_grid": { - "width": (self.config.get("ascii_sketch", {}) or {}).get("grid_width", 110), - "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), - }, - }, - } - - # ── Element occlusion (design doc D14) ──────────────────────────────────── - - def is_element_occluded(self, element_bounds: Bounds, target_hwnd: Any, - all_windows: List[WindowInfo]) -> bool: - """ - True iff every pixel of *element_bounds* is covered by another window - above the target in Z-order, or the element lies entirely off-screen. - - On platforms without Z-order info, returns False (assumed visible). - """ - screen = self.get_screen_bounds() - clipped = _intersect_bounds(element_bounds, screen) - if not clipped: - return True - regions = [clipped] - try: - occluders = self._adapter.get_windows_above_bounds(target_hwnd) - except Exception: - occluders = [] - for occ in occluders: - regions = _subtract_rect(regions, occ) - return not regions - - # ── Visibility helpers ──────────────────────────────────────────────────── - - def get_screen_bounds(self) -> Bounds: - """Return the bounding rect of the combined virtual screen (all monitors).""" - try: - import mss - with mss.MSS() as sct: - m = sct.monitors[0] # index 0 = union of all monitors - return Bounds(m["left"], m["top"], m["width"], m["height"]) - except Exception: - return Bounds(0, 0, 65535, 65535) - - def get_visible_areas(self, target_hwnd: Any, - all_windows: List[WindowInfo]) -> List[Dict]: - """ - Return a list of {x, y, width, height} dicts for the portions of the - window identified by *target_hwnd* that are on-screen and not covered - by windows above it in Z-order. - - On Windows the Z-order is queried precisely via win32gui. - On macOS/Linux Z-order is unavailable, so the full clipped-to-screen - bounds are returned (assuming the window is on top). - """ - target = next((w for w in all_windows if w.handle == target_hwnd), None) - if target is None: - return [] - - screen = self.get_screen_bounds() - clipped = _intersect_bounds(target.bounds, screen) - if not clipped: - return [] - - visible: List[Bounds] = [clipped] - occluders = self._adapter.get_windows_above_bounds(target_hwnd) - for occ in occluders: - visible = _subtract_rect(visible, occ) - - return [b.to_dict() for b in visible] - - def bring_to_foreground(self, target_hwnd: Any, - all_windows: List[WindowInfo]) -> Dict: - """ - Bring a window to the foreground. - - Strategy (in order) - ------------------- - 1. Platform API — SetForegroundWindow / NSRunningApplication.activate / - wmctrl. Does not click, so it cannot accidentally maximise the window - and works even when the window is fully occluded. - 2. Click on title bar — fallback when the API call is unavailable or - blocked (e.g. Windows foreground-lock policy). Requires at least one - visible pixel; returns an error only when this is also impossible. - """ - target = next((w for w in all_windows if w.handle == target_hwnd), None) - if target is None: - return {"success": False, "error": "Window not found"} - - # ── 1. Try platform API first ───────────────────────────────────────── - api_ok, api_note = self._activate_via_api(target_hwnd, target) - if api_ok: - return {"success": True, "action": "activate_api", - "window": target.title, "note": api_note} - - # ── 2. Fall back to clicking the title bar ──────────────────────────── - regions = self.get_visible_areas(target_hwnd, all_windows) - if not regions: - return { - "success": False, - "error": ( - "Window has no visible area (fully occluded or off-screen) " - f"and platform API also failed: {api_note}" - ), - } - - best = min(regions, key=lambda r: (r["y"], -r["width"])) - click_x, click_y = self._title_bar_click_point(target_hwnd, target, best) - result = self.perform_action("click_at", - value={"x": click_x, "y": click_y, - "button": "left", "double": False}) - result["clicked_x"] = click_x - result["clicked_y"] = click_y - result.setdefault("action", "click_title_bar") - result["api_note"] = api_note - return result - - def _activate_via_api(self, hwnd: Any, - info: "WindowInfo") -> tuple: - """ - Attempt to raise *hwnd* using the platform's native window-focus API. - - Returns (success: bool, note: str). Never raises — all errors are - caught and returned as (False, reason). - """ - if PLATFORM == "Windows": - return self._activate_windows(hwnd) - if PLATFORM == "Darwin": - return self._activate_macos(info) - if PLATFORM == "Linux": - # WSL: try X11 tools (wmctrl/xdotool) when DISPLAY is set; they - # will gracefully fail and return (False, reason) when absent. - return self._activate_linux(hwnd) - return False, f"Platform {PLATFORM!r} has no API activate implementation" - - def _activate_windows(self, hwnd: int) -> tuple: - try: - import ctypes - import ctypes.wintypes - user32 = ctypes.windll.user32 - kernel32 = ctypes.windll.kernel32 - - # Restore if minimised (IsIconic) or hidden. - if user32.IsIconic(hwnd): - user32.ShowWindow(hwnd, 9) # SW_RESTORE - - def _is_foreground() -> bool: - return user32.GetForegroundWindow() == hwnd - - if _is_foreground(): - return True, "already foreground" - - # ── Attempt 1: AttachThreadInput trick ──────────────────────────── - fg_hwnd = user32.GetForegroundWindow() - fg_tid = user32.GetWindowThreadProcessId(fg_hwnd, None) - this_tid = kernel32.GetCurrentThreadId() - attached = False - if fg_tid and fg_tid != this_tid: - attached = bool(user32.AttachThreadInput(this_tid, fg_tid, True)) - try: - user32.SetForegroundWindow(hwnd) - finally: - if attached: - user32.AttachThreadInput(this_tid, fg_tid, False) - - if _is_foreground(): - return True, "SetForegroundWindow (AttachThreadInput)" - - # ── Attempt 2: synthesise a keypress to acquire foreground lock ─── - # Windows grants foreground rights to processes that just received - # user input. A zero-vkey keybd_event satisfies that requirement. - KEYEVENTF_KEYUP = 0x0002 - user32.keybd_event(0, 0, 0, 0) - user32.keybd_event(0, 0, KEYEVENTF_KEYUP, 0) - user32.SetForegroundWindow(hwnd) - - if _is_foreground(): - return True, "SetForegroundWindow (keybd_event unlock)" - - # ── Attempt 3: BringWindowToTop + SetForegroundWindow ───────────── - user32.BringWindowToTop(hwnd) - user32.SetForegroundWindow(hwnd) - - if _is_foreground(): - return True, "BringWindowToTop + SetForegroundWindow" - - return False, "SetForegroundWindow failed after all attempts" - except Exception as e: - return False, f"Windows API error: {e}" - - def _activate_macos(self, info: "WindowInfo") -> tuple: - try: - import AppKit - ws = AppKit.NSWorkspace.sharedWorkspace() - apps = ws.runningApplications() - pid = info.pid - for app in apps: - if int(app.processIdentifier()) == pid: - NSApplicationActivateIgnoringOtherApps = 1 << 1 - app.activateWithOptions_(NSApplicationActivateIgnoringOtherApps) - return True, f"NSRunningApplication.activate pid={pid}" - return False, f"No running application found for pid={pid}" - except Exception as e: - return False, f"macOS API error: {e}" - - def _activate_linux(self, hwnd: int) -> tuple: - import subprocess - # Try wmctrl first (X11; commonly available). - try: - r = subprocess.run( - ["wmctrl", "-ia", hex(hwnd)], - capture_output=True, timeout=3, - ) - if r.returncode == 0: - return True, f"wmctrl -ia {hex(hwnd)} succeeded" - except FileNotFoundError: - pass - except Exception as e: - return False, f"wmctrl error: {e}" - # Try xdotool as a second option. - try: - r = subprocess.run( - ["xdotool", "windowactivate", "--sync", str(hwnd)], - capture_output=True, timeout=3, - ) - if r.returncode == 0: - return True, f"xdotool windowactivate {hwnd} succeeded" - return False, f"xdotool returned {r.returncode}: {r.stderr.decode().strip()}" - except FileNotFoundError: - return False, "neither wmctrl nor xdotool found" - except Exception as e: - return False, f"xdotool error: {e}" - - # ── Title-bar targeting (used by bring_to_foreground) ──────────────────── - - # Reserve these pixel margins for native window-control widgets. - _LEFT_CONTROL_MARGIN = 90 # macOS traffic-lights - _RIGHT_CONTROL_MARGIN = 160 # Windows / Linux minimize/maximize/close - _MIN_SAFE_WIDTH = 260 # below this, fall back to centre - - _TITLEBAR_ROLES = {"TitleBar", "AXTitleBar", "title bar", "Title bar", - "AXWindow"} - - def _title_bar_click_point(self, target_hwnd: Any, - window: Optional["WindowInfo"], - region: Dict) -> tuple: - """Return (x, y) for a safe click inside the title bar of *region*.""" - # 1. Try the accessibility tree. - try: - tree = self.get_element_tree(target_hwnd) - except Exception: - tree = None - bar = self._find_title_bar(tree, window.title if window else "") if tree else None - if bar is not None and bar.bounds and bar.bounds.width > 0: - tb = bar.bounds - # Intersect with the visible region so we don't click off-screen. - x1 = max(region["x"], tb.x) - y1 = max(region["y"], tb.y) - x2 = min(region["x"] + region["width"], tb.right) - y2 = min(region["y"] + region["height"], tb.bottom) - if x2 > x1 and y2 > y1: - # Click 1/4 from the left of the title bar's visible width, - # but skip the leftmost control margin. - safe_left = x1 + min(self._LEFT_CONTROL_MARGIN, (x2 - x1) // 4) - safe_right = x2 - min(self._RIGHT_CONTROL_MARGIN, (x2 - x1) // 4) - if safe_right <= safe_left: - cx = (x1 + x2) // 2 - else: - quarter = x1 + (x2 - x1) // 4 - cx = max(safe_left, min(safe_right, quarter)) - cy = y1 + max(1, (y2 - y1) // 2) - return cx, cy - - # 2. Fall back to the visible region's top strip. - rx = region["x"] - rw = region["width"] - ry = region["y"] - rh = region["height"] - if rw < self._MIN_SAFE_WIDTH: - cx = rx + rw // 2 - else: - safe_left = rx + self._LEFT_CONTROL_MARGIN - safe_right = rx + rw - self._RIGHT_CONTROL_MARGIN - candidate = rx + rw // 4 - if safe_right <= safe_left: - cx = rx + rw // 2 - else: - cx = max(safe_left, min(safe_right, candidate)) - title_bar_offset = min(12, max(1, (rh - 1) // 2)) - cy = ry + title_bar_offset - # Clamp inside the region. - cx = max(rx, min(rx + rw - 1, cx)) - cy = max(ry, min(ry + rh - 1, cy)) - return cx, cy - - def _find_title_bar(self, root: Optional["UIElement"], - window_title: str) -> Optional["UIElement"]: - """Locate a title-bar-like element by role or by matching name. - - Returns None for the root window itself (clicking the whole window's - center is precisely what we are trying to avoid). - """ - if root is None: - return None - from collections import deque - queue = deque([(root, 0)]) - while queue: - elem, depth = queue.popleft() - # Never return the root — its centre is the window body, not the - # title bar. - if depth > 0: - if (elem.role or "") in self._TITLEBAR_ROLES: - return elem - if window_title and (elem.name or "").strip() == window_title.strip(): - # Must be near the top edge AND short — a real title bar. - if (elem.bounds and root.bounds - and (elem.bounds.y - root.bounds.y) < 40 - and elem.bounds.height <= 48): - return elem - if depth > 3: - continue - for c in elem.children: - queue.append((c, depth + 1)) - return None - - -# ───────────────────────────────────────────────────────────────────────────── -# Rectangle geometry helpers -# ───────────────────────────────────────────────────────────────────────────── - -def _intersect_bounds(a: Bounds, b: Bounds) -> Optional[Bounds]: - x1 = max(a.x, b.x) - y1 = max(a.y, b.y) - x2 = min(a.right, b.right) - y2 = min(a.bottom, b.bottom) - if x2 <= x1 or y2 <= y1: - return None - return Bounds(x1, y1, x2 - x1, y2 - y1) - - -def _subtract_rect(rects: List[Bounds], occluder: Bounds) -> List[Bounds]: - """Subtract occluder from each rect, splitting into up to 4 sub-rects.""" - result: List[Bounds] = [] - for r in rects: - ix1 = max(r.x, occluder.x) - iy1 = max(r.y, occluder.y) - ix2 = min(r.right, occluder.right) - iy2 = min(r.bottom, occluder.bottom) - - if ix2 <= ix1 or iy2 <= iy1: - result.append(r) - continue - - # Top strip - if iy1 > r.y: - result.append(Bounds(r.x, r.y, r.width, iy1 - r.y)) - # Bottom strip - if iy2 < r.bottom: - result.append(Bounds(r.x, iy2, r.width, r.bottom - iy2)) - # Left strip (height = intersection height) - if ix1 > r.x: - result.append(Bounds(r.x, iy1, ix1 - r.x, iy2 - iy1)) - # Right strip (height = intersection height) - if ix2 < r.right: - result.append(Bounds(ix2, iy1, r.right - ix2, iy2 - iy1)) - return result diff --git a/observer/__init__.py b/observer/__init__.py new file mode 100644 index 0000000..3009ce8 --- /dev/null +++ b/observer/__init__.py @@ -0,0 +1,82 @@ +""" +observer — Core screen observation package (package form of observer.py). + +Provides a platform-aware ScreenObserver that exposes a uniform interface +for: enumerating windows, walking the accessibility element tree, capturing +screenshots, and dispatching input actions. Platform adapters (Windows/macOS/ +Linux/WSL/Mock) share a common protocol and are selected automatically at +runtime. + +Data model +---------- + Bounds — screen-coordinate bounding rectangle + UIElement — one node of the accessibility tree + WindowInfo — top-level window metadata + +P3 decomposition: the implementation now lives in submodules (models, +platform_info, adapters/, core, activation, occlusion). This __init__ +re-exports the entire pre-split public surface so `import observer` / +`from observer import X` keep working unchanged. +""" + +from __future__ import annotations + +from observer.adapters import ( + LinuxAdapter, + MacOSAdapter, + MockAdapter, + WindowsAdapter, + WSLAdapter, + _UIA_ACCEL_KEY, + _UIA_ACCESS_KEY, + _UIA_AUTOMATION_ID, + _UIA_BOUNDING_RECT, + _UIA_CACHED_PROPS, + _UIA_CTRL_TYPE, + _UIA_ENABLED, + _UIA_EXPAND_STATE, + _UIA_FOCUSED, + _UIA_HELP_TEXT, + _UIA_IS_SELECTED, + _UIA_NAME, + _UIA_RANGE_MAX, + _UIA_RANGE_MIN, + _UIA_RANGE_VALUE, + _UIA_SCOPE_CHILDREN, + _UIA_TYPE_TO_ROLE, + _UIA_VALUE, +) +from observer.activation import ActivationMixin +from observer.core import ScreenObserver +from observer.models import ( + Bounds, + UIElement, + WindowInfo, + WindowResolution, + find_element_by_path, + prune_tree_depth, +) +from observer.occlusion import OcclusionMixin, _intersect_bounds, _subtract_rect +from observer.platform_info import EFFECTIVE_PLATFORM, IS_WSL, PLATFORM, _is_wsl + +__all__ = [ + # models + "Bounds", "UIElement", "WindowInfo", "WindowResolution", + "find_element_by_path", "prune_tree_depth", + # platform detection + "EFFECTIVE_PLATFORM", "IS_WSL", "PLATFORM", "_is_wsl", + # adapters + "LinuxAdapter", "MacOSAdapter", "MockAdapter", "WindowsAdapter", + "WSLAdapter", + # UIA constants + "_UIA_ACCEL_KEY", "_UIA_ACCESS_KEY", "_UIA_AUTOMATION_ID", + "_UIA_BOUNDING_RECT", "_UIA_CACHED_PROPS", "_UIA_CTRL_TYPE", + "_UIA_ENABLED", "_UIA_EXPAND_STATE", "_UIA_FOCUSED", "_UIA_HELP_TEXT", + "_UIA_IS_SELECTED", "_UIA_NAME", "_UIA_RANGE_MAX", "_UIA_RANGE_MIN", + "_UIA_RANGE_VALUE", "_UIA_SCOPE_CHILDREN", "_UIA_TYPE_TO_ROLE", + "_UIA_VALUE", + # core + mixins + "ScreenObserver", "ActivationMixin", "OcclusionMixin", + # geometry helpers + "_intersect_bounds", "_subtract_rect", +] diff --git a/observer/activation.py b/observer/activation.py new file mode 100644 index 0000000..60286a5 --- /dev/null +++ b/observer/activation.py @@ -0,0 +1,287 @@ +""" +Window activation: bring_to_foreground, platform API +activation strategies and title-bar click targeting (mixin consumed by +ScreenObserver). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from observer.models import UIElement, WindowInfo +from observer.platform_info import PLATFORM + + +class ActivationMixin: + """Foreground-activation methods of ScreenObserver.""" + + # Provided by the concrete ScreenObserver / OcclusionMixin. + _adapter: Any + + if TYPE_CHECKING: # pragma: no cover — typing-only method stubs + def get_element_tree(self, hwnd: Any = None, + window_uid: Optional[str] = None, + use_cache: bool = True + ) -> Optional[UIElement]: ... + + def get_visible_areas(self, target_hwnd: Any, + all_windows: List[WindowInfo] + ) -> List[Dict]: ... + + def perform_action(self, action: str, + element_id: Optional[str] = None, + value: Any = None, hwnd: Any = None + ) -> Dict: ... + def bring_to_foreground(self, target_hwnd: Any, + all_windows: List[WindowInfo]) -> Dict: + """ + Bring a window to the foreground. + + Strategy (in order) + ------------------- + 1. Platform API — SetForegroundWindow / NSRunningApplication.activate / + wmctrl. Does not click, so it cannot accidentally maximise the window + and works even when the window is fully occluded. + 2. Click on title bar — fallback when the API call is unavailable or + blocked (e.g. Windows foreground-lock policy). Requires at least one + visible pixel; returns an error only when this is also impossible. + """ + target = next((w for w in all_windows if w.handle == target_hwnd), None) + if target is None: + return {"success": False, "error": "Window not found"} + + # ── 1. Try platform API first ───────────────────────────────────────── + api_ok, api_note = self._activate_via_api(target_hwnd, target) + if api_ok: + return {"success": True, "action": "activate_api", + "window": target.title, "note": api_note} + + # ── 2. Fall back to clicking the title bar ──────────────────────────── + regions = self.get_visible_areas(target_hwnd, all_windows) + if not regions: + return { + "success": False, + "error": ( + "Window has no visible area (fully occluded or off-screen) " + f"and platform API also failed: {api_note}" + ), + } + + best = min(regions, key=lambda r: (r["y"], -r["width"])) + click_x, click_y = self._title_bar_click_point(target_hwnd, target, best) + result = self.perform_action("click_at", + value={"x": click_x, "y": click_y, + "button": "left", "double": False}) + result["clicked_x"] = click_x + result["clicked_y"] = click_y + result.setdefault("action", "click_title_bar") + result["api_note"] = api_note + return result + + def _activate_via_api(self, hwnd: Any, + info: "WindowInfo") -> tuple: + """ + Attempt to raise *hwnd* using the platform's native window-focus API. + + Returns (success: bool, note: str). Never raises — all errors are + caught and returned as (False, reason). + """ + if PLATFORM == "Windows": + return self._activate_windows(hwnd) + if PLATFORM == "Darwin": + return self._activate_macos(info) + if PLATFORM == "Linux": + # WSL: try X11 tools (wmctrl/xdotool) when DISPLAY is set; they + # will gracefully fail and return (False, reason) when absent. + return self._activate_linux(hwnd) + return False, f"Platform {PLATFORM!r} has no API activate implementation" + + def _activate_windows(self, hwnd: int) -> tuple: + try: + import ctypes + import ctypes.wintypes + user32 = ctypes.windll.user32 # type: ignore[attr-defined] + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + + # Restore if minimised (IsIconic) or hidden. + if user32.IsIconic(hwnd): + user32.ShowWindow(hwnd, 9) # SW_RESTORE + + def _is_foreground() -> bool: + return user32.GetForegroundWindow() == hwnd + + if _is_foreground(): + return True, "already foreground" + + # ── Attempt 1: AttachThreadInput trick ──────────────────────────── + fg_hwnd = user32.GetForegroundWindow() + fg_tid = user32.GetWindowThreadProcessId(fg_hwnd, None) + this_tid = kernel32.GetCurrentThreadId() + attached = False + if fg_tid and fg_tid != this_tid: + attached = bool(user32.AttachThreadInput(this_tid, fg_tid, True)) + try: + user32.SetForegroundWindow(hwnd) + finally: + if attached: + user32.AttachThreadInput(this_tid, fg_tid, False) + + if _is_foreground(): + return True, "SetForegroundWindow (AttachThreadInput)" + + # ── Attempt 2: synthesise a keypress to acquire foreground lock ─── + # Windows grants foreground rights to processes that just received + # user input. A zero-vkey keybd_event satisfies that requirement. + KEYEVENTF_KEYUP = 0x0002 + user32.keybd_event(0, 0, 0, 0) + user32.keybd_event(0, 0, KEYEVENTF_KEYUP, 0) + user32.SetForegroundWindow(hwnd) + + if _is_foreground(): + return True, "SetForegroundWindow (keybd_event unlock)" + + # ── Attempt 3: BringWindowToTop + SetForegroundWindow ───────────── + user32.BringWindowToTop(hwnd) + user32.SetForegroundWindow(hwnd) + + if _is_foreground(): + return True, "BringWindowToTop + SetForegroundWindow" + + return False, "SetForegroundWindow failed after all attempts" + except Exception as e: + return False, f"Windows API error: {e}" + + def _activate_macos(self, info: "WindowInfo") -> tuple: + try: + import AppKit + ws = AppKit.NSWorkspace.sharedWorkspace() + apps = ws.runningApplications() + pid = info.pid + for app in apps: + if int(app.processIdentifier()) == pid: + NSApplicationActivateIgnoringOtherApps = 1 << 1 + app.activateWithOptions_(NSApplicationActivateIgnoringOtherApps) + return True, f"NSRunningApplication.activate pid={pid}" + return False, f"No running application found for pid={pid}" + except Exception as e: + return False, f"macOS API error: {e}" + + def _activate_linux(self, hwnd: int) -> tuple: + import subprocess + # Try wmctrl first (X11; commonly available). + try: + r = subprocess.run( + ["wmctrl", "-ia", hex(hwnd)], + capture_output=True, timeout=3, + ) + if r.returncode == 0: + return True, f"wmctrl -ia {hex(hwnd)} succeeded" + except FileNotFoundError: + pass + except Exception as e: + return False, f"wmctrl error: {e}" + # Try xdotool as a second option. + try: + r = subprocess.run( + ["xdotool", "windowactivate", "--sync", str(hwnd)], + capture_output=True, timeout=3, + ) + if r.returncode == 0: + return True, f"xdotool windowactivate {hwnd} succeeded" + return False, f"xdotool returned {r.returncode}: {r.stderr.decode().strip()}" + except FileNotFoundError: + return False, "neither wmctrl nor xdotool found" + except Exception as e: + return False, f"xdotool error: {e}" + + # ── Title-bar targeting (used by bring_to_foreground) ──────────────────── + + # Reserve these pixel margins for native window-control widgets. + _LEFT_CONTROL_MARGIN = 90 # macOS traffic-lights + _RIGHT_CONTROL_MARGIN = 160 # Windows / Linux minimize/maximize/close + _MIN_SAFE_WIDTH = 260 # below this, fall back to centre + + _TITLEBAR_ROLES = {"TitleBar", "AXTitleBar", "title bar", "Title bar", + "AXWindow"} + + def _title_bar_click_point(self, target_hwnd: Any, + window: Optional["WindowInfo"], + region: Dict) -> tuple: + """Return (x, y) for a safe click inside the title bar of *region*.""" + # 1. Try the accessibility tree. + try: + tree = self.get_element_tree(target_hwnd) + except Exception: + tree = None + bar = self._find_title_bar(tree, window.title if window else "") if tree else None + if bar is not None and bar.bounds and bar.bounds.width > 0: + tb = bar.bounds + # Intersect with the visible region so we don't click off-screen. + x1 = max(region["x"], tb.x) + y1 = max(region["y"], tb.y) + x2 = min(region["x"] + region["width"], tb.right) + y2 = min(region["y"] + region["height"], tb.bottom) + if x2 > x1 and y2 > y1: + # Click 1/4 from the left of the title bar's visible width, + # but skip the leftmost control margin. + safe_left = x1 + min(self._LEFT_CONTROL_MARGIN, (x2 - x1) // 4) + safe_right = x2 - min(self._RIGHT_CONTROL_MARGIN, (x2 - x1) // 4) + if safe_right <= safe_left: + cx = (x1 + x2) // 2 + else: + quarter = x1 + (x2 - x1) // 4 + cx = max(safe_left, min(safe_right, quarter)) + cy = y1 + max(1, (y2 - y1) // 2) + return cx, cy + + # 2. Fall back to the visible region's top strip. + rx = region["x"] + rw = region["width"] + ry = region["y"] + rh = region["height"] + if rw < self._MIN_SAFE_WIDTH: + cx = rx + rw // 2 + else: + safe_left = rx + self._LEFT_CONTROL_MARGIN + safe_right = rx + rw - self._RIGHT_CONTROL_MARGIN + candidate = rx + rw // 4 + if safe_right <= safe_left: + cx = rx + rw // 2 + else: + cx = max(safe_left, min(safe_right, candidate)) + title_bar_offset = min(12, max(1, (rh - 1) // 2)) + cy = ry + title_bar_offset + # Clamp inside the region. + cx = max(rx, min(rx + rw - 1, cx)) + cy = max(ry, min(ry + rh - 1, cy)) + return cx, cy + + def _find_title_bar(self, root: Optional["UIElement"], + window_title: str) -> Optional["UIElement"]: + """Locate a title-bar-like element by role or by matching name. + + Returns None for the root window itself (clicking the whole window's + center is precisely what we are trying to avoid). + """ + if root is None: + return None + from collections import deque + queue = deque([(root, 0)]) + while queue: + elem, depth = queue.popleft() + # Never return the root — its centre is the window body, not the + # title bar. + if depth > 0: + if (elem.role or "") in self._TITLEBAR_ROLES: + return elem + if window_title and (elem.name or "").strip() == window_title.strip(): + # Must be near the top edge AND short — a real title bar. + if (elem.bounds and root.bounds + and (elem.bounds.y - root.bounds.y) < 40 + and elem.bounds.height <= 48): + return elem + if depth > 3: + continue + for c in elem.children: + queue.append((c, depth + 1)) + return None diff --git a/observer/adapters/__init__.py b/observer/adapters/__init__.py new file mode 100644 index 0000000..7ed2dd8 --- /dev/null +++ b/observer/adapters/__init__.py @@ -0,0 +1,47 @@ +""" +Platform adapters for ScreenObserver. + +Split out of observer.py (P3); behavior is unchanged. The top-level +mac_adapter.py / linux_adapter.py modules still hold the optional +pyobjc / pyatspi runtime upgrades that ScreenObserver installs over the +macOS / Linux stub adapters defined here. +""" + +from __future__ import annotations + +from observer.adapters.linux import LinuxAdapter +from observer.adapters.macos import MacOSAdapter +from observer.adapters.mock import MockAdapter +from observer.adapters.windows import ( + WindowsAdapter, + _UIA_ACCEL_KEY, + _UIA_ACCESS_KEY, + _UIA_AUTOMATION_ID, + _UIA_BOUNDING_RECT, + _UIA_CACHED_PROPS, + _UIA_CTRL_TYPE, + _UIA_ENABLED, + _UIA_EXPAND_STATE, + _UIA_FOCUSED, + _UIA_HELP_TEXT, + _UIA_IS_SELECTED, + _UIA_NAME, + _UIA_RANGE_MAX, + _UIA_RANGE_MIN, + _UIA_RANGE_VALUE, + _UIA_SCOPE_CHILDREN, + _UIA_TYPE_TO_ROLE, + _UIA_VALUE, +) +from observer.adapters.wsl import WSLAdapter + +__all__ = [ + "LinuxAdapter", "MacOSAdapter", "MockAdapter", "WindowsAdapter", + "WSLAdapter", + "_UIA_ACCEL_KEY", "_UIA_ACCESS_KEY", "_UIA_AUTOMATION_ID", + "_UIA_BOUNDING_RECT", "_UIA_CACHED_PROPS", "_UIA_CTRL_TYPE", + "_UIA_ENABLED", "_UIA_EXPAND_STATE", "_UIA_FOCUSED", "_UIA_HELP_TEXT", + "_UIA_IS_SELECTED", "_UIA_NAME", "_UIA_RANGE_MAX", "_UIA_RANGE_MIN", + "_UIA_RANGE_VALUE", "_UIA_SCOPE_CHILDREN", "_UIA_TYPE_TO_ROLE", + "_UIA_VALUE", +] diff --git a/observer/adapters/linux.py b/observer/adapters/linux.py new file mode 100644 index 0000000..ff74605 --- /dev/null +++ b/observer/adapters/linux.py @@ -0,0 +1,147 @@ +""" +Linux adapter (screenshot works; AT-SPI tree is a stub +pending pyatspi work — upgraded at runtime by top-level linux_adapter.py). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, UIElement, WindowInfo + +logger = logging.getLogger(__name__) + + +class LinuxAdapter: + def __init__(self, config: dict): + self.config = config + logger.info("[LinuxAdapter:__init__] Linux adapter loaded (AT-SPI tree is stub)") + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + return [] # Z-order unavailable without Xlib/wnck + + def list_windows(self) -> List[WindowInfo]: + # Prefer wmctrl when present; otherwise fall back to Xlib so the + # adapter still works on systems without the wmctrl binary installed. + import subprocess + try: + r = subprocess.run(["wmctrl", "-lG"], capture_output=True, text=True, timeout=5) + except FileNotFoundError: + logger.info("[LinuxAdapter:list_windows] wmctrl not installed; trying Xlib fallback") + return self._list_windows_xlib() + except Exception as e: + logger.warning("[LinuxAdapter:list_windows] wmctrl failed: %s; trying Xlib fallback", e) + return self._list_windows_xlib() + try: + windows = [] + for line in r.stdout.strip().split("\n"): + if not line: + continue + parts = line.split(None, 8) + if len(parts) < 8: + continue + hwnd = int(parts[0], 16) + x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5]) + title = parts[8] if len(parts) > 8 else "(no title)" + windows.append(WindowInfo(hwnd, title, title, 0, + Bounds(x, y, w, h), False, + window_uid=f"x11:{hwnd:x}")) + return windows + except Exception as e: + print(f"[LinuxAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def _list_windows_xlib(self) -> List[WindowInfo]: + try: + from Xlib import display, X # noqa: F401 + from Xlib.error import XError + except ImportError: + logger.warning( + "[LinuxAdapter:list_windows] neither 'wmctrl' nor python-xlib " + "is available; install one (e.g. `apt install wmctrl` or " + "`pip install python-xlib`) to enumerate windows" + ) + return [] + try: + d = display.Display() + root = d.screen().root + NET_CLIENT_LIST = d.intern_atom("_NET_CLIENT_LIST") + NET_WM_NAME = d.intern_atom("_NET_WM_NAME") + UTF8_STRING = d.intern_atom("UTF8_STRING") + prop = root.get_full_property(NET_CLIENT_LIST, X.AnyPropertyType) + if prop is None: + return [] + windows: List[WindowInfo] = [] + for wid in prop.value: + try: + w = d.create_resource_object("window", wid) + geom = w.get_geometry() + coords = w.translate_coords(root, 0, 0) + x, y = -coords.x, -coords.y + title = "" + name_prop = w.get_full_property(NET_WM_NAME, UTF8_STRING) + if name_prop and name_prop.value: + title = name_prop.value.decode("utf-8", errors="replace") + else: + wm_name = w.get_wm_name() + if wm_name: + title = wm_name if isinstance(wm_name, str) else wm_name.decode("utf-8", errors="replace") + title = title or "(no title)" + windows.append(WindowInfo(int(wid), title, title, 0, + Bounds(x, y, geom.width, geom.height), False, + window_uid=f"x11:{int(wid):x}")) + except XError: + continue + return windows + except Exception as e: + print(f"[LinuxAdapter:_list_windows_xlib] {e}") + traceback.print_exc() + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + logger.warning("[LinuxAdapter:get_element_tree] Full tree requires pyatspi; returning stub") + return UIElement("root", "Linux Application (AT-SPI stub)", "Window", + bounds=Bounds(0, 0, 1920, 1080)) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + # mss needs a running X server (DISPLAY must be set). + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[1]) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + logger.debug("[LinuxAdapter:get_screenshot] mss failed: %s; trying scrot", e) + # scrot can write PNG directly to stdout (-z suppresses notifications). + try: + import subprocess + r = subprocess.run(["scrot", "-z", "-"], capture_output=True, timeout=10) + if r.returncode == 0 and r.stdout: + return r.stdout + except Exception as e: + logger.debug("[LinuxAdapter:get_screenshot] scrot failed: %s", e) + return None + + def perform_action(self, action: str, element_id=None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True} + if action == "key" and value: + pyautogui.hotkey(*str(value).lower().split("+")) + return {"success": True} + return {"success": False, "error": f"Unsupported: {action}"} + except Exception as e: + print(f"[LinuxAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/macos.py b/observer/adapters/macos.py new file mode 100644 index 0000000..8c83cd8 --- /dev/null +++ b/observer/adapters/macos.py @@ -0,0 +1,76 @@ +""" +macOS adapter (screenshot works; AX tree is a stub +pending pyobjc work — upgraded at runtime by top-level mac_adapter.py). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, UIElement, WindowInfo + +logger = logging.getLogger(__name__) + + +class MacOSAdapter: + def __init__(self, config: dict): + self.config = config + logger.info("[MacOSAdapter:__init__] macOS adapter loaded (AX tree is stub)") + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + return [] # Z-order unavailable without Quartz CGWindowList + + def list_windows(self) -> List[WindowInfo]: + try: + import subprocess + script = ('tell application "System Events" to get name of every ' + 'process whose background only is false') + r = subprocess.run(["osascript", "-e", script], + capture_output=True, text=True, timeout=5) + apps = [a.strip() for a in r.stdout.split(",") if a.strip()] + return [WindowInfo(i, a, a, 0, Bounds(0, 0, 1920, 1080), i == 0, + window_uid=f"mac:{i}") + for i, a in enumerate(apps)] + except Exception as e: + print(f"[MacOSAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + logger.warning("[MacOSAdapter:get_element_tree] Full AX tree requires pyobjc; returning stub") + return UIElement("root", "macOS Application (AX tree stub)", "Window", + bounds=Bounds(0, 0, 1920, 1080)) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[1]) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + print(f"[MacOSAdapter:get_screenshot] {e}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id=None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True} + if action == "key" and value: + pyautogui.hotkey(*str(value).lower().split("+")) + return {"success": True} + return {"success": False, "error": f"Unsupported: {action}"} + except Exception as e: + print(f"[MacOSAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/mock.py b/observer/adapters/mock.py new file mode 100644 index 0000000..9fd995d --- /dev/null +++ b/observer/adapters/mock.py @@ -0,0 +1,184 @@ +""" +Mock adapter (no OS dependencies; safe anywhere). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import traceback +from typing import Any, Callable, Dict, List, Optional + +from observer.models import ( + Bounds, UIElement, WindowInfo, find_element_by_path, prune_tree_depth, +) + + +class MockAdapter: + """Synthetic data adapter for development and testing.""" + + def __init__(self) -> None: + import secrets as _s + self._nonce = _s.token_hex(4) + # Optional scenario hook (design doc §15.5). Set by main.py when + # --scenario is supplied; methods route through the scenario when + # active so that input actions can drive state transitions. + self.scenario: Optional[Any] = None + # Test hooks (P1 perf work): capture_count increments on every tree + # walk so tests can assert cache hits avoided adapter work; + # tree_mutator, when set, post-processes (or replaces) each captured + # tree so tests can simulate UI changes between captures. + self.capture_count: int = 0 + self.tree_mutator: Optional[Callable[[UIElement], UIElement]] = None + + def get_windows_above_bounds(self, hwnd) -> List["Bounds"]: + return [] # Mock assumes the target window is on top + + def list_windows(self) -> List[WindowInfo]: + if self.scenario is not None: + return self.scenario.list_windows(self._nonce) + return [ + WindowInfo(1001, "Untitled — Notepad", "notepad.exe", 1234, + Bounds(80, 60, 800, 600), True, + window_uid=f"mock:0:{self._nonce}"), + WindowInfo(1002, "GitHub · Where software is built — Google Chrome", + "chrome.exe", 5678, Bounds(0, 0, 1920, 1050), False, + window_uid=f"mock:1:{self._nonce}"), + WindowInfo(1003, "screen_observer.py — Visual Studio Code", + "code.exe", 9012, Bounds(960, 0, 960, 1050), False, + window_uid=f"mock:2:{self._nonce}"), + ] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + self.capture_count += 1 + tree = self._build_tree(hwnd) + if tree is not None and self.tree_mutator is not None: + mutated = self.tree_mutator(tree) + if mutated is not None: + tree = mutated + return tree + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path*, to *max_depth* + levels below it. The mock world is synthetic, so this navigates the + positional element-id path of a fresh capture.""" + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + def _build_tree(self, hwnd=None) -> Optional[UIElement]: + if self.scenario is not None: + return self.scenario.get_element_tree(hwnd) + root = UIElement("root", "Untitled — Notepad", "Window", + bounds=Bounds(80, 60, 800, 600)) + + # Menu bar + menubar = UIElement("root.0", "MenuBar", "MenuBar", + bounds=Bounds(80, 60, 800, 22)) + for i, lbl in enumerate(["File", "Edit", "Format", "View", "Help"]): + menubar.children.append(UIElement( + f"root.0.{i}", lbl, "MenuItem", + bounds=Bounds(80 + i * 58, 60, 56, 22) + )) + root.children.append(menubar) + + # Main editing area + editor = UIElement( + "root.1", "Text Editor", "Document", + value="Hello, world!\nThis is a test document.\nLine 3 has some content here.\n", + bounds=Bounds(80, 82, 800, 514), focused=True + ) + root.children.append(editor) + + # Horizontal scroll bar + hscroll = UIElement("root.2", "Horizontal ScrollBar", "ScrollBar", + bounds=Bounds(80, 596, 784, 18)) + root.children.append(hscroll) + + # Vertical scroll bar + vscroll = UIElement("root.3", "Vertical ScrollBar", "ScrollBar", + bounds=Bounds(864, 82, 18, 532)) + root.children.append(vscroll) + + # Progress bar — exercises value_now/min/max for the role glyph path. + root.children.append(UIElement( + "root.5", "Saving", "ProgressBar", + bounds=Bounds(560, 600, 200, 10), + value_now=40.0, value_min=0.0, value_max=100.0, + )) + + # Word-wrap toggle checkbox — exercises selected=True path. + root.children.append(UIElement( + "root.6", "Word Wrap", "CheckBox", + bounds=Bounds(80, 636, 120, 18), selected=True, + )) + + # Status bar + sb = UIElement("root.4", "Status Bar", "StatusBar", + bounds=Bounds(80, 614, 800, 22)) + for i, (lbl, val) in enumerate([ + ("Position", "Ln 1, Col 1"), + ("Zoom", "100%"), + ("Encoding", "UTF-8"), + ("EOL", "Windows (CRLF)"), + ]): + sb.children.append(UIElement( + f"root.4.{i}", lbl, "Text", value=val, + bounds=Bounds(80 + i * 190, 614, 188, 22) + )) + root.children.append(sb) + + return root + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + from PIL import Image, ImageDraw + img = Image.new("RGB", (800, 600), "#1a1e2e") + draw = ImageDraw.Draw(img) + + # Title bar + draw.rectangle([0, 0, 800, 30], fill="#2d3250") + draw.text((10, 8), "Untitled — Notepad", fill="#c8d3f5") + + # Menu bar + draw.rectangle([0, 30, 800, 52], fill="#1e2030") + for i, m in enumerate(["File", "Edit", "Format", "View", "Help"]): + draw.text((10 + i * 58, 36), m, fill="#a9b1d6") + + # Editor area + draw.rectangle([0, 52, 782, 570], fill="#1a1e2e") + for i, ln in enumerate(["Hello, world!", "This is a test document.", + "Line 3 has some content here.", ""]): + draw.text((6, 58 + i * 18), ln, fill="#c0caf5") + + # Scrollbars + draw.rectangle([782, 52, 800, 570], fill="#24283b") + draw.rectangle([0, 570, 782, 586], fill="#24283b") + + # Status bar + draw.rectangle([0, 586, 800, 600], fill="#16161e") + draw.text((6, 589), "Ln 1, Col 1 100% UTF-8 Windows (CRLF)", + fill="#565f89") + + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + print(f"[MockAdapter:get_screenshot] {e}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + if self.scenario is not None: + handled = self.scenario.handle_action(action=action, + element_id=element_id, + value=value, hwnd=hwnd) + if handled is not None: + return handled + return { + "success": True, + "action": action, + "note": "Mock adapter — no real OS action performed", + } diff --git a/observer/adapters/windows.py b/observer/adapters/windows.py new file mode 100644 index 0000000..c1f866a --- /dev/null +++ b/observer/adapters/windows.py @@ -0,0 +1,640 @@ +""" +Windows UIA adapter (pywinauto, pywin32, raw COM walker). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import io +import logging +import traceback +from typing import Any, Dict, List, Optional + +from observer.models import ( + Bounds, UIElement, WindowInfo, find_element_by_path, prune_tree_depth, +) + +logger = logging.getLogger(__name__) + + +# UIA control-type ID → pywinauto-compatible role name. +# Values from UIAutomationClient.h; must match what selectors/descriptions expect. +_UIA_TYPE_TO_ROLE: Dict[int, str] = { + 50000: "Button", 50001: "Calendar", 50002: "CheckBox", + 50003: "ComboBox", 50004: "Edit", 50005: "Hyperlink", + 50006: "Image", 50007: "ListItem", 50008: "ListBox", + 50009: "Menu", 50010: "MenuBar", 50011: "MenuItem", + 50012: "ProgressBar", 50013: "RadioButton", 50014: "ScrollBar", + 50015: "Slider", 50016: "Spinner", 50017: "StatusBar", + 50018: "TabControl", 50019: "TabItem", 50020: "Text", + 50021: "Toolbar", 50022: "ToolTip", 50023: "Tree", + 50024: "TreeItem", 50025: "Custom", 50026: "GroupBox", + 50027: "Thumb", 50028: "DataGrid", 50029: "DataItem", + 50030: "Document", 50031: "SplitButton", 50032: "Dialog", + 50033: "Pane", 50034: "Header", 50035: "HeaderItem", + 50036: "Table", 50037: "TitleBar", 50038: "Separator", + 50039: "SemanticZoom",50040: "AppBar", +} + +# UIA property IDs (UIAutomationClient.h) used by the raw-COM walker. +_UIA_BOUNDING_RECT = 30001 +_UIA_NAME = 30005 +_UIA_CTRL_TYPE = 30003 +_UIA_ENABLED = 30010 +_UIA_FOCUSED = 30008 +_UIA_VALUE = 30045 +_UIA_ACCESS_KEY = 30023 # keyboard mnemonic, e.g. "Alt+F" +_UIA_ACCEL_KEY = 30022 # accelerator, e.g. "Ctrl+Z" +_UIA_HELP_TEXT = 30013 # tooltip / description +_UIA_AUTOMATION_ID = 30011 +_UIA_RANGE_VALUE = 30047 +_UIA_RANGE_MIN = 30049 +_UIA_RANGE_MAX = 30050 +_UIA_IS_SELECTED = 30079 +_UIA_EXPAND_STATE = 30084 # 0=collapsed 1=expanded 2=partial 3=leaf +_UIA_SCOPE_CHILDREN = 0x2 + +# Properties bulk-fetched via a UIA CacheRequest so each level of the walk is +# one COM round trip (FindAllBuildCache) instead of ~15 per node. +_UIA_CACHED_PROPS = ( + _UIA_BOUNDING_RECT, _UIA_NAME, _UIA_CTRL_TYPE, _UIA_ENABLED, + _UIA_FOCUSED, _UIA_VALUE, _UIA_ACCESS_KEY, _UIA_ACCEL_KEY, + _UIA_HELP_TEXT, _UIA_AUTOMATION_ID, _UIA_RANGE_VALUE, _UIA_RANGE_MIN, + _UIA_RANGE_MAX, _UIA_IS_SELECTED, _UIA_EXPAND_STATE, +) + + +class WindowsAdapter: + """Full Windows UIA adapter using pywinauto + pywin32.""" + + def __init__(self, config: dict): + self.config = config + try: + import win32gui # noqa: F401 + import win32process # noqa: F401 + import psutil # noqa: F401 + from pywinauto import Application # noqa: F401 + self._Application = Application + logger.info("[WindowsAdapter:__init__] pywinauto/pywin32 ready") + except ImportError as e: + print(f"[WindowsAdapter:__init__] Missing dependency: {e}") + traceback.print_exc() + raise + + def list_windows(self) -> List[WindowInfo]: + try: + import win32gui + import win32process + import psutil + + results: List[WindowInfo] = [] + fg = win32gui.GetForegroundWindow() + + def _cb(hwnd, _): + try: + if not win32gui.IsWindowVisible(hwnd): + return + title = win32gui.GetWindowText(hwnd) + if not title: + return + rect = win32gui.GetWindowRect(hwnd) + w, h = rect[2] - rect[0], rect[3] - rect[1] + if w <= 0 or h <= 0: + return + try: + _, pid = win32process.GetWindowThreadProcessId(hwnd) + proc_name = psutil.Process(pid).name() + except Exception: + pid, proc_name = 0, "unknown" + results.append(WindowInfo( + handle=hwnd, title=title, process_name=proc_name, pid=pid, + bounds=Bounds(rect[0], rect[1], w, h), + is_focused=(hwnd == fg), + window_uid=f"win:{pid}:{hwnd}", + )) + except Exception as inner: + logger.debug(f"[WindowsAdapter:list_windows:_cb] {inner}") + + win32gui.EnumWindows(_cb, None) + return sorted(results, key=lambda w: (not w.is_focused, w.title.lower())) + except Exception as e: + print(f"[WindowsAdapter:list_windows] {e}") + traceback.print_exc() + return [] + + def get_windows_above_bounds(self, hwnd) -> List[Bounds]: + """Return bounds of visible windows that are above hwnd in Z-order.""" + try: + import win32gui + GW_HWNDNEXT = 2 + above: List[Bounds] = [] + h = win32gui.GetTopWindow(None) + while h and h != hwnd: + try: + if win32gui.IsWindowVisible(h): + rect = win32gui.GetWindowRect(h) + w = rect[2] - rect[0] + hh = rect[3] - rect[1] + if w > 0 and hh > 0: + above.append(Bounds(rect[0], rect[1], w, hh)) + except Exception as e: + logger.debug(f"[occlusion] window rect probe failed " + f"for hwnd {h}: {e}") + try: + h = win32gui.GetWindow(h, GW_HWNDNEXT) + except Exception: + break + return above + except Exception as e: + logger.debug(f"[WindowsAdapter:get_windows_above_bounds] {e}") + return [] + + def get_element_tree(self, hwnd=None) -> Optional[UIElement]: + try: + import win32gui + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + except Exception as e: + print(f"[WindowsAdapter:get_element_tree] win32gui: {e}") + return None + + # Run both walkers and synthesise: UIA crosses Chromium fragment boundaries + # (gets web content); pywinauto may surface native-control properties that + # UIA omits. The merged result gives the LLM everything either source sees. + # tree.strategy == "uia_only" skips the second pywinauto walk and the + # synthesis pass entirely — roughly halving capture time — at the cost + # of the extra native-control properties the merge would contribute. + strategy = str(self.config.get("tree", {}).get("strategy", + "merged")).lower() + uia_tree = self._uia_walk(hwnd) + if strategy == "uia_only" and uia_tree is not None: + return uia_tree + pw_tree = None + try: + app = self._Application(backend="uia").connect(handle=hwnd) + window = app.window(handle=hwnd) + wrapper = window.wrapper_object() + max_depth = self.config.get("tree", {}).get("max_depth", 8) + pw_tree = self._walk(wrapper, "root", 0, max_depth) + except Exception as e: + logger.debug(f"[WindowsAdapter:get_element_tree] pywinauto: {e}") + + if uia_tree is None and pw_tree is None: + return None + if uia_tree is None: + return pw_tree + if pw_tree is None: + return uia_tree + return self._synthesize_trees(uia_tree, pw_tree) + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None + ) -> Optional[UIElement]: + """Walk only the subtree rooted at *element_path* via raw UIA. + + Navigates the positional child indices of the element-id path + ('root.3.2' → child 3 → child 2) so only the requested branch is + traversed. Falls back to a full walk plus extraction when the path + contains non-positional segments (synthesized ids) or navigation + fails.""" + try: + import win32gui + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + except Exception as e: + print(f"[WindowsAdapter:get_element_subtree] win32gui: {e}") + return None + + if max_depth is None: + max_depth = self.config.get("tree", {}).get("max_depth", 8) + + indices = self._parse_positional_path(element_path) + if indices is not None: + try: + raw_uia, true_cond = self._uia_handles() + elem = raw_uia.ElementFromHandle(hwnd) + for idx in indices: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + if idx < 0 or idx >= kids.Length: + elem = None + break + elem = kids.GetElement(idx) + if elem is not None: + return self._uia_walk_element( + elem, element_path, 0, max_depth, true_cond, + cache_request=self._uia_cache_request(raw_uia)) + except Exception as e: + logger.debug(f"[WindowsAdapter:get_element_subtree] " + f"navigation failed ({e}); falling back") + + # Fallback: full walk + extraction. + tree = self.get_element_tree(hwnd) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + @staticmethod + def _parse_positional_path(element_path: str) -> Optional[List[int]]: + """'root.3.2' → [3, 2]; None when the path is not purely positional.""" + segs = (element_path or "").split(".") + if not segs or segs[0] != "root": + return None + try: + return [int(s) for s in segs[1:]] + except ValueError: + return None # synthesized ids like 'root.2.x1' + + def _uia_handles(self): + """Return (raw IUIAutomation interface, true condition). + + pywinauto already initialised comtypes/UIA at import time; retrieve + the raw COM interface from its singleton (pywinauto ≥0.6 exposes it + as .iuia; older versions expose it directly).""" + from pywinauto.uia_defines import IUIA + _iuia_obj = IUIA() + raw_uia = getattr(_iuia_obj, "iuia", _iuia_obj) + return raw_uia, raw_uia.CreateTrueCondition() + + def _uia_walk(self, hwnd: int) -> Optional[UIElement]: + """Walk the accessibility tree via raw IUIAutomation COM calls. + + pywinauto's children() falls back to EnumChildWindows for HWND-backed + elements (e.g. Chrome_RenderWidgetHostHWND), missing all web content. + Using FindAll(TreeScope_Children) directly on the IUIAutomationElement + always uses UIA and correctly crosses Chromium fragment boundaries. + + When the COM API supports it, a CacheRequest bulk-fetches the walked + properties per level (FindAllBuildCache + GetCachedPropertyValue) — + one cross-process round trip per node's children instead of ~15 per + node. Any CacheRequest failure falls back to the per-property path. + """ + try: + raw_uia, true_cond = self._uia_handles() + root = raw_uia.ElementFromHandle(hwnd) + max_depth = self.config.get("tree", {}).get("max_depth", 8) + cache_request = self._uia_cache_request(raw_uia) + return self._uia_walk_element(root, "root", 0, max_depth, + true_cond, + cache_request=cache_request) + except Exception as e: + logger.debug(f"[WindowsAdapter:_uia_walk] {e}") + return None + + @staticmethod + def _uia_cache_request(raw_uia): + """Build a CacheRequest covering the properties the walker reads. + Returns None (per-property fallback) when construction fails.""" + try: + cr = raw_uia.CreateCacheRequest() + for pid in _UIA_CACHED_PROPS: + cr.AddProperty(pid) + return cr + except Exception as e: + logger.debug(f"[WindowsAdapter:_uia_cache_request] CacheRequest " + f"unavailable ({e}); using per-property fetches") + return None + + @staticmethod + def _uia_prop(elem, pid, default=None, cached=False): + if cached: + try: + v = elem.GetCachedPropertyValue(pid) + return v if v is not None else default + except Exception: + pass # cache miss/failure — fall back to a live fetch + try: + v = elem.GetCurrentPropertyValue(pid) + return v if v is not None else default + except Exception: + return default + + @staticmethod + def _uia_bounds(elem, cached=False) -> Bounds: + if cached: + try: + r = elem.CachedBoundingRectangle + return Bounds(r.left, r.top, + r.right - r.left, r.bottom - r.top) + except Exception: + try: + arr = elem.GetCachedPropertyValue(_UIA_BOUNDING_RECT) + if arr is not None and len(arr) == 4: + # VT_R8 SAFEARRAY: [left, top, width, height] + return Bounds(int(arr[0]), int(arr[1]), + int(arr[2]), int(arr[3])) + except Exception: + pass + try: + r = elem.CurrentBoundingRectangle + return Bounds(r.left, r.top, r.right - r.left, r.bottom - r.top) + except Exception: + return Bounds(0, 0, 0, 0) + + def _uia_walk_element(self, elem, elem_id: str, depth: int, + max_depth: int, true_cond, + cache_request=None, cached: bool = False + ) -> UIElement: + """Build a UIElement for *elem* and recurse into its children. + + *cached* means this element was fetched via FindAllBuildCache and its + properties can be read with GetCachedPropertyValue (no round trip). + """ + def _prop(pid, default=None): + return self._uia_prop(elem, pid, default, cached=cached) + + name = _prop(_UIA_NAME, "") or "" + ctrl = _prop(_UIA_CTRL_TYPE, 0) or 0 + role = _UIA_TYPE_TO_ROLE.get(ctrl, "Pane") + bounds = self._uia_bounds(elem, cached=cached) + enabled = bool(_prop(_UIA_ENABLED, True)) + focused = bool(_prop(_UIA_FOCUSED, False)) + value = _prop(_UIA_VALUE) or None + # Keyboard shortcut: prefer access key, fall back to accelerator. + ks = _prop(_UIA_ACCESS_KEY) or _prop(_UIA_ACCEL_KEY) or None + desc = _prop(_UIA_HELP_TEXT) or None + aid = _prop(_UIA_AUTOMATION_ID) or None + # RangeValue pattern: slider / progress / scrollbar + vn = _prop(_UIA_RANGE_VALUE) + vmin = _prop(_UIA_RANGE_MIN) + vmax = _prop(_UIA_RANGE_MAX) + # SelectionItem.IsSelected: checkbox / radio / tab / menuitem + sel_raw = _prop(_UIA_IS_SELECTED) + sel = bool(sel_raw) if sel_raw is not None else None + # ExpandCollapse: combobox / treeitem / menuitem + exp_raw = _prop(_UIA_EXPAND_STATE) + if exp_raw in (0, 1): + exp = bool(exp_raw) + else: + exp = None + try: + vn_f = float(vn) if vn is not None else None + except Exception: + vn_f = None + try: + vmin_f = float(vmin) if vmin is not None else None + except Exception: + vmin_f = None + try: + vmax_f = float(vmax) if vmax is not None else None + except Exception: + vmax_f = None + node = UIElement( + element_id=elem_id, name=name, role=role, value=value, + bounds=bounds, enabled=enabled, focused=focused, + keyboard_shortcut=ks or None, + description=desc or None, + selected=sel, expanded=exp, + value_now=vn_f, value_min=vmin_f, value_max=vmax_f, + identifier=str(aid) if aid else None, + ) + if depth < max_depth: + kids = None + kids_cached = False + if cache_request is not None: + try: + kids = elem.FindAllBuildCache(_UIA_SCOPE_CHILDREN, + true_cond, cache_request) + kids_cached = True + except Exception: + kids = None # per-node fallback below + if kids is None: + try: + kids = elem.FindAll(_UIA_SCOPE_CHILDREN, true_cond) + except Exception: + kids = None + if kids is not None: + try: + for i in range(kids.Length): + node.children.append(self._uia_walk_element( + kids.GetElement(i), f"{elem_id}.{i}", + depth + 1, max_depth, true_cond, + cache_request=cache_request, + cached=kids_cached)) + except Exception as e: + logger.debug(f"[uia] child walk truncated at " + f"{elem_id}: {e}") + return node + + def _walk(self, wrapper, elem_id: str, depth: int, max_depth: int) -> UIElement: + try: + try: + rect = wrapper.rectangle() + bounds = Bounds(rect.left, rect.top, + rect.right - rect.left, rect.bottom - rect.top) + except Exception: + bounds = Bounds(0, 0, 0, 0) + + try: + name = wrapper.window_text() or "" + except Exception: + name = "" + + try: + role = wrapper.friendly_class_name() or "Unknown" + except Exception: + role = "Unknown" + + try: + value = wrapper.get_value() if hasattr(wrapper, "get_value") else None + except Exception: + value = None + + try: + enabled = wrapper.is_enabled() + except Exception: + enabled = True + + try: + focused = wrapper.has_keyboard_focus() + except Exception: + focused = False + + elem = UIElement( + element_id=elem_id, name=name, role=role, value=value, + bounds=bounds, enabled=enabled, focused=focused, + ) + + if depth < max_depth: + try: + for i, child in enumerate(wrapper.children()): + elem.children.append( + self._walk(child, f"{elem_id}.{i}", depth + 1, max_depth) + ) + except Exception as ce: + logger.debug(f"[WindowsAdapter:_walk:{elem_id}:children] {ce}") + + return elem + except Exception as e: + print(f"[WindowsAdapter:_walk:{elem_id}] {e}") + traceback.print_exc() + return UIElement(elem_id, "[error]", "Unknown", bounds=Bounds(0, 0, 0, 0)) + + def _synthesize_trees(self, primary: UIElement, secondary: UIElement) -> UIElement: + """Merge two accessibility trees into one richer tree. + + Uses the primary (UIA) tree as the base — it sees web content. + For each node in secondary (pywinauto) matched by bounds, enrich the + primary node with any non-empty properties the primary is missing. + Secondary nodes whose bounds don't appear anywhere in the primary tree + are injected under the deepest primary ancestor that contains them. + """ + # Build a flat index of primary nodes keyed by (x, y, w, h). + bounds_index: Dict[tuple, UIElement] = {} + + def _index(node: UIElement) -> None: + key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) + if key not in bounds_index: + bounds_index[key] = node + for c in node.children: + _index(c) + + _index(primary) + + # Enrich matched nodes; collect unmatched ones with their bounds. + unmatched: List[UIElement] = [] + + def _enrich(node: UIElement) -> None: + key = (node.bounds.x, node.bounds.y, node.bounds.width, node.bounds.height) + target = bounds_index.get(key) + if target is not None: + # Copy over properties the primary left empty. + if not target.keyboard_shortcut and node.keyboard_shortcut: + target.keyboard_shortcut = node.keyboard_shortcut + if not target.description and node.description: + target.description = node.description + if not target.value and node.value: + target.value = node.value + else: + # Not in primary — keep for injection. + w, h = node.bounds.width, node.bounds.height + if w > 0 and h > 0: # ignore zero-size ghost elements + unmatched.append(node) + for c in node.children: + _enrich(c) + + _enrich(secondary) + + # Inject unmatched secondary nodes under the deepest primary ancestor + # whose bounds contain them (largest-area match wins → most specific). + def _contains(outer: Bounds, inner: Bounds) -> bool: + return (outer.x <= inner.x and outer.y <= inner.y and + outer.x + outer.width >= inner.x + inner.width and + outer.y + outer.height >= inner.y + inner.height) + + for node in unmatched: + best: Optional[UIElement] = None + best_area = float("inf") + for pnode in bounds_index.values(): + if _contains(pnode.bounds, node.bounds): + area = pnode.bounds.width * pnode.bounds.height + if area < best_area: + best_area = area + best = pnode + if best is None: + best = primary + # Renumber the injected element_id to avoid collisions. + node.element_id = f"{best.element_id}.x{len(best.children)}" + best.children.append(node) + + return primary + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + try: + import win32gui + import win32ui + from PIL import Image + import ctypes + + if hwnd is None: + hwnd = win32gui.GetForegroundWindow() + + rect = win32gui.GetWindowRect(hwnd) + width = rect[2] - rect[0] + height = rect[3] - rect[1] + + if width <= 0 or height <= 0: + return None + + hwnd_dc = win32gui.GetWindowDC(hwnd) + mfc_dc = win32ui.CreateDCFromHandle(hwnd_dc) + save_dc = mfc_dc.CreateCompatibleDC() + save_bmp = win32ui.CreateBitmap() + save_bmp.CreateCompatibleBitmap(mfc_dc, width, height) + save_dc.SelectObject(save_bmp) + + # PW_RENDERFULLCONTENT (0x2) renders hardware-accelerated content too + ok = ctypes.windll.user32.PrintWindow( # type: ignore[attr-defined] + hwnd, save_dc.GetSafeHdc(), 2) + + bmpinfo = save_bmp.GetInfo() + bmpstr = save_bmp.GetBitmapBits(True) + + win32gui.DeleteObject(save_bmp.GetHandle()) + save_dc.DeleteDC() + mfc_dc.DeleteDC() + win32gui.ReleaseDC(hwnd, hwnd_dc) + + if ok: + img = Image.frombuffer( + "RGB", + (bmpinfo["bmWidth"], bmpinfo["bmHeight"]), + bmpstr, "raw", "BGRX", 0, 1, + ) + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + + # PrintWindow failed — fall back to screen-region capture + logger.warning("[WindowsAdapter:get_screenshot] PrintWindow failed; falling back to mss") + raise RuntimeError("PrintWindow returned 0") + + except Exception as e: + logger.debug(f"[WindowsAdapter:get_screenshot] PrintWindow path failed ({e}); trying mss") + try: + import mss + from PIL import Image + import win32gui + + with mss.MSS() as sct: + if hwnd: + rect = win32gui.GetWindowRect(hwnd) + region = {"left": rect[0], "top": rect[1], + "width": rect[2] - rect[0], "height": rect[3] - rect[1]} + else: + region = sct.monitors[1] + raw = sct.grab(region) + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e2: + print(f"[WindowsAdapter:get_screenshot] {e2}") + traceback.print_exc() + return None + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + try: + import pyautogui + + if action == "type" and value: + pyautogui.write(str(value), interval=0.02) + return {"success": True, "action": "type", "text": value} + + elif action == "key" and value: + keys = str(value).lower().split("+") + pyautogui.hotkey(*keys) + return {"success": True, "action": "key", "keys": value} + + elif action == "click_at" and isinstance(value, dict): + pyautogui.click(value["x"], value["y"]) + return {"success": True, "action": "click_at", **value} + + elif action == "scroll" and isinstance(value, dict): + pyautogui.scroll(value.get("clicks", 3), x=value.get("x"), y=value.get("y")) + return {"success": True, "action": "scroll"} + + else: + return {"success": False, "error": f"Unsupported action: {action}"} + except Exception as e: + print(f"[WindowsAdapter:perform_action] {e}") + traceback.print_exc() + return {"success": False, "error": str(e)} diff --git a/observer/adapters/wsl.py b/observer/adapters/wsl.py new file mode 100644 index 0000000..c4141ed --- /dev/null +++ b/observer/adapters/wsl.py @@ -0,0 +1,119 @@ +""" +WSL adapter (WSL 1 + WSL 2: X11 when DISPLAY is set, +PowerShell interop fallback). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import logging +import os +from typing import List, Optional + +from observer.adapters.linux import LinuxAdapter +from observer.models import Bounds, WindowInfo + +logger = logging.getLogger(__name__) + + +class WSLAdapter(LinuxAdapter): + """Adapter for Windows Subsystem for Linux. + + Prefers X11-based tools (wmctrl, mss) when DISPLAY is set. Falls back to + PowerShell / cmd.exe interop, which is always available in both WSL 1 and + WSL 2 via the Windows binary execution layer. + """ + + def __init__(self, config: dict) -> None: + self.config = config + self._has_display = bool(os.environ.get("DISPLAY")) + logger.info( + "[WSLAdapter:__init__] WSL detected; " + "DISPLAY=%s", "set" if self._has_display else "not set (PowerShell fallback active)", + ) + + # ── Window listing ──────────────────────────────────────────────────────── + + def list_windows(self) -> List[WindowInfo]: + if self._has_display: + result = LinuxAdapter.list_windows(self) + if result: + return result + return self._list_windows_ps() + + def _list_windows_ps(self) -> List[WindowInfo]: + """Enumerate visible Windows windows via PowerShell ConvertTo-Json.""" + try: + import json + import subprocess + ps = ( + "Get-Process " + "| Where-Object { $_.MainWindowTitle -ne '' } " + "| Select-Object Id,ProcessName,MainWindowTitle " + "| ConvertTo-Json -Compress" + ) + r = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], + capture_output=True, text=True, timeout=10, + ) + if r.returncode != 0 or not r.stdout.strip(): + return [] + data = json.loads(r.stdout) + if isinstance(data, dict): + data = [data] + results: List[WindowInfo] = [] + for i, item in enumerate(data or []): + pid = int(item.get("Id", 0)) + name = str(item.get("ProcessName", "unknown")) + title = str(item.get("MainWindowTitle", "")) + if not title: + continue + results.append(WindowInfo( + handle=pid, title=title, process_name=name, pid=pid, + bounds=Bounds(0, 0, 1920, 1080), is_focused=(i == 0), + window_uid=f"wsl:{pid}", + )) + return results + except Exception as e: + logger.debug("[WSLAdapter:_list_windows_ps] %s", e) + return [] + + # ── Screenshot ──────────────────────────────────────────────────────────── + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + if self._has_display: + result = LinuxAdapter.get_screenshot(self, hwnd) + if result: + return result + return self._screenshot_ps() + + def _screenshot_ps(self) -> Optional[bytes]: + """Capture the primary screen via PowerShell, returning PNG bytes.""" + try: + import base64 + import subprocess + # Capture screen to a MemoryStream and emit as base64 — avoids + # WSL↔Windows path translation issues entirely. + ps = ( + "Add-Type -AssemblyName System.Windows.Forms,System.Drawing;" + "$b=[System.Windows.Forms.Screen]::PrimaryScreen.Bounds;" + "$bmp=New-Object System.Drawing.Bitmap $b.Width,$b.Height;" + "$g=[System.Drawing.Graphics]::FromImage($bmp);" + "$g.CopyFromScreen($b.Location,[System.Drawing.Point]::Empty,$b.Size);" + "$ms=New-Object System.IO.MemoryStream;" + "$bmp.Save($ms,[System.Drawing.Imaging.ImageFormat]::Png);" + "$g.Dispose();$bmp.Dispose();" + "[Convert]::ToBase64String($ms.ToArray())" + ) + r = subprocess.run( + ["powershell.exe", "-NoProfile", "-NonInteractive", "-Command", ps], + capture_output=True, text=True, timeout=20, + ) + if r.returncode == 0 and r.stdout.strip(): + return base64.b64decode(r.stdout.strip()) + except Exception as e: + logger.debug("[WSLAdapter:_screenshot_ps] %s", e) + return None + + # ── get_windows_above_bounds: returns [] (inherited from LinuxAdapter) ──── + # ── get_element_tree: upgraded by linux_adapter.install_into if pyatspi ── + # ── perform_action: inherited (pyautogui; needs DISPLAY) ───────────────── diff --git a/observer/core.py b/observer/core.py new file mode 100644 index 0000000..823c63d --- /dev/null +++ b/observer/core.py @@ -0,0 +1,367 @@ +""" +ScreenObserver — public interface; selects a platform adapter +and wires the session tree cache. + +Split out of observer.py (P3); behavior is unchanged. +""" + +import logging +import io +import time +import traceback +from typing import Any, Dict, List, Optional, Tuple + +from observer.activation import ActivationMixin +from observer.adapters.linux import LinuxAdapter +from observer.adapters.macos import MacOSAdapter +from observer.adapters.mock import MockAdapter +from observer.adapters.windows import WindowsAdapter +from observer.adapters.wsl import WSLAdapter +from observer.models import ( + UIElement, WindowInfo, WindowResolution, find_element_by_path, + prune_tree_depth, +) +from observer.occlusion import OcclusionMixin +from observer.platform_info import EFFECTIVE_PLATFORM + +logger = logging.getLogger(__name__) + + +class ScreenObserver(ActivationMixin, OcclusionMixin): + """ + Platform-aware screen observer. All consumers should program against + this class rather than the platform adapters directly. + """ + + def __init__(self, config: dict): + self.config = config + self._adapter = self._select_adapter() + # Try to upgrade stub adapters to real AX implementations. + try: + if isinstance(self._adapter, MacOSAdapter): + import mac_adapter + if mac_adapter.install_into(self): + logger.info("[ScreenObserver] mac_adapter installed (pyobjc)") + elif isinstance(self._adapter, LinuxAdapter): + import linux_adapter + if linux_adapter.install_into(self): + logger.info("[ScreenObserver] linux_adapter installed (pyatspi)") + except Exception: + logger.exception("real adapter upgrade failed") + + def _select_adapter(self): + if self.config.get("mock", False): + logger.info("[ScreenObserver] Using MockAdapter") + return MockAdapter() + + target = self.config.get("platform", "auto") + # EFFECTIVE_PLATFORM is "WSL" when running inside WSL, otherwise same + # as platform.system(). Explicit config overrides auto-detection. + sys_plat = EFFECTIVE_PLATFORM if target == "auto" else target + + adapters = { + "Windows": WindowsAdapter, + "Darwin": MacOSAdapter, + "Linux": LinuxAdapter, + "WSL": WSLAdapter, + } + + cls = adapters.get(sys_plat) + if cls is None: + logger.warning("[ScreenObserver] Unknown platform '%s'; using MockAdapter", sys_plat) + return MockAdapter() + + try: + return cls(self.config) + except Exception as e: + print(f"[ScreenObserver:_select_adapter] Platform adapter failed: {e}; falling back to Mock") + traceback.print_exc() + return MockAdapter() + + @property + def is_mock(self) -> bool: + return isinstance(self._adapter, MockAdapter) + + def list_windows(self) -> List[WindowInfo]: + return self._adapter.list_windows() + + def get_element_tree(self, hwnd=None, window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return the accessibility tree for a window. + + When *window_uid* is supplied the per-window tree cache is consulted: + a fresh cached capture (within ``tree.cache_ttl_s``) is returned + without walking the adapter; a miss walks and stores. Pass + ``use_cache=False`` to force a fresh walk (post-action re-reads); + the fresh capture still refreshes the cache. Calls without a + *window_uid* always walk and are never cached. + """ + tree, _meta = self.get_element_tree_with_meta( + hwnd, window_uid=window_uid, use_cache=use_cache) + return tree + + def get_element_tree_with_meta( + self, hwnd=None, *, window_uid: Optional[str] = None, + use_cache: bool = True, + ) -> Tuple[Optional[UIElement], Dict[str, Any]]: + """get_element_tree plus capture metadata. + + Returns (tree, meta) where meta is + ``{"cache": "hit"|"miss"|"bypass", "capture_ms": int, + "node_count": int}``. + """ + tree_cfg = self.config.get("tree", {}) or {} + ttl = float(tree_cfg.get("cache_ttl_s", 2.0)) + cache = self._tree_cache() + + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, ttl_s=ttl) + if entry is not None: + return entry.tree, { + "cache": "hit", + "capture_ms": 0, + "node_count": entry.node_count, + } + + started = time.time() + tree = self._adapter.get_element_tree(hwnd) + capture_ms = int((time.time() - started) * 1000) + node_count = len(tree.flat_list()) if tree is not None else 0 + + if tree is not None and window_uid and cache is not None: + from hashing import tree_hash as _tree_hash + named = sum( + 1 for e in tree.flat_list()[1:] if (e.name or "").strip() + ) + cache.put( + window_uid, + tree=tree, + serialized=tree.to_dict(), + tree_hash=_tree_hash(tree), + max_depth=int(tree_cfg.get("max_depth", 8)), + capture_ms=capture_ms, + node_count=node_count, + named_node_count=named, + ) + + return tree, { + "cache": "bypass" if not use_cache else "miss", + "capture_ms": capture_ms, + "node_count": node_count, + } + + @staticmethod + def _tree_cache(): + """The session-scoped TreeCache (lazy import avoids cycles).""" + try: + from session import get_session + return get_session().tree_cache + except Exception: + return None + + def get_element_subtree(self, hwnd=None, element_path: str = "root", + max_depth: Optional[int] = None, + window_uid: Optional[str] = None, + use_cache: bool = True) -> Optional[UIElement]: + """Return only the subtree rooted at *element_path* ('root.3.2'). + + Resolution order: + 1. a fresh cached full capture (no walk at all), + 2. the adapter's native get_element_subtree (walks just the branch), + 3. full walk + extraction. + The result is depth-limited to *max_depth* levels below the subtree + root and safe to mutate (cache extraction returns copies).""" + tree_cfg = self.config.get("tree", {}) or {} + if max_depth is None: + max_depth = int(tree_cfg.get("max_depth", 8)) + + # 1. Serve from a fresh cached capture. + cache = self._tree_cache() + if use_cache and window_uid and cache is not None: + entry = cache.get(window_uid, + ttl_s=float(tree_cfg.get("cache_ttl_s", 2.0))) + if entry is not None: + sub = find_element_by_path(entry.tree, element_path) + if sub is not None: + return prune_tree_depth(sub, max_depth) + + # 2. Adapter-native scoped walk. + native = getattr(self._adapter, "get_element_subtree", None) + if native is not None: + try: + sub = native(hwnd, element_path, max_depth) + if sub is not None: + return sub + except Exception: + logger.exception("[ScreenObserver:get_element_subtree] " + "adapter subtree walk failed; falling back") + + # 3. Full walk + extraction. + tree = self.get_element_tree(hwnd, window_uid=window_uid, + use_cache=use_cache) + sub = find_element_by_path(tree, element_path) + return prune_tree_depth(sub, max_depth) + + def get_screenshot(self, hwnd=None) -> Optional[bytes]: + return self._adapter.get_screenshot(hwnd) + + def get_full_display_screenshot(self) -> Optional[bytes]: + """Capture the entire virtual desktop (all monitors combined) as a PNG.""" + try: + import mss + from PIL import Image + with mss.MSS() as sct: + raw = sct.grab(sct.monitors[0]) # 0 = union of all monitors + img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX") + buf = io.BytesIO() + img.save(buf, "PNG") + return buf.getvalue() + except Exception as e: + logger.warning(f"[ScreenObserver:get_full_display_screenshot] {e}; falling back") + return self._adapter.get_screenshot() + + def perform_action(self, action: str, element_id: Optional[str] = None, + value: Any = None, hwnd=None) -> Dict: + return self._adapter.perform_action(action, element_id, value, hwnd) + + def window_by_index(self, windows: List[WindowInfo], + index: Optional[int]) -> Optional[WindowInfo]: + """Convenience: return a WindowInfo by list index, or None.""" + if index is None or not windows: + return None + if 0 <= index < len(windows): + return windows[index] + return None + + def window_by_uid(self, windows: List[WindowInfo], + uid: Optional[str]) -> Optional[WindowInfo]: + """Resolve a window by stable uid; returns None if not found.""" + if not uid or not windows: + return None + for w in windows: + if w.window_uid == uid: + return w + return None + + def resolve_window(self, windows: List[WindowInfo], + window_uid: Optional[str], + window_index: Optional[int], + window_title: Optional[str] = None) -> "WindowResolution": + """Resolve a window by uid (preferred), index, or title substring.""" + if window_uid: + info = self.window_by_uid(windows, window_uid) + warning = ("both window_index and window_uid given; window_uid used" + if window_index is not None else None) + return WindowResolution(info=info, warning=warning, + used_uid=True, requested_uid=window_uid) + if window_index is not None: + info = self.window_by_index(windows, window_index) + resolved_uid = info.window_uid if info else None + return WindowResolution(info=info, warning=None, + used_uid=bool(resolved_uid), + requested_uid=resolved_uid) + if window_title: + needle = window_title.lower() + info = next((w for w in windows if needle in (w.title or "").lower()), None) + resolved_uid = info.window_uid if info else None + return WindowResolution(info=info, warning=None, + used_uid=bool(resolved_uid), + requested_uid=resolved_uid) + return WindowResolution(info=None, warning=None, + used_uid=False, requested_uid=None) + + # ── Monitors / DPI (design doc §6.3) ────────────────────────────────────── + + def get_monitors(self) -> List[Dict[str, Any]]: + """Return per-monitor metadata via mss.""" + try: + import mss + with mss.MSS() as sct: + mons = sct.monitors # [0] is union; [1..] are individual + out: List[Dict[str, Any]] = [] + for i, m in enumerate(mons[1:]): + out.append({ + "index": i, + "primary": (i == 0), + "bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + "scale_factor": 1.0, + "logical_bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + "physical_bounds": {"x": m["left"], "y": m["top"], + "width": m["width"], "height": m["height"]}, + }) + return out + except Exception: + return [] + + # ── Capability discovery (design doc §6.4) ──────────────────────────────── + + def get_capabilities(self) -> Dict[str, Any]: + adapter_name = type(self._adapter).__name__ + is_windows = adapter_name == "WindowsAdapter" + is_macos = adapter_name == "MacOSAdapter" + is_wsl = adapter_name == "WSLAdapter" + is_linux = adapter_name in ("LinuxAdapter", "WSLAdapter") + is_mock = adapter_name == "MockAdapter" + + # Probe optional libs. + def _has(mod: str) -> bool: + try: + __import__(mod) + return True + except Exception: + return False + + if is_macos: + ax_tree = _has("AppKit") or _has("ApplicationServices") or _has("Cocoa") + elif is_linux: + ax_tree = _has("pyatspi") + else: + ax_tree = is_windows or is_mock + + return { + "ok": True, + "platform": EFFECTIVE_PLATFORM, + "adapter": adapter_name, + "version": (self.config.get("mcp", {}) or {}).get("version", "0.2.0"), + "protocol_version": 2, + "supports": { + "accessibility_tree": bool(ax_tree), + "uia_invoke": is_windows, + "occlusion_detection": is_windows or is_mock or _has("Quartz") or _has("Xlib"), + "drag": True, + "ocr": _has("pytesseract"), + "vlm": bool((self.config.get("vlm") or {}).get("enabled") + and (self.config.get("vlm") or {}).get("model")), + "redaction": True, + "scenarios": is_mock, + "tracing": True, + "replay": True, + "image_blur": _has("PIL"), + "wsl_powershell": is_wsl, + # Action capabilities always present via REST + MCP. + "bring_to_foreground": True, + "element_targeting": bool(ax_tree), # click/focus/invoke/set_value via element_id + "observe_with_diff": True, # /api/observe returns diff token + }, + "config": { + "tree_max_depth": (self.config.get("tree", {}) or {}).get("max_depth", 8), + "tree_default_depth": (self.config.get("tree", {}) or {}).get("default_depth", 5), + "ascii_grid": { + "width": (self.config.get("ascii_sketch", {}) or {}).get("grid_width", 110), + "height": (self.config.get("ascii_sketch", {}) or {}).get("grid_height", 38), + }, + }, + # Per-window last-capture statistics (node_count, + # named_node_count, capture_ms, captured_at) so agents can spot + # accessibility-dark windows without another walk. + "tree_stats": self._last_capture_stats(), + } + + def _last_capture_stats(self) -> Dict[str, Dict[str, Any]]: + cache = self._tree_cache() + try: + return cache.stats() if cache is not None else {} + except Exception: + return {} diff --git a/observer/models.py b/observer/models.py new file mode 100644 index 0000000..4bfc39d --- /dev/null +++ b/observer/models.py @@ -0,0 +1,197 @@ +""" +Data model: Bounds, UIElement, WindowInfo, WindowResolution +plus subtree helpers (find_element_by_path, prune_tree_depth). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from dataclasses import dataclass, field, replace as _dc_replace +from typing import Any, Dict, List, Optional + + +@dataclass +class Bounds: + x: int + y: int + width: int + height: int + + @property + def right(self) -> int: + return self.x + self.width + + @property + def bottom(self) -> int: + return self.y + self.height + + @property + def center_x(self) -> int: + return self.x + self.width // 2 + + @property + def center_y(self) -> int: + return self.y + self.height // 2 + + def to_dict(self) -> Dict: + return {"x": self.x, "y": self.y, "width": self.width, "height": self.height} + + def __bool__(self) -> bool: + return self.width > 0 and self.height > 0 + + +@dataclass +class UIElement: + element_id: str + name: str + role: str + value: Optional[str] = None + bounds: Bounds = field(default_factory=lambda: Bounds(0, 0, 0, 0)) + enabled: bool = True + focused: bool = False + keyboard_shortcut: Optional[str] = None + description: Optional[str] = None + children: List["UIElement"] = field(default_factory=list) + # Extended a11y signals — None means "adapter could not determine". + # Populated where the platform exposes the pattern (UIA SelectionItem / + # ExpandCollapse / RangeValue, AX AXValue/AXMinValue/AXMaxValue, AT-SPI + # STATE_SELECTED / STATE_EXPANDED / IValue) and consumed by the ASCII + # renderer for role-aware glyphs and the structured sidecar. + selected: Optional[bool] = None + expanded: Optional[bool] = None + value_now: Optional[float] = None + value_min: Optional[float] = None + value_max: Optional[float] = None + identifier: Optional[str] = None + + def to_dict(self) -> Dict: + d: Dict[str, Any] = { + "id": self.element_id, + "name": self.name, + "role": self.role, + "value": self.value, + "bounds": self.bounds.to_dict(), + "enabled": self.enabled, + "focused": self.focused, + "keyboard_shortcut": self.keyboard_shortcut, + "description": self.description, + "children": [c.to_dict() for c in self.children], + } + # Omit extended fields when unset so existing API consumers do not + # see a flood of nulls; include them when populated. + for k in ("selected", "expanded", "value_now", "value_min", + "value_max", "identifier"): + v = getattr(self, k) + if v is not None: + d[k] = v + return d + + def flat_list(self) -> List["UIElement"]: + """Return all elements in this subtree as a flat list (DFS order).""" + result = [self] + for child in self.children: + result.extend(child.flat_list()) + return result + + +@dataclass +class WindowInfo: + handle: Any # platform-specific: HWND (int) on Windows; int index elsewhere + title: str + process_name: str + pid: int + bounds: Bounds + is_focused: bool + # Stable cross-call identifier; populated by adapters (design doc §6.1). + window_uid: str = "" + # Optional multi-monitor metadata (design doc §6.3). Populated when the + # adapter knows; left None on adapters that do not. + monitor_index: Optional[int] = None + scale_factor: Optional[float] = None + logical_bounds: Optional[Bounds] = None + physical_bounds: Optional[Bounds] = None + + def to_dict(self) -> Dict: + d: Dict[str, Any] = { + "handle": str(self.handle), + "title": self.title, + "process": self.process_name, + "pid": self.pid, + "bounds": self.bounds.to_dict(), + "focused": self.is_focused, + "window_uid": self.window_uid, + } + if self.monitor_index is not None: + d["monitor_index"] = self.monitor_index + if self.scale_factor is not None: + d["scale_factor"] = self.scale_factor + if self.logical_bounds is not None: + d["logical_bounds"] = self.logical_bounds.to_dict() + if self.physical_bounds is not None: + d["physical_bounds"] = self.physical_bounds.to_dict() + return d + + +# ─── Window resolution result ──────────────────────────────────────────────── + +@dataclass +class WindowResolution: + info: Optional[WindowInfo] + warning: Optional[str] + used_uid: bool + requested_uid: Optional[str] + + +# ─── Subtree helpers (P1 perf: scoped drill-in) ────────────────────────────── + +def find_element_by_path(root: Optional[UIElement], + element_path: str) -> Optional[UIElement]: + """Locate an element by its positional element-id path (e.g. 'root.3.2'). + + Prefers walking the id prefixes level by level (cheap); falls back to a + full DFS by exact id for trees whose ids are not strictly positional + (e.g. nodes injected by tree synthesis get ids like 'root.2.x1').""" + if root is None or not element_path: + return None + if root.element_id == element_path: + return root + # Fast path: navigate children whose ids extend the current prefix. + if element_path.startswith(root.element_id + "."): + node = root + prefix = root.element_id + rest = element_path[len(prefix) + 1:] + found = True + for seg in rest.split("."): + prefix = f"{prefix}.{seg}" + nxt = next((c for c in node.children if c.element_id == prefix), + None) + if nxt is None: + found = False + break + node = nxt + if found: + return node + # Fallback: exhaustive search by exact id. + stack = [root] + while stack: + e = stack.pop() + if e.element_id == element_path: + return e + stack.extend(e.children) + return None + + +def prune_tree_depth(elem: Optional[UIElement], + max_depth: Optional[int]) -> Optional[UIElement]: + """Return a copy of *elem* limited to *max_depth* levels below it. + + Nodes are shallow-copied (bounds objects are shared) so the original — + possibly cache-resident — tree is never mutated.""" + if elem is None or max_depth is None: + return elem + + def _copy(e: UIElement, depth: int) -> UIElement: + kids = ([] if depth >= max_depth + else [_copy(c, depth + 1) for c in e.children]) + return _dc_replace(e, children=kids) + + return _copy(elem, 0) diff --git a/observer/occlusion.py b/observer/occlusion.py new file mode 100644 index 0000000..4ba954d --- /dev/null +++ b/observer/occlusion.py @@ -0,0 +1,116 @@ +""" +Occlusion / visibility: visible areas, screen bounds and +rectangle geometry helpers (mixin consumed by ScreenObserver). + +Split out of observer.py (P3); behavior is unchanged. +""" + +from typing import Any, Dict, List, Optional + +from observer.models import Bounds, WindowInfo + + +class OcclusionMixin: + """Occlusion / visibility methods of ScreenObserver.""" + + # Provided by the concrete ScreenObserver. + _adapter: Any + # ── Element occlusion (design doc D14) ──────────────────────────────────── + + def is_element_occluded(self, element_bounds: Bounds, target_hwnd: Any, + all_windows: List[WindowInfo]) -> bool: + """ + True iff every pixel of *element_bounds* is covered by another window + above the target in Z-order, or the element lies entirely off-screen. + + On platforms without Z-order info, returns False (assumed visible). + """ + screen = self.get_screen_bounds() + clipped = _intersect_bounds(element_bounds, screen) + if not clipped: + return True + regions = [clipped] + try: + occluders = self._adapter.get_windows_above_bounds(target_hwnd) + except Exception: + occluders = [] + for occ in occluders: + regions = _subtract_rect(regions, occ) + return not regions + + # ── Visibility helpers ──────────────────────────────────────────────────── + + def get_screen_bounds(self) -> Bounds: + """Return the bounding rect of the combined virtual screen (all monitors).""" + try: + import mss + with mss.MSS() as sct: + m = sct.monitors[0] # index 0 = union of all monitors + return Bounds(m["left"], m["top"], m["width"], m["height"]) + except Exception: + return Bounds(0, 0, 65535, 65535) + + def get_visible_areas(self, target_hwnd: Any, + all_windows: List[WindowInfo]) -> List[Dict]: + """ + Return a list of {x, y, width, height} dicts for the portions of the + window identified by *target_hwnd* that are on-screen and not covered + by windows above it in Z-order. + + On Windows the Z-order is queried precisely via win32gui. + On macOS/Linux Z-order is unavailable, so the full clipped-to-screen + bounds are returned (assuming the window is on top). + """ + target = next((w for w in all_windows if w.handle == target_hwnd), None) + if target is None: + return [] + + screen = self.get_screen_bounds() + clipped = _intersect_bounds(target.bounds, screen) + if not clipped: + return [] + + visible: List[Bounds] = [clipped] + occluders = self._adapter.get_windows_above_bounds(target_hwnd) + for occ in occluders: + visible = _subtract_rect(visible, occ) + + return [b.to_dict() for b in visible] + + +def _intersect_bounds(a: Bounds, b: Bounds) -> Optional[Bounds]: + x1 = max(a.x, b.x) + y1 = max(a.y, b.y) + x2 = min(a.right, b.right) + y2 = min(a.bottom, b.bottom) + if x2 <= x1 or y2 <= y1: + return None + return Bounds(x1, y1, x2 - x1, y2 - y1) + + +def _subtract_rect(rects: List[Bounds], occluder: Bounds) -> List[Bounds]: + """Subtract occluder from each rect, splitting into up to 4 sub-rects.""" + result: List[Bounds] = [] + for r in rects: + ix1 = max(r.x, occluder.x) + iy1 = max(r.y, occluder.y) + ix2 = min(r.right, occluder.right) + iy2 = min(r.bottom, occluder.bottom) + + if ix2 <= ix1 or iy2 <= iy1: + result.append(r) + continue + + # Top strip + if iy1 > r.y: + result.append(Bounds(r.x, r.y, r.width, iy1 - r.y)) + # Bottom strip + if iy2 < r.bottom: + result.append(Bounds(r.x, iy2, r.width, r.bottom - iy2)) + # Left strip (height = intersection height) + if ix1 > r.x: + result.append(Bounds(r.x, iy1, ix1 - r.x, iy2 - iy1)) + # Right strip (height = intersection height) + if ix2 < r.right: + result.append(Bounds(ix2, iy1, r.right - ix2, iy2 - iy1)) + return result diff --git a/observer/platform_info.py b/observer/platform_info.py new file mode 100644 index 0000000..e47a073 --- /dev/null +++ b/observer/platform_info.py @@ -0,0 +1,24 @@ +""" +Platform detection (Windows / macOS / Linux / WSL). + +Split out of observer.py (P3); behavior is unchanged. +""" + +import platform + +PLATFORM = platform.system() + + +def _is_wsl() -> bool: + """True when running inside Windows Subsystem for Linux (WSL 1 or WSL 2).""" + if PLATFORM != "Linux": + return False + try: + with open("/proc/version") as _f: + return "microsoft" in _f.read().lower() + except Exception: + return False + + +IS_WSL = _is_wsl() +EFFECTIVE_PLATFORM = "WSL" if IS_WSL else PLATFORM diff --git a/ocr_util.py b/ocr_util.py index debc855..8b5a048 100644 --- a/ocr_util.py +++ b/ocr_util.py @@ -92,7 +92,7 @@ def diagnose(config: Optional[Dict[str, Any]]) -> Dict[str, Any]: info["error"] = (f"tesseract_cmd={cmd!r} does not exist on disk and " f"is not on PATH. {INSTALL_HINT}") return info - info["configured_path_exists"] = bool(cmd) and ( + info["configured_path_exists"] = cmd is not None and ( os.path.exists(cmd) or shutil.which(cmd) is not None ) try: diff --git a/ollama_setup.py b/ollama_setup.py index 3f7b461..952be08 100644 --- a/ollama_setup.py +++ b/ollama_setup.py @@ -89,7 +89,8 @@ def _ask_runner() -> Optional[List[str]]: file=sys.stderr, ) - options: List[Tuple[str, List[str]]] = [] + # Each option's prefix is a token list; None marks the "custom" choice. + options: List[Tuple[str, Optional[List[str]]]] = [] # Option 1: native ollama on PATH if _test_runner(["ollama"]): @@ -118,8 +119,8 @@ def _ask_runner() -> Optional[List[str]]: if raw.isdigit(): idx = int(raw) - 1 if 0 <= idx < len(options): - _, prefix = options[idx] - if prefix is None: + _, chosen = options[idx] + if chosen is None: # Custom entry try: custom = input( @@ -145,9 +146,9 @@ def _ask_runner() -> Optional[List[str]]: if ok != "y": continue return tokens - if prefix == []: # skip + if chosen == []: # skip return [] - return prefix + return chosen print(f" Please enter a number 1–{len(options)}.", file=sys.stderr) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0081e3f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +# Tooling configuration (quality gates, P2). +# Runtime packaging still goes through requirements*.txt. + +# ─── ruff ───────────────────────────────────────────────────────────────────── +# Mirrors ruff's default rule set explicitly so E722 (bare `except:`) is a +# documented, guaranteed gate — silent bare excepts must not come back. +[tool.ruff] +extend-exclude = ["tests"] + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] # E7 includes E722 (bare except) + +# ─── mypy ───────────────────────────────────────────────────────────────────── +# Lenient globally (large legacy surface, optional platform deps), strict-ish +# for the small, fully-typed core modules that everything else builds on. +[tool.mypy] +python_version = "3.11" +ignore_missing_imports = true +follow_imports = "silent" +warn_unused_configs = true +# Platform-specific adapters poke optional OS libraries; exclude the ones +# that cannot even be imported on CI's platform matrix. +exclude = ["tests/"] + +[[tool.mypy.overrides]] +module = ["tree_cache", "errors", "session"] +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +warn_return_any = true +no_implicit_optional = true + +# ─── coverage ──────────────────────────────────────────────────────────────── +[tool.coverage.run] +# Measure all source modules (even ones a given platform never imports, +# e.g. window_agent / mac_adapter) so the CI floor has a stable basis. +source = ["."] +omit = ["tests/*"] + +[tool.coverage.report] +precision = 1 diff --git a/redaction.py b/redaction.py index bc72774..9270497 100644 --- a/redaction.py +++ b/redaction.py @@ -1,8 +1,17 @@ """ -redaction.py — Sensitive-region redaction (design doc §18, D7). +redaction.py — Sensitive-region redaction (design doc §18, D7) and +untrusted-screen-content marking (P2 trust boundary). Default: tree node names/values + OCR text + VLM preamble are scrubbed. Opt-in: redaction.blur_screenshots paints matched bboxes solid black. + +Trust boundary: everything read off the screen (window titles, element +names/values, OCR words, VLM descriptions) is attacker-influenced data — +a web page or document on screen can contain prompt-injection text. +``mark_untrusted`` flags the results of screen-text-carrying tools with +``untrusted: true`` and strips ANSI escape sequences / control characters +so extracted text cannot smuggle terminal escapes to downstream consumers. +MCP clients must treat these fields as data, never as instructions. """ from __future__ import annotations @@ -10,13 +19,79 @@ import io import logging import re -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) DEFAULT_REPLACEMENT = "[REDACTED]" +# ─── Untrusted screen content (P2 trust boundary) ──────────────────────────── + +# CSI (ESC [ … cmd), OSC (ESC ] … BEL/ST), and single-char escapes. +_ANSI_ESCAPE_RE = re.compile( + r"\x1b(?:\[[0-9;?]*[ -/]*[@-~]" # CSI sequences + r"|\][^\x07\x1b]*(?:\x07|\x1b\\)?" # OSC sequences + r"|[@-Z\\-_])" # 2-byte escapes (incl. lone ESC-x) +) +# C0/C1 control characters except \t \n \r (layout-preserving whitespace). +_CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") + +# Tool results that carry text extracted from the screen (window titles, +# element names/values, OCR words, descriptions). Their responses are +# flagged ``untrusted: true`` and sanitized. Action receipts also embed +# screen-derived selectors, but flagging every action result would drown +# the signal; perception results are where injection text actually lands. +UNTRUSTED_RESULT_TOOLS = frozenset({ + "list_windows", + "find_element", + "get_window_structure", + "observe_window", + "wait_for", + "snapshot_get", + "snapshot_diff", + "get_ocr", + "get_screen_description", +}) + +# Keys whose values are machine artifacts (base64 blobs, opaque tokens) — +# skipped during sanitization for performance and fidelity. +_SANITIZE_SKIP_KEYS = frozenset({ + "data", "screenshot_b64", "tree_token", "base_token", + "snapshot_id", "confirm_token", +}) + + +def sanitize_screen_text(text: Optional[str]) -> Optional[str]: + """Strip ANSI escape sequences and non-whitespace control characters + from screen-extracted text. Idempotent; returns non-str input as-is.""" + if not isinstance(text, str) or not text: + return text + text = _ANSI_ESCAPE_RE.sub("", text) + return _CONTROL_CHARS_RE.sub("", text) + + +def _sanitize_value(value: Any, key: str = "") -> Any: + if isinstance(value, str): + return sanitize_screen_text(value) + if isinstance(value, dict): + return {k: (v if k in _SANITIZE_SKIP_KEYS else _sanitize_value(v, k)) + for k, v in value.items()} + if isinstance(value, list): + return [_sanitize_value(v) for v in value] + return value + + +def mark_untrusted(tool: str, result: Any) -> Any: + """Flag screen-text-carrying tool results as untrusted and sanitize + their string payloads. No-op for tools that return no screen text.""" + if tool not in UNTRUSTED_RESULT_TOOLS or not isinstance(result, dict): + return result + out = _sanitize_value(result) + out["untrusted"] = True + return out + + class Redactor: def __init__(self, config: Dict[str, Any]) -> None: cfg = (config.get("redaction") or {}) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f02ad0..924a257 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ pytest>=8.0 pytest-cov>=4.0 ruff>=0.4 +mypy>=1.8 diff --git a/scripts/check_module_size.py b/scripts/check_module_size.py new file mode 100644 index 0000000..29357b9 --- /dev/null +++ b/scripts/check_module_size.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +check_module_size.py — CI guard against god-files creeping back. + +The design doc (agentic_features_design.md) caps modules at ~600 LOC: +"Keep modules under ~600 LOC; split when growing past that." After the +P3 decomposition every tracked non-test source module fits under that +cap except a small set of intentional remainders listed in ALLOWLIST. + +Fails (exit 1) when any git-tracked non-test .py file exceeds MAX_LINES +and is not allowlisted, and also when an allowlisted file shrinks back +under the cap (so stale allowlist entries are removed rather than +becoming silent escape hatches). +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +MAX_LINES = 600 + +# Intentional remainders. Every entry needs a justification; new code +# modules must be split instead of added here. +ALLOWLIST: dict[str, str] = { + # Declarative data blobs, not logic: one schema/asset entry per line + # would gain nothing from splitting across files. + "web_inspector/assets.py": + "single inline HTML/CSS/JS template string for the SPA UI", + "window_agent/tool_schemas.py": + "declarative LLM tool catalogue (schemas + tiers + keyword groups)", + "mcp_server/tool_schemas.py": + "declarative MCP tools/list schema payload (_TOOLS)", + # Cohesive single-responsibility modules slightly over the cap; + # splitting them would separate tightly coupled halves. + "ascii_renderer.py": + "one renderer: grid projection + glyph rules + OCR overlay share state", + "description.py": + "one generator: accessibility/OCR/VLM description passes share helpers", + "observer/adapters/windows.py": + "one adapter: UIA COM walker + pywinauto walk + tree synthesis", +} + + +def tracked_py_files(root: Path) -> list[str]: + out = subprocess.run( + ["git", "ls-files", "*.py"], + cwd=root, capture_output=True, text=True, check=True, + ).stdout + return [p for p in out.splitlines() + if p and not p.startswith("tests/")] + + +def main() -> int: + root = Path(__file__).resolve().parent.parent + failures: list[str] = [] + stale: list[str] = [] + seen_allowlisted: set[str] = set() + + for rel in tracked_py_files(root): + path = root / rel + try: + n = sum(1 for _ in path.open("rb")) + except OSError as e: + failures.append(f"{rel}: unreadable ({e})") + continue + if rel in ALLOWLIST: + seen_allowlisted.add(rel) + if n <= MAX_LINES: + stale.append(f"{rel}: {n} lines — now under the cap; " + f"remove its ALLOWLIST entry") + continue + if n > MAX_LINES: + failures.append( + f"{rel}: {n} lines exceeds the {MAX_LINES}-line module cap " + f"(design doc: split modules growing past ~600 LOC)") + + for rel in sorted(set(ALLOWLIST) - seen_allowlisted): + stale.append(f"{rel}: allowlisted but not tracked — remove the entry") + + for msg in failures + stale: + print(f"check_module_size: {msg}", file=sys.stderr) + if failures or stale: + return 1 + print(f"check_module_size: OK — all tracked non-test modules are " + f"<= {MAX_LINES} lines ({len(ALLOWLIST)} allowlisted exceptions)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/session.py b/session.py index 6d55834..b813048 100644 --- a/session.py +++ b/session.py @@ -4,6 +4,7 @@ Holds per-process state shared between the REST and MCP interfaces: - step_id counter and last input step (for caused_by_step_id) - tree token ring buffer per window_uid (for since= diffs) + - per-window tree cache (avoids re-walking the accessibility tree) - snapshots (TTL-bounded, LRU) - confirmation tokens (TTL-bounded) - active trace handle (set by tracing.py) @@ -20,6 +21,8 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Tuple +from tree_cache import TreeCache + # ─── Tunables (overridable from config) ────────────────────────────────────── @@ -212,6 +215,7 @@ def next_id(self, *, is_input: bool) -> Tuple[int, Optional[int]]: with self._lock: sid = self._next self._next += 1 + caused_by: Optional[int] if is_input: caused_by = sid self._last_input_step = sid @@ -234,6 +238,7 @@ def uptime_s(self) -> float: @dataclass class Session: tree_tokens: _TreeTokenStore = field(default_factory=_TreeTokenStore) + tree_cache: TreeCache = field(default_factory=TreeCache) snapshots: _SnapshotStore = field(default_factory=_SnapshotStore) confirms: _ConfirmStore = field(default_factory=_ConfirmStore) steps: _StepCounter = field(default_factory=_StepCounter) diff --git a/tests/test_security_defaults.py b/tests/test_security_defaults.py new file mode 100644 index 0000000..15dc863 --- /dev/null +++ b/tests/test_security_defaults.py @@ -0,0 +1,140 @@ +"""[P2] Secure-defaults tests: loopback bind default, opt-in remote bind, +and config-driven CORS behavior. + +The HTTP API is unauthenticated by design (no auth is added — binding +policy is the security control): + - built-in default bind must be loopback (127.0.0.1); + - --host / web_ui.host remain the explicit opt-in path to 0.0.0.0; + - non-loopback binds must produce a prominent startup warning; + - no CORS headers are sent unless web_ui.cors_origins is configured. +""" +from __future__ import annotations + +import copy +import json + +import main as _main +from main import _DEFAULT_CONFIG, bind_warning, load_config + + +# ── Bind default resolution ────────────────────────────────────────────────── + + +def test_builtin_default_bind_is_loopback(): + assert _DEFAULT_CONFIG["web_ui"]["host"] == "127.0.0.1" + + +def test_builtin_default_cors_origins_empty(): + assert _DEFAULT_CONFIG["web_ui"]["cors_origins"] == [] + + +def test_load_config_without_host_key_resolves_loopback(tmp_path): + p = tmp_path / "config.json" + p.write_text(json.dumps({"web_ui": {"port": 6001}})) + cfg = load_config(str(p)) + assert cfg["web_ui"]["host"] == "127.0.0.1" + assert cfg["web_ui"]["port"] == 6001 + + +def test_load_config_explicit_opt_in_preserved(tmp_path): + p = tmp_path / "config.json" + p.write_text(json.dumps({"web_ui": {"host": "0.0.0.0"}})) + cfg = load_config(str(p)) + assert cfg["web_ui"]["host"] == "0.0.0.0" + + +def test_example_config_binds_loopback(tmp_path): + # config.json.example is the bootstrap source of truth for first runs. + import os + example = os.path.join(os.path.dirname(_main.__file__), + "config.json.example") + with open(example, encoding="utf-8") as f: + cfg = json.load(f) + assert cfg["web_ui"]["host"] == "127.0.0.1" + assert cfg["web_ui"]["cors_origins"] == [] + + +# ── Non-loopback bind warning ──────────────────────────────────────────────── + + +def test_bind_warning_none_for_loopback(): + assert bind_warning("127.0.0.1") is None + assert bind_warning("localhost") is None + assert bind_warning("::1") is None + + +def test_bind_warning_for_all_interfaces(): + w = bind_warning("0.0.0.0") + assert w is not None + assert "unauthenticated action API exposed to the network" in w + + +def test_bind_warning_for_lan_address(): + w = bind_warning("192.168.1.20") + assert w is not None + assert "192.168.1.20" in w + + +# ── CORS behavior via Flask test client ────────────────────────────────────── + + +def _make_client(cors_origins=None): + from ascii_renderer import ASCIIRenderer + from description import DescriptionGenerator + from observer import ScreenObserver + from web_inspector import create_web_app + + cfg = copy.deepcopy(_DEFAULT_CONFIG) + cfg["mock"] = True + if cors_origins is not None: + cfg["web_ui"]["cors_origins"] = cors_origins + app = create_web_app(ScreenObserver(cfg), ASCIIRenderer(cfg), + DescriptionGenerator(cfg), cfg) + return app.test_client() + + +def test_no_cors_headers_by_default(fresh_session): + client = _make_client() + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + assert r.status_code == 200 + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_no_cors_on_action_by_default(fresh_session): + client = _make_client() + r = client.post("/api/action", + json={"action": "click_at", "x": 10, "y": 10}, + headers={"Origin": "http://evil.example"}) + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_no_cors_preflight_grant_on_action_by_default(fresh_session): + client = _make_client() + r = client.options( + "/api/action", + headers={"Origin": "http://evil.example", + "Access-Control-Request-Method": "POST"}) + assert "Access-Control-Allow-Origin" not in r.headers + + +def test_cors_wildcard_opt_in(fresh_session): + client = _make_client(cors_origins=["*"]) + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + # flask-cors echoes the requesting origin for a "*" allowlist; either + # form grants cross-origin access. + assert r.headers.get("Access-Control-Allow-Origin") in ( + "*", "http://evil.example") + + +def test_cors_explicit_origin_allowed(fresh_session): + client = _make_client(cors_origins=["http://localhost:3000"]) + r = client.get("/api/windows", + headers={"Origin": "http://localhost:3000"}) + assert (r.headers.get("Access-Control-Allow-Origin") + == "http://localhost:3000") + + +def test_cors_explicit_origin_rejects_others(fresh_session): + client = _make_client(cors_origins=["http://localhost:3000"]) + r = client.get("/api/windows", headers={"Origin": "http://evil.example"}) + assert "Access-Control-Allow-Origin" not in r.headers diff --git a/tests/test_surface_parity.py b/tests/test_surface_parity.py new file mode 100644 index 0000000..d3ed13d --- /dev/null +++ b/tests/test_surface_parity.py @@ -0,0 +1,176 @@ +"""[P2] REST ↔ MCP ↔ dispatch parity (design doc §27). + +Every tool in tools.REGISTRY must be reachable with the same parameter +surface from both public surfaces: + + - REST: the generic tool console (POST /api/tool/) forwards the + JSON body verbatim to tools.dispatch, and GET /api/tools enumerates + the registry. Most tools additionally have a dedicated route. + - MCP: mcp_server advertises a schema in _TOOLS and forwards + `arguments` verbatim to tools.dispatch for every REGISTRY tool. + +Exceptions are explicit, documented allowlists below — any new tool that +is added to one surface but not the others fails these tests. +""" +from __future__ import annotations + +import pytest + +import mcp_server as _mcp +import tools as _tools + +# REGISTRY tools with no dedicated MCP schema: pure aliases of click_at — +# MCP callers pass button="right" / double=true to click_at instead. +MCP_SCHEMA_EXCLUSIONS = { + "right_click_at", + "double_click_at", +} + +# MCP tools handled by legacy composite handlers inside mcp_server._dispatch +# (not part of tools.REGISTRY); their REST equivalents are /api/sketch and +# /api/full_screenshot. +MCP_LEGACY_ONLY = { + "get_screen_sketch", + "get_full_screenshot", +} + +# REGISTRY tools with no dedicated REST route (reachable via the generic +# console only): pure aliases of click_at — REST callers POST /api/action +# with {"action": "click_at", "button": "right"} / {"double": true}. +REST_DEDICATED_ROUTE_EXCLUSIONS = { + "right_click_at", + "double_click_at", +} + +# tool name -> dedicated REST route (path prefix as registered in Flask). +REST_ROUTES = { + "list_windows": "/api/windows", + "get_capabilities": "/api/capabilities", + "get_monitors": "/api/monitors", + "find_element": "/api/find_element", + "get_window_structure": "/api/structure", + "get_screenshot": "/api/screenshot", + "get_visible_areas": "/api/visible_areas", + "click_element": "/api/element/click", + "focus_element": "/api/element/focus", + "set_value": "/api/element/set_value", + "invoke_element": "/api/element/invoke", + "select_option": "/api/element/select", + "click_at": "/api/action", + "type_text": "/api/action", + "press_key": "/api/action", + "scroll": "/api/action", + "bring_to_foreground": "/api/bring_to_foreground", + "observe_window": "/api/observe", + "snapshot": "/api/snapshot", + "snapshot_get": "/api/snapshot/", + "snapshot_diff": "/api/snapshot/diff", + "snapshot_drop": "/api/snapshot/", + "wait_for": "/api/wait_for", + "wait_idle": "/api/wait_idle", + "click_element_and_observe": "/api/element/click_and_observe", + "type_and_observe": "/api/type_and_observe", + "press_key_and_observe": "/api/key_and_observe", + "get_screenshot_cropped": "/api/screenshot/cropped", + "get_ocr": "/api/ocr", + "get_screen_description": "/api/description", + "trace_start": "/api/trace/start", + "trace_stop": "/api/trace/stop", + "trace_status": "/api/trace/status", + "replay_start": "/api/replay/start", + "replay_step": "/api/replay/step", + "replay_status": "/api/replay/status", + "replay_stop": "/api/replay/stop", + "load_scenario": "/api/scenario/load", + "assert_state": "/api/assert_state", + "get_budget_status": "/api/budget_status", + "get_redaction_status": "/api/redaction_status", + "propose_action": "/api/propose_action", + "hover_at": "/api/hover", + "hover_element": "/api/hover", + "right_click_element": "/api/element/right_click", + "double_click_element": "/api/element/double_click", + "drag": "/api/drag", + "key_into_element": "/api/element/key", + "clear_text": "/api/element/clear_text", +} + +# Tools not invoked in the smoke loop: session-stateful (an active trace +# would then record every subsequent call in this process's global session). +SMOKE_SKIP = {"trace_start", "trace_stop"} + +# Minimal args so no tool blocks (waits) or errors on missing input in a +# way unrelated to reachability. +SMOKE_ARGS = { + "wait_for": {"timeout_ms": 20, + "any_of": [{"type": "window_appears", "title_regex": "."}]}, + "wait_idle": {"timeout_ms": 20, "quiet_ms": 5}, + "hover_at": {"hover_ms": 1}, + "hover_element": {"hover_ms": 1, + "selector": 'MenuItem[name="File"]'}, +} + + +def test_mcp_advertises_every_registry_tool(): + mcp_names = {t["name"] for t in _mcp._TOOLS} + missing = set(_tools.REGISTRY) - mcp_names - MCP_SCHEMA_EXCLUSIONS + assert not missing, f"REGISTRY tools missing an MCP schema: {sorted(missing)}" + + +def test_mcp_schema_exclusions_are_not_advertised_stale(): + # Keep the exclusion list honest: everything on it must exist in the + # REGISTRY and must genuinely lack an MCP schema. + mcp_names = {t["name"] for t in _mcp._TOOLS} + for name in MCP_SCHEMA_EXCLUSIONS: + assert name in _tools.REGISTRY + assert name not in mcp_names + + +def test_every_mcp_tool_is_dispatchable(): + mcp_names = {t["name"] for t in _mcp._TOOLS} + unknown = mcp_names - set(_tools.REGISTRY) - MCP_LEGACY_ONLY + assert not unknown, f"MCP schemas with no dispatch target: {sorted(unknown)}" + + +def test_mcp_schemas_have_object_parameter_surface(): + # MCP forwards `arguments` verbatim to tools.dispatch, so every schema + # must declare an object inputSchema (same dict-shaped surface REST + # forwards from the JSON body). + for t in _mcp._TOOLS: + schema = t.get("inputSchema") or {} + assert schema.get("type") == "object", t["name"] + assert isinstance(schema.get("properties", {}), dict), t["name"] + + +def test_rest_console_enumerates_registry(client): + data = client.get("/api/tools").get_json() + assert data["ok"] is True + assert data["tools"] == sorted(_tools.REGISTRY) + + +def test_every_registry_tool_has_dedicated_rest_route(app): + rules = {r.rule for r in app.url_map.iter_rules()} + for name in _tools.REGISTRY: + if name in REST_DEDICATED_ROUTE_EXCLUSIONS: + assert name not in REST_ROUTES + continue + assert name in REST_ROUTES, f"no documented REST route for {name}" + assert REST_ROUTES[name] in rules, ( + f"{name}: documented route {REST_ROUTES[name]} not registered") + + +@pytest.mark.parametrize("name", sorted(_tools.REGISTRY)) +def test_rest_console_reaches_tool(client, name): + """POST /api/tool/ must reach dispatch for every REGISTRY tool + (a structured envelope comes back — never Flask's 404/405 or the + dispatcher's 'unknown tool' error).""" + if name in SMOKE_SKIP: + pytest.skip("session-stateful; reachability covered by url_map test") + r = client.post(f"/api/tool/{name}", json=SMOKE_ARGS.get(name, {})) + assert r.status_code != 404 or r.get_json() is not None + data = r.get_json() + assert isinstance(data, dict), name + assert "ok" in data, name + if data["ok"] is False: + msg = (data.get("error") or {}).get("message", "") + assert not msg.startswith("unknown tool"), name diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py new file mode 100644 index 0000000..1f8dc41 --- /dev/null +++ b/tests/test_telemetry.py @@ -0,0 +1,97 @@ +"""[P2] Telemetry tests: /api/healthz and /api/metrics expose step counts, +tree-cache hit/miss counters, and a capture-latency summary.""" +from __future__ import annotations + +import re + + +def _metric(body: str, name: str) -> float: + m = re.search(rf"^{re.escape(name)} ([0-9.]+)$", body, re.M) + assert m, f"metric {name} not found in:\n{body}" + return float(m.group(1)) + + +# ── /api/healthz ───────────────────────────────────────────────────────────── + + +def test_healthz_includes_tree_cache_counters(client): + data = client.get("/api/healthz").get_json() + assert data["ok"] is True + tc = data["tree_cache"] + for key in ("hits", "misses", "entries", "capture_count", + "capture_ms_total", "capture_ms_max", "capture_ms_mean"): + assert key in tc, key + + +def test_healthz_tree_cache_counts_hits_and_misses(client): + client.get("/api/structure") # cold: miss + capture + client.get("/api/structure") # warm (TTL 2s): hit + tc = client.get("/api/healthz").get_json()["tree_cache"] + assert tc["misses"] >= 1 + assert tc["hits"] >= 1 + assert tc["capture_count"] >= 1 + + +def test_healthz_step_count_increments(client): + before = client.get("/api/healthz").get_json()["step_count"] + client.get("/api/windows") + after = client.get("/api/healthz").get_json()["step_count"] + assert after == before + 1 + + +def test_healthz_ocr_diagnostic_computed_once(client, monkeypatch): + """healthz must stay cheap: the tesseract probe (subprocess spawn) runs + at most once per process, not per poll.""" + import ocr_util + calls = {"n": 0} + real = ocr_util.diagnose + + def counting_diagnose(cfg): + calls["n"] += 1 + return real(cfg) + + monkeypatch.setattr(ocr_util, "diagnose", counting_diagnose) + r1 = client.get("/api/healthz") + r2 = client.get("/api/healthz") + assert r1.status_code == 200 and r2.status_code == 200 + assert calls["n"] <= 1 + + +# ── /api/metrics ───────────────────────────────────────────────────────────── + + +def test_metrics_prometheus_content_type(client): + r = client.get("/api/metrics") + assert r.status_code == 200 + assert r.content_type.startswith("text/plain") + + +def test_metrics_exposes_tree_cache_and_latency(client): + body = client.get("/api/metrics").get_data(as_text=True) + for name in ("oso_step_count", + "oso_uptime_seconds", + "oso_tree_cache_hits_total", + "oso_tree_cache_misses_total", + "oso_tree_cache_entries", + "oso_tree_capture_ms_sum", + "oso_tree_capture_ms_count", + "oso_tree_capture_ms_max"): + _metric(body, name) + + +def test_metrics_cache_counters_move(client): + client.get("/api/structure") # miss + capture + client.get("/api/structure") # hit + body = client.get("/api/metrics").get_data(as_text=True) + assert _metric(body, "oso_tree_cache_misses_total") >= 1 + assert _metric(body, "oso_tree_cache_hits_total") >= 1 + assert _metric(body, "oso_tree_capture_ms_count") >= 1 + assert _metric(body, "oso_tree_cache_entries") >= 1 + + +def test_metrics_help_and_type_lines(client): + body = client.get("/api/metrics").get_data(as_text=True) + assert "# TYPE oso_tree_cache_hits_total counter" in body + assert "# TYPE oso_tree_cache_misses_total counter" in body + assert "# TYPE oso_tree_cache_entries gauge" in body + assert "# TYPE oso_tree_capture_ms summary" in body diff --git a/tests/test_tools_p2.py b/tests/test_tools_p2.py index 0210e0e..73505ef 100644 --- a/tests/test_tools_p2.py +++ b/tests/test_tools_p2.py @@ -85,3 +85,75 @@ def test_click_and_observe(client): }).get_json() assert r["ok"] is True assert "observation" in r + + +# ─── P1 perf: changed_only + perf telemetry ────────────────────────────────── + +def _mutate_editor(tree): + for c in tree.children: + if c.role == "Document": + c.value = "changed content" + return tree + + +def test_observe_changed_only_unchanged(client): + client.get("/api/observe?window_index=0") # warm the baseline + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["changed_only"] is True + assert r["unchanged"] is True + assert r["tree_hash"].startswith("sha1:") + assert "tree" not in r and "changes" not in r # tiny response + + +def test_observe_changed_only_returns_diff(client, observer): + base = client.get("/api/observe?window_index=0").get_json() + observer._adapter.tree_mutator = _mutate_editor + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["unchanged"] is False + assert r["format"] == "custom" + assert r["changes"] # non-empty diff + assert r["tree_hash"] != base["tree_hash"] + assert "tree" not in r # diff, not full tree + + +def test_observe_changed_only_without_baseline_returns_full(client): + r = client.get("/api/observe?window_index=0&changed_only=1").get_json() + assert r["ok"] is True + assert r["format"] == "full" + assert "tree" in r + assert r["perf"]["cache"] == "bypass" + + +def test_observe_perf_telemetry(client): + r = client.get("/api/observe?window_index=0").get_json() + perf = r["perf"] + assert set(perf) == {"capture_ms", "node_count", "cache", "depth_used"} + assert perf["cache"] == "miss" + assert perf["node_count"] > 1 + assert perf["depth_used"] == 5 + + +def test_structure_perf_telemetry_cache_hit(client): + r1 = client.get("/api/structure?window_index=0").get_json() + assert r1["perf"]["cache"] == "miss" + r2 = client.get("/api/structure?window_index=0").get_json() + assert r2["perf"]["cache"] == "hit" + assert r2["perf"]["node_count"] == r1["perf"]["node_count"] + + +def test_structure_scoped_perf(client): + client.get("/api/structure?window_index=0") # warm cache + r = client.get("/api/structure?window_index=0&scope=root.4").get_json() + assert r["perf"]["cache"] == "hit" + assert r["perf"]["depth_used"] == 5 + + +def test_observe_since_perf(client): + full = client.get("/api/observe?window_index=0").get_json() + diff = client.get( + f"/api/observe?window_index=0&since={full['tree_token']}" + ).get_json() + assert "perf" in diff + assert diff["perf"]["cache"] in ("hit", "miss") diff --git a/tests/test_tools_p3.py b/tests/test_tools_p3.py index 2ada7f8..ac9ede3 100644 --- a/tests/test_tools_p3.py +++ b/tests/test_tools_p3.py @@ -55,3 +55,136 @@ def test_description_focus_element(client): f"/api/description?window_index=0&mode=accessibility&focus_element={child_id}" ).get_json() assert r["ok"] is True + + +# ─── P1 perf: bounded depth + subtree scope ────────────────────────────────── + +from observer import Bounds, UIElement # noqa: E402 + + +def _graft_deep_chain(tree, levels=9): + """Attach a linear chain of Group nodes below the tree root.""" + node = tree + for i in range(levels): + child = UIElement(f"{node.element_id}.{len(node.children)}", + f"Deep{i}", "Group", bounds=Bounds(0, 0, 5, 5)) + node.children.append(child) + node = child + return tree + + +def _max_depth_of(node, depth=0): + kids = node.get("children") or [] + if not kids: + return depth + return max(_max_depth_of(c, depth + 1) for c in kids) + + +def _find_truncated(node): + if node.get("truncated"): + return node + for c in node.get("children") or []: + f = _find_truncated(c) + if f is not None: + return f + return None + + +def test_structure_default_depth_truncates(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/structure?window_index=0").get_json() + assert r["ok"] is True + assert r["depth_used"] == 5 # tree.default_depth + assert r["depth_truncated"] is True + assert _max_depth_of(r["tree"]) == 5 + marker = _find_truncated(r["tree"]) + assert marker is not None + assert marker["child_count"] == 1 + assert marker["children"] == [] + + +def test_structure_explicit_depth_capped(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/structure?window_index=0&depth=7").get_json() + assert r["depth_used"] == 7 + assert _max_depth_of(r["tree"]) == 7 + + r = client.get("/api/structure?window_index=0&depth=20").get_json() + assert r["depth_used"] == 8 # clamped to tree.max_depth + + +def test_structure_shallow_tree_not_marked(client): + r = client.get("/api/structure?window_index=0").get_json() + assert r["depth_truncated"] is False # mock tree is only 2 deep + assert _find_truncated(r["tree"]) is None + + +def test_structure_scope_subtree(client): + r = client.get("/api/structure?window_index=0&scope=root.4").get_json() + assert r["ok"] is True + assert r["scope"] == "root.4" + assert r["tree"]["id"] == "root.4" + assert r["tree"]["role"] == "StatusBar" + assert len(r["tree"]["children"]) == 4 + assert r["tree_token"] is None # scoped ≠ diff baseline + + +def test_structure_scope_with_depth(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + deep_root = "root.7" # chain head appended at index 7 + r = client.get( + f"/api/structure?window_index=0&scope={deep_root}&depth=2" + ).get_json() + assert r["ok"] is True + assert r["tree"]["id"] == deep_root + assert _max_depth_of(r["tree"]) <= 2 + + +def test_structure_scope_not_found(client): + r = client.get("/api/structure?window_index=0&scope=root.99").get_json() + assert r["ok"] is False + assert r["error"]["code"] == "ElementNotFound" + + +def test_observe_full_is_depth_bounded(client, observer): + observer._adapter.tree_mutator = _graft_deep_chain + r = client.get("/api/observe?window_index=0").get_json() + assert r["format"] == "full" + assert r["depth_used"] == 5 + assert r["depth_truncated"] is True + assert _max_depth_of(r["tree"]) == 5 + + # Diffs still compare the FULL captures (depth only bounds the payload). + diff = client.get( + f"/api/observe?window_index=0&since={r['tree_token']}" + ).get_json() + assert diff["unchanged"] is True + + +def test_mock_adapter_subtree_walk(observer): + adapter = observer._adapter + sub = adapter.get_element_subtree(None, "root.0", max_depth=0) + assert sub.element_id == "root.0" + assert sub.role == "MenuBar" + assert sub.children == [] # depth 0 prunes children + + sub = adapter.get_element_subtree(None, "root.0", max_depth=1) + assert len(sub.children) == 5 + + +def test_observer_subtree_served_from_cache(observer): + w = observer.list_windows()[0] + observer.get_element_tree(w.handle, window_uid=w.window_uid) + assert observer._adapter.capture_count == 1 + sub = observer.get_element_subtree(w.handle, "root.4", max_depth=2, + window_uid=w.window_uid) + assert sub is not None and sub.element_id == "root.4" + assert observer._adapter.capture_count == 1 # extracted from cache + + # Extraction copies — the cached tree keeps its children intact. + shallow = observer.get_element_subtree(w.handle, "root.4", max_depth=0, + window_uid=w.window_uid) + assert shallow.children == [] + full = observer.get_element_tree(w.handle, window_uid=w.window_uid) + sb = next(c for c in full.children if c.element_id == "root.4") + assert len(sb.children) == 4 diff --git a/tests/test_tree_cache.py b/tests/test_tree_cache.py new file mode 100644 index 0000000..85f814d --- /dev/null +++ b/tests/test_tree_cache.py @@ -0,0 +1,262 @@ +"""Tests for the per-window tree cache (P1 perception performance).""" +from __future__ import annotations + +import time + +import pytest + +import tools as _tools +from observer import Bounds, UIElement +from session import get_session +from tree_cache import TreeCache + + +# ─── Helpers / fixtures ─────────────────────────────────────────────────────── + +def _make_tree(name: str = "T") -> UIElement: + return UIElement("root", name, "Window", bounds=Bounds(0, 0, 10, 10)) + + +def _entry_kwargs(name: str = "T") -> dict: + t = _make_tree(name) + return dict(tree=t, serialized=t.to_dict(), tree_hash=f"sha1:{name}", + max_depth=8, capture_ms=1, node_count=1, named_node_count=1) + + +@pytest.fixture() +def ctx(config, observer, renderer, describer): + return _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + +def _first_window(observer): + w = observer.list_windows()[0] + return w.handle, w.window_uid + + +# ─── TreeCache unit tests ───────────────────────────────────────────────────── + +def test_cache_put_get_hit(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs("A")) + e = c.get("w1") + assert e is not None + assert e.tree_hash == "sha1:A" + assert c.get("nope") is None + + +def test_cache_ttl_expiry(): + c = TreeCache(ttl_s=60) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 120 # age the entry past the TTL + assert c.get("w1") is None # expired entries are dropped + assert "w1" not in c + + +def test_cache_per_call_ttl_override(): + c = TreeCache(ttl_s=0.0) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 1 + assert c.get("w1", ttl_s=60) is not None + + +def test_cache_peek_ignores_ttl(): + c = TreeCache(ttl_s=60) + entry = c.put("w1", **_entry_kwargs()) + entry.captured_at -= 120 + assert c.peek("w1") is not None # baseline lookups ignore TTL + + +def test_cache_invalidate(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs()) + assert c.invalidate("w1") is True + assert c.get("w1") is None + assert c.invalidate("w1") is False + + +def test_cache_lru_eviction(): + c = TreeCache(ttl_s=60, max_windows=3) + for i in range(3): + c.put(f"w{i}", **_entry_kwargs()) + c.get("w0") # touch w0 → w1 becomes LRU + c.put("w3", **_entry_kwargs()) + assert "w0" in c and "w2" in c and "w3" in c + assert "w1" not in c + assert len(c) == 3 + + +def test_cache_stats_survive_invalidation(): + c = TreeCache(ttl_s=60) + c.put("w1", **_entry_kwargs()) + c.invalidate("w1") + stats = c.stats() + assert "w1" in stats + assert stats["w1"]["node_count"] == 1 + assert stats["w1"]["named_node_count"] == 1 + + +# ─── Observer-level caching ─────────────────────────────────────────────────── + +def test_observer_cache_hit_avoids_walk(observer): + hwnd, uid = _first_window(observer) + adapter = observer._adapter + t1 = observer.get_element_tree(hwnd, window_uid=uid) + assert adapter.capture_count == 1 + t2 = observer.get_element_tree(hwnd, window_uid=uid) + assert adapter.capture_count == 1 # served from cache + assert t2 is t1 + + +def test_observer_cache_meta_hit_miss_bypass(observer): + hwnd, uid = _first_window(observer) + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid) + assert meta["cache"] == "miss" + assert meta["node_count"] > 1 + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid) + assert meta["cache"] == "hit" + _, meta = observer.get_element_tree_with_meta(hwnd, window_uid=uid, + use_cache=False) + assert meta["cache"] == "bypass" + assert observer._adapter.capture_count == 2 + + +def test_observer_cache_ttl_expiry_rewalks(observer): + hwnd, uid = _first_window(observer) + observer.get_element_tree(hwnd, window_uid=uid) + entry = get_session().tree_cache.peek(uid) + entry.captured_at = time.time() - 100 # simulate TTL expiry + observer.get_element_tree(hwnd, window_uid=uid) + assert observer._adapter.capture_count == 2 + + +def test_observer_no_uid_is_never_cached(observer): + hwnd, _ = _first_window(observer) + observer.get_element_tree(hwnd) + observer.get_element_tree(hwnd) + assert observer._adapter.capture_count == 2 + assert len(get_session().tree_cache) == 0 + + +def test_observer_bypass_still_refreshes_cache(observer): + hwnd, uid = _first_window(observer) + observer.get_element_tree(hwnd, window_uid=uid, use_cache=False) + assert get_session().tree_cache.peek(uid) is not None + + +# ─── Invalidation through tools.dispatch ───────────────────────────────────── + +def test_input_tool_invalidates_target_window(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert uid in get_session().tree_cache + + r = _tools.dispatch(ctx, "click_element", { + "window_index": 0, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }) + assert r["ok"] is True + assert uid not in get_session().tree_cache + + +def test_legacy_input_tool_invalidates_all(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert uid in get_session().tree_cache + + _tools.dispatch(ctx, "type_text", {"text": "hi"}) # no window binding + assert len(get_session().tree_cache) == 0 + + +def test_read_only_tool_does_not_invalidate(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + before = observer._adapter.capture_count + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert observer._adapter.capture_count == before # cache hit + assert uid in get_session().tree_cache + + +def test_receipt_after_state_bypasses_cache(ctx, observer): + """A mutation applied after the cache is warmed must be visible in the + ActionReceipt's after-state (post-action reads bypass the cache).""" + adapter = observer._adapter + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + + def _rename(tree): + tree.name = "MUTATED TITLE" + return tree + + adapter.tree_mutator = _rename + r = _tools.dispatch(ctx, "click_element", { + "window_index": 0, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }) + assert r["ok"] is True + # before came from the warm cache; after re-walked and saw the mutation. + assert r["before"]["tree_hash"] != r["after"]["tree_hash"] + assert r["changed"] is True + + +def test_mock_adapter_mutation_hook(observer): + adapter = observer._adapter + adapter.tree_mutator = lambda t: UIElement( + "root", "Replaced", "Window", bounds=Bounds(0, 0, 1, 1)) + tree = observer.get_element_tree(None) + assert tree.name == "Replaced" + assert adapter.capture_count == 1 + + +# ─── P1: degradation signal (sparse accessibility trees) ───────────────────── + +def _sparse_tree(tree): + """Simulate an accessibility-dark window: one named child only.""" + return UIElement( + "root", "Game Window", "Window", bounds=Bounds(0, 0, 800, 600), + children=[ + UIElement("root.0", "Canvas", "Pane", bounds=Bounds(0, 0, 800, 600)), + UIElement("root.1", "", "Pane", bounds=Bounds(0, 0, 10, 10)), + ]) + + +def test_sparse_tree_attaches_degraded(ctx, observer): + observer._adapter.tree_mutator = _sparse_tree + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["ok"] is True + d = r["degraded"] + assert d["reason"] == "sparse_accessibility_tree" + assert d["named_node_count"] == 1 # unnamed nodes don't count + assert d["suggested_fallbacks"] == ["get_ocr", "get_screen_description"] + + +def test_rich_tree_has_no_degraded(ctx): + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["ok"] is True + assert "degraded" not in r + + +def test_sparse_threshold_configurable(ctx, observer, config): + config.setdefault("tree", {})["sparse_threshold"] = 100 + r = _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + assert r["degraded"]["threshold"] == 100 # even the rich mock tree trips + + +def test_capabilities_reports_tree_stats(ctx, observer): + _, uid = _first_window(observer) + caps0 = _tools.dispatch(ctx, "get_capabilities", {}) + assert caps0["tree_stats"] == {} # nothing captured yet + + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + caps = _tools.dispatch(ctx, "get_capabilities", {}) + stats = caps["tree_stats"][uid] + assert stats["node_count"] > 1 + assert stats["named_node_count"] >= 5 + assert "capture_ms" in stats and "captured_at" in stats + + +def test_capabilities_stats_survive_input_invalidation(ctx, observer): + _, uid = _first_window(observer) + _tools.dispatch(ctx, "get_window_structure", {"window_index": 0}) + _tools.dispatch(ctx, "type_text", {"text": "x"}) # invalidates cache + caps = _tools.dispatch(ctx, "get_capabilities", {}) + assert uid in caps["tree_stats"] diff --git a/tests/test_uia_walk.py b/tests/test_uia_walk.py new file mode 100644 index 0000000..6236055 --- /dev/null +++ b/tests/test_uia_walk.py @@ -0,0 +1,267 @@ +"""Unit tests for the WindowsAdapter raw-UIA walker (P1 perf). + +Real COM/UIA is unavailable off-Windows, so these tests inject fake +comtypes/pywinauto/pywin32 modules via sys.modules and exercise both the +CacheRequest bulk-fetch path and the per-property fallback path. +Manual verification on real Windows is still required. +""" +from __future__ import annotations + +import sys +import types + +import pytest + +from observer import WindowsAdapter + + +# ─── Fake COM / UIA infrastructure ─────────────────────────────────────────── + +_NAME, _CTRL_TYPE = 30005, 30003 +_BOUNDING_RECT = 30001 + + +class FakeRect: + def __init__(self, left=0, top=0, right=10, bottom=10): + self.left, self.top, self.right, self.bottom = left, top, right, bottom + + +class FakeElementList: + def __init__(self, items): + self._items = list(items) + + @property + def Length(self): + return len(self._items) + + def GetElement(self, i): + return self._items[i] + + +class FakeUIAElement: + """IUIAutomationElement stand-in with call counters.""" + + def __init__(self, name, ctrl_type=50000, children=(), + fail_build_cache=False): + self._props = {_NAME: name, _CTRL_TYPE: ctrl_type} + self._children = list(children) + self.fail_build_cache = fail_build_cache + self.current_calls = 0 # GetCurrentPropertyValue round trips + self.cached_calls = 0 + self.find_all_calls = 0 + self.build_cache_calls = 0 + self.fetched_via_cache = False + + # Per-property (cross-process on real Windows). + def GetCurrentPropertyValue(self, pid): + self.current_calls += 1 + return self._props.get(pid) + + def GetCachedPropertyValue(self, pid): + self.cached_calls += 1 + if pid == _BOUNDING_RECT: + return [1.0, 2.0, 3.0, 4.0] + return self._props.get(pid) + + @property + def CurrentBoundingRectangle(self): + self.current_calls += 1 + return FakeRect() + + @property + def CachedBoundingRectangle(self): + raise RuntimeError("not cached — use GetCachedPropertyValue") + + def FindAll(self, scope, cond): + self.find_all_calls += 1 + return FakeElementList(self._children) + + def FindAllBuildCache(self, scope, cond, cache_request): + self.build_cache_calls += 1 + if self.fail_build_cache: + raise RuntimeError("BuildCache unsupported here") + assert cache_request is not None + for c in self._children: + c.fetched_via_cache = True + return FakeElementList(self._children) + + +class FakeCacheRequest: + def __init__(self): + self.properties = [] + + def AddProperty(self, pid): + self.properties.append(pid) + + +class FakeRawUIA: + def __init__(self, root, cache_request_fails=False): + self._root = root + self.cache_request_fails = cache_request_fails + self.cache_requests_built = 0 + + def ElementFromHandle(self, hwnd): + return self._root + + def CreateTrueCondition(self): + return object() + + def CreateCacheRequest(self): + if self.cache_request_fails: + raise RuntimeError("CacheRequest not supported") + self.cache_requests_built += 1 + return FakeCacheRequest() + + +class FakeApplication: + """pywinauto.Application stand-in that records connect() attempts.""" + connect_calls = 0 + + def __init__(self, backend=None): + pass + + def connect(self, handle=None): + FakeApplication.connect_calls += 1 + raise RuntimeError("no real UIA here") # → pw_tree stays None + + +def _make_fake_tree(): + """root → [child0(Button) → [grand0(Text)], child1(Edit)]""" + grand0 = FakeUIAElement("Grand", 50020) + child0 = FakeUIAElement("Child A", 50000, children=[grand0]) + child1 = FakeUIAElement("Child B", 50004) + root = FakeUIAElement("Root Window", 50032, children=[child0, child1]) + return root, child0, child1, grand0 + + +@pytest.fixture() +def fake_win(monkeypatch): + """Install fake win32/pywinauto modules; returns a factory that builds a + WindowsAdapter wired to a fake UIA tree.""" + win32gui = types.ModuleType("win32gui") + win32gui.GetForegroundWindow = lambda: 1 + monkeypatch.setitem(sys.modules, "win32gui", win32gui) + monkeypatch.setitem(sys.modules, "win32process", + types.ModuleType("win32process")) + monkeypatch.setitem(sys.modules, "psutil", types.ModuleType("psutil")) + + FakeApplication.connect_calls = 0 + pywinauto = types.ModuleType("pywinauto") + pywinauto.Application = FakeApplication + monkeypatch.setitem(sys.modules, "pywinauto", pywinauto) + + uia_defines = types.ModuleType("pywinauto.uia_defines") + pywinauto.uia_defines = uia_defines + monkeypatch.setitem(sys.modules, "pywinauto.uia_defines", uia_defines) + + def _build(config=None, cache_request_fails=False): + root, *rest = _make_fake_tree() + raw = FakeRawUIA(root, cache_request_fails=cache_request_fails) + + class FakeIUIA: + def __init__(self): + self.iuia = raw + + uia_defines.IUIA = FakeIUIA + adapter = WindowsAdapter(config or {"tree": {"max_depth": 8}}) + return adapter, raw, root, rest + + return _build + + +# ─── CacheRequest bulk-fetch path ──────────────────────────────────────────── + +def test_uia_walk_uses_cache_request(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win() + tree = adapter._uia_walk(1) + + assert tree is not None + assert tree.name == "Root Window" + assert [c.name for c in tree.children] == ["Child A", "Child B"] + assert tree.children[0].children[0].name == "Grand" + assert tree.children[0].children[0].element_id == "root.0.0" + + assert raw.cache_requests_built == 1 + # Children were fetched with FindAllBuildCache, not FindAll. + assert root.build_cache_calls == 1 + assert root.find_all_calls == 0 + # Cached elements were read via GetCachedPropertyValue — zero per-node + # GetCurrentPropertyValue round trips. + for el in (child0, child1, grand0): + assert el.fetched_via_cache is True + assert el.current_calls == 0 + assert el.cached_calls > 0 + # Cached bounds came from the BoundingRectangle SAFEARRAY fallback. + assert tree.children[0].bounds.to_dict() == { + "x": 1, "y": 2, "width": 3, "height": 4} + + +def test_cache_request_covers_walked_properties(fake_win): + adapter, raw, _, _ = fake_win() + cr = adapter._uia_cache_request(raw) + assert _NAME in cr.properties and _CTRL_TYPE in cr.properties + assert _BOUNDING_RECT in cr.properties + assert len(cr.properties) >= 12 + + +# ─── Fallback paths ────────────────────────────────────────────────────────── + +def test_uia_walk_falls_back_when_cache_request_fails(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win( + cache_request_fails=True) + tree = adapter._uia_walk(1) + + assert tree is not None + assert [c.name for c in tree.children] == ["Child A", "Child B"] + # No BuildCache attempted; classic FindAll + per-property fetches. + assert root.build_cache_calls == 0 + assert root.find_all_calls == 1 + for el in (child0, child1): + assert el.fetched_via_cache is False + assert el.current_calls > 0 + + +def test_uia_walk_falls_back_per_node_when_build_cache_raises(fake_win): + adapter, raw, root, (child0, child1, grand0) = fake_win() + child0.fail_build_cache = True # BuildCache breaks below child0 + tree = adapter._uia_walk(1) + + assert tree is not None + # child0's children still walked, via the FindAll fallback… + assert child0.find_all_calls == 1 + assert tree.children[0].children[0].name == "Grand" + # …and grand0 was therefore read per-property, not from a cache. + assert grand0.fetched_via_cache is False + assert grand0.current_calls > 0 + + +# ─── tree.strategy = uia_only ──────────────────────────────────────────────── + +def test_uia_only_skips_pywinauto_walk(fake_win): + adapter, _, _, _ = fake_win(config={ + "tree": {"max_depth": 8, "strategy": "uia_only"}}) + tree = adapter.get_element_tree(1) + assert tree is not None + assert tree.name == "Root Window" + assert FakeApplication.connect_calls == 0 + + +def test_merged_strategy_still_attempts_pywinauto(fake_win): + adapter, _, _, _ = fake_win(config={ + "tree": {"max_depth": 8, "strategy": "merged"}}) + tree = adapter.get_element_tree(1) + assert tree is not None # uia tree survives pw failure + assert FakeApplication.connect_calls == 1 + + +# ─── Subtree navigation (commit 2 path, exercised against fake COM) ───────── + +def test_get_element_subtree_navigates_indices(fake_win): + adapter, _, root, (child0, child1, grand0) = fake_win() + sub = adapter.get_element_subtree(1, "root.0", max_depth=8) + assert sub is not None + assert sub.element_id == "root.0" + assert sub.name == "Child A" + assert sub.children[0].name == "Grand" + # Only the requested branch was walked — child1 untouched. + assert child1.current_calls == 0 and child1.cached_calls == 0 diff --git a/tests/test_untrusted_and_confirmation.py b/tests/test_untrusted_and_confirmation.py new file mode 100644 index 0000000..76cfc2a --- /dev/null +++ b/tests/test_untrusted_and_confirmation.py @@ -0,0 +1,231 @@ +"""[P2] Trust-boundary tests: untrusted-content marking on screen-derived +results, ANSI/control-character sanitization, and the design-doc §21 +confirmation-token audit (every destructive element-targeted verb must +route through the confirmation gate when confirmation mode is enabled). +""" +from __future__ import annotations + +import pytest + +import tools as _tools +from redaction import ( + UNTRUSTED_RESULT_TOOLS, mark_untrusted, sanitize_screen_text, +) + + +# ── sanitize_screen_text ───────────────────────────────────────────────────── + + +def test_sanitize_strips_csi_sequences(): + assert sanitize_screen_text("a\x1b[31mred\x1b[0mb") == "aredb" + + +def test_sanitize_strips_osc_sequences(): + assert sanitize_screen_text("x\x1b]0;evil title\x07y") == "xy" + + +def test_sanitize_strips_control_chars(): + assert sanitize_screen_text("a\x00b\x08c\x7fd") == "abcd" + + +def test_sanitize_keeps_layout_whitespace(): + assert sanitize_screen_text("line1\nline2\tcol\r") == "line1\nline2\tcol\r" + + +def test_sanitize_non_string_passthrough(): + assert sanitize_screen_text(None) is None + assert sanitize_screen_text(42) == 42 + assert sanitize_screen_text("") == "" + + +# ── mark_untrusted ─────────────────────────────────────────────────────────── + + +def test_mark_untrusted_flags_and_sanitizes_nested_text(): + result = { + "ok": True, + "tree": {"name": "evil\x1b[2Jname", "value": "v\x00v", + "children": [{"name": "child\x1b[31m"}]}, + "windows": [{"title": "T\x07itle"}], + } + out = mark_untrusted("get_window_structure", result) + assert out["untrusted"] is True + assert out["tree"]["name"] == "evilname" + assert out["tree"]["value"] == "vv" + assert out["tree"]["children"][0]["name"] == "child" + assert out["windows"][0]["title"] == "Title" + + +def test_mark_untrusted_skips_opaque_fields(): + result = {"ok": True, "data": "base64\x1b[31mstuff", + "tree_token": "tt:ab\x00cd"} + out = mark_untrusted("get_ocr", result) + assert out["data"] == "base64\x1b[31mstuff" # untouched + assert out["tree_token"] == "tt:ab\x00cd" # untouched + assert out["untrusted"] is True + + +def test_mark_untrusted_noop_for_action_tools(): + result = {"ok": True, "action": "click_element"} + out = mark_untrusted("click_element", result) + assert "untrusted" not in out + + +def test_untrusted_tool_list_is_read_tools_only(): + # No destructive verb should be in the untrusted-marking list; + # perception tools carry the screen text. + for name in UNTRUSTED_RESULT_TOOLS: + assert not _tools._is_input_tool(name) + + +# ── Wire-through: REST results carry the flag ──────────────────────────────── + + +def test_windows_result_is_untrusted(client): + data = client.get("/api/windows").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_structure_result_is_untrusted(client): + data = client.get("/api/structure").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_observe_result_is_untrusted(client): + data = client.get("/api/observe").get_json() + assert data["ok"] is True + assert data["untrusted"] is True + + +def test_action_receipt_not_flagged(client): + ws = client.get("/api/windows").get_json() + uid = ws["windows"][0]["window_uid"] + r = client.post("/api/element/click", json={ + "window_uid": uid, + "selector": 'Window/MenuBar/MenuItem[name="File"]', + }).get_json() + assert r["ok"] is True + assert "untrusted" not in r + + +# ── §21 confirmation audit ─────────────────────────────────────────────────── +# +# Every destructive verb that resolves a concrete element target must route +# through _check_confirmation when confirmation_required rules are set. +# +# Coordinate/global verbs carry no element identity, so the role/name-based +# rules of §21 cannot apply to them by construction. They are the documented +# exclusion list below; the completeness test asserts every input tool is +# accounted for in exactly one of the two groups. + +ELEMENT_VERBS = [ + ("click_element", {}), + ("focus_element", {}), + ("set_value", {"value": "x"}), + ("invoke_element", {}), + ("select_option", {"option_name": "x"}), + ("hover_element", {"hover_ms": 1}), + ("right_click_element", {}), + ("double_click_element", {}), + ("key_into_element", {"keys": "ctrl+a"}), + ("clear_text", {}), + ("click_element_and_observe", {}), +] + +# §21 scope limit: rules match element role/name; these verbs act on raw +# coordinates / global input focus and never resolve an element, so no +# rule can name them a destructive target. (click_element_and_observe is +# gated via click_element; type/press_key composites wrap the excluded +# global verbs.) `drag` IS gated when either endpoint is element-addressed. +COORDINATE_VERB_EXCLUSIONS = { + "click_at", "right_click_at", "double_click_at", + "type_text", "press_key", "scroll", + "hover_at", "bring_to_foreground", + "type_and_observe", "press_key_and_observe", +} + + +@pytest.fixture() +def confirm_ctx(config, observer, renderer, describer): + config["confirmation_required"] = [{"name_regex": "(?i)file"}] + return _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + +def _file_menu_args(ctx): + win = ctx.observer.list_windows()[0] + return {"window_uid": win.window_uid, + "selector": 'Window/MenuBar/MenuItem[name="File"]'} + + +@pytest.mark.parametrize("verb,extra", ELEMENT_VERBS) +def test_element_verb_requires_confirmation(confirm_ctx, verb, extra): + args = {**_file_menu_args(confirm_ctx), **extra} + r = _tools.dispatch(confirm_ctx, verb, args) + assert r["ok"] is False, verb + assert r["error"]["code"] == "ConfirmationRequired", verb + + +def test_drag_requires_confirmation_for_element_endpoint(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationRequired" + + +def test_drag_requires_confirmation_for_destination_endpoint(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"x": 100, "y": 100}, + "to": {"selector": base["selector"]}, + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationRequired" + + +def test_drag_with_valid_token_passes_gate(confirm_ctx): + base = _file_menu_args(confirm_ctx) + prop = _tools.dispatch(confirm_ctx, "propose_action", { + "action": "drag", + "args": base, + }) + assert prop["ok"] is True + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + "confirm_token": prop["confirm_token"], + }) + # The gate is satisfied; the drag itself may still fail on headless CI + # (pyautogui unavailable) — but never with a confirmation error. + if r["ok"] is False: + assert r["error"]["code"] not in ("ConfirmationRequired", + "ConfirmationInvalid") + + +def test_drag_with_bogus_token_rejected(confirm_ctx): + base = _file_menu_args(confirm_ctx) + r = _tools.dispatch(confirm_ctx, "drag", { + "window_uid": base["window_uid"], + "from": {"selector": base["selector"]}, + "to": {"x": 300, "y": 300}, + "confirm_token": "ct:doesnotexist", + }) + assert r["ok"] is False + assert r["error"]["code"] == "ConfirmationInvalid" + + +def test_confirmation_audit_covers_every_input_tool(): + gated = {v for v, _ in ELEMENT_VERBS} | {"drag"} + accounted = gated | COORDINATE_VERB_EXCLUSIONS + input_tools = {n for n in _tools.REGISTRY if _tools._is_input_tool(n)} + unaccounted = input_tools - accounted + assert not unaccounted, ( + f"input tools missing from the §21 audit: {sorted(unaccounted)}") diff --git a/tools.py b/tools.py deleted file mode 100644 index 27ef2e1..0000000 --- a/tools.py +++ /dev/null @@ -1,2103 +0,0 @@ -""" -tools.py — Central tool implementations. - -Both mcp_server.py and web_inspector.py dispatch into this module; the -MCP and REST surfaces are thin wrappers. Every tool returns a dict in -one of two shapes: - - {ok: true, step_id: …, …tool-specific fields…} - {ok: false, success: false, step_id: …, error: {code, message, …}} - -For backwards compatibility (design doc D5) success-shaped legacy fields -are preserved alongside the new `ok` / `error` object on existing tools. -""" - -from __future__ import annotations - -import base64 -import logging -import re -import time -from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple - -import element_selectors as sel -from errors import Code, error_dict, annotate_legacy_result -from hashing import focused_selector, tree_hash -from observer import ( - ScreenObserver, UIElement, WindowInfo, WindowResolution, -) -from session import Session, get_session - -logger = logging.getLogger(__name__) - - -# ─── Context ────────────────────────────────────────────────────────────────── - -@dataclass -class ToolContext: - observer: ScreenObserver - renderer: Any - describer: Any - config: Dict[str, Any] - - @property - def session(self) -> Session: - return get_session() - - -# ─── Helpers ────────────────────────────────────────────────────────────────── - -def _is_input_tool(name: str) -> bool: - return name in { - "click_at", "type_text", "press_key", "scroll", "bring_to_foreground", - "click_element", "focus_element", "set_value", "invoke_element", - "select_option", "hover_at", "hover_element", - "right_click_at", "right_click_element", - "double_click_at", "double_click_element", - "drag", "key_into_element", "clear_text", - "click_element_and_observe", "type_and_observe", "press_key_and_observe", - } - - -def _new_step_id(name: str) -> Tuple[int, Optional[int]]: - return get_session().steps.next_id(is_input=_is_input_tool(name)) - - -def _resolve_window(ctx: ToolContext, args: Dict[str, Any] - ) -> Tuple[List[WindowInfo], WindowResolution]: - windows = ctx.observer.list_windows() - res = ctx.observer.resolve_window( - windows, - window_uid=args.get("window_uid"), - window_index=args.get("window_index"), - window_title=args.get("window_title"), - ) - return windows, res - - -def _focused_window(windows: List[WindowInfo]) -> Optional[WindowInfo]: - for w in windows: - if w.is_focused: - return w - return windows[0] if windows else None - - -def _resolve_element(tree: UIElement, args: Dict[str, Any] - ) -> Tuple[Optional[UIElement], Optional[str], Optional[Dict]]: - """ - Returns (element, selector_string, error_dict). Either *element* is set - or *error_dict* is. The selector string is resolved or derived. - """ - selector = args.get("selector") - element_id = args.get("element_id") - - # Try selector first, then element_id. When both are provided, element_id - # acts as a fallback so a bad/unmatched selector doesn't block the call. - selector_err = None - if selector: - try: - parsed = sel.parse(selector) - result = sel.resolve(tree, parsed) - if result.matches: - return result.matches[0], parsed.canonical(), None - selector_err = error_dict( - Code.ELEMENT_NOT_FOUND, - f"no element matches selector {selector!r}", - selector=selector, - ) - except sel.SelectorParseError as e: - selector_err = error_dict(Code.BAD_REQUEST, - f"selector parse error: {e}", - selector=selector) - - if element_id: - elem = _find_by_id(tree, element_id) - if elem is not None: - derived = sel.selector_for(tree, element_id) or "" - return elem, derived, None - # element_id didn't match any internal ID — try parsing it as a selector - # (LLMs often pass the display-format string e.g. 'TabItem "name"' as an id). - try: - parsed = sel.parse(element_id) - result = sel.resolve(tree, parsed) - if result.matches: - return result.matches[0], parsed.canonical(), None - except (sel.SelectorParseError, Exception): - pass - # Both failed — prefer selector error if we have one (more informative). - if selector_err: - return None, None, selector_err - return None, None, error_dict( - Code.ELEMENT_NOT_FOUND, - f"no element with id {element_id!r}", - element_id=element_id, - ) - - if selector_err: - return None, None, selector_err - - return None, None, error_dict(Code.BAD_REQUEST, - "either element_id or selector is required") - - -def _find_by_id(elem: UIElement, target: str) -> Optional[UIElement]: - if elem.element_id == target: - return elem - for c in elem.children: - r = _find_by_id(c, target) - if r is not None: - return r - return None - - -def _new_dialogs(before: List[WindowInfo], after: List[WindowInfo]) -> List[Dict]: - before_uids = {w.window_uid for w in before} - return [ - {"window_uid": w.window_uid, "title": w.title} - for w in after if w.window_uid and w.window_uid not in before_uids - ] - - -# ─── Read-only tools ────────────────────────────────────────────────────────── - -def list_windows(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("list_windows") - try: - windows = ctx.observer.list_windows() - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "is_mock": ctx.observer.is_mock, - "count": len(windows), - "windows": [{"index": i, **w.to_dict()} for i, w in enumerate(windows)], - } - except Exception as e: - logger.exception("list_windows failed") - return error_dict(Code.INTERNAL, str(e), step_id=step_id) - - -def get_capabilities(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_capabilities") - out = ctx.observer.get_capabilities() - out.update({"step_id": step_id, "caused_by_step_id": caused_by, "success": True}) - return out - - -def get_monitors(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_monitors") - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "monitors": ctx.observer.get_monitors(), - } - - -def find_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("find_element") - selector_text = args.get("selector") - if not selector_text: - return error_dict(Code.BAD_REQUEST, "selector is required", - step_id=step_id) - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - try: - parsed = sel.parse(selector_text) - except sel.SelectorParseError as e: - return error_dict(Code.BAD_REQUEST, f"selector parse error: {e}", - step_id=step_id) - result = sel.resolve(tree, parsed) - if not result.matches: - return error_dict(Code.ELEMENT_NOT_FOUND, - f"no element matches {selector_text!r}", - step_id=step_id, selector=selector_text, - window_uid=info.window_uid) - first = result.matches[0] - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, - "element_id": first.element_id, - "selector": parsed.canonical(), - "bounds": first.bounds.to_dict(), - "ambiguous_matches": len(result.matches), - "all_matches": [ - {"element_id": m.element_id, "bounds": m.bounds.to_dict(), - "name": m.name, "role": m.role} - for m in result.matches - ], - } - - -# ─── Receipts and action wrappers ──────────────────────────────────────────── - -def _build_receipt(*, step_id: int, action: str, target: Dict[str, Any], - before_tree: Optional[UIElement], - before_windows: List[WindowInfo], - after_tree: Optional[UIElement], - after_windows: List[WindowInfo], - duration_ms: int, dry_run: bool, ok: bool, - extra: Optional[Dict[str, Any]] = None, - error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - before_hash = tree_hash(before_tree) if before_tree else "" - after_hash = tree_hash(after_tree) if after_tree else "" - receipt: Dict[str, Any] = { - "ok": ok, "success": ok, - "step_id": step_id, "caused_by_step_id": step_id, - "action": action, - "dry_run": dry_run, - "target": target, - "before": { - "tree_hash": before_hash, - "focused_selector": focused_selector(before_tree) if before_tree else "", - }, - "after": { - "tree_hash": after_hash, - "focused_selector": focused_selector(after_tree) if after_tree else "", - }, - "changed": (before_hash != after_hash) and not dry_run, - "new_dialogs": _new_dialogs(before_windows, after_windows), - "duration_ms": duration_ms, - } - if extra: - receipt.update(extra) - if error: - receipt["error"] = error - return receipt - - -def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, Any], - executor: Callable[[UIElement, WindowInfo, Dict[str, Any]], - Dict[str, Any]]) -> Dict[str, Any]: - step_id, _ = _new_step_id(action_name) - dry_run = bool(args.get("dry_run")) - - # Budget gate (no-op until P5 plumbs budgets in). - sess = get_session() - if sess.budgets is not None: - gate = sess.budgets.gate(action_name) - if gate is not None: - return {**gate, "step_id": step_id} - - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - - elem, selector_str, err = _resolve_element(tree, args) - if err: - return {**err, "step_id": step_id} - - if not elem.enabled: - return error_dict(Code.ELEMENT_DISABLED, - f"element is disabled: {selector_str}", - step_id=step_id, selector=selector_str) - - occluded = ctx.observer.is_element_occluded(elem.bounds, info.handle, windows) - if occluded: - return error_dict(Code.ELEMENT_OCCLUDED, - f"element is occluded: {selector_str}", - step_id=step_id, selector=selector_str) - - target = { - "window_uid": info.window_uid, - "element_id": elem.element_id, - "selector": selector_str, - "bounds": elem.bounds.to_dict(), - } - - # Confirmation gate (no-op until P5 plumbs confirms in). - confirm_check = _check_confirmation(ctx, action_name, args, target) - if confirm_check is not None: - return {**confirm_check, "step_id": step_id} - - started = time.time() - if dry_run: - executor_result = {"success": True, "dry_run": True} - else: - try: - executor_result = executor(elem, info, args) - except Exception as e: - logger.exception(f"{action_name} executor failed") - executor_result = {"success": False, "error": str(e)} - - duration_ms = int((time.time() - started) * 1000) - after_windows = ctx.observer.list_windows() - info_after = ctx.observer.window_by_uid(after_windows, info.window_uid) - after_tree = (ctx.observer.get_element_tree(info_after.handle) - if info_after else None) - - ok = bool(executor_result.get("success", True)) - err_obj = None - if not ok: - err_obj = { - "code": executor_result.get("error_code", Code.INTERNAL), - "message": str(executor_result.get("error", "action failed")), - "recoverable": False, - "suggested_next_tool": None, - "context": {}, - } - - extra: Dict[str, Any] = {} - if "warning" in (res.warning or ""): - extra["warning"] = res.warning - if res.warning: - extra["warning"] = res.warning - - return _build_receipt( - step_id=step_id, action=action_name, target=target, - before_tree=tree, before_windows=windows, - after_tree=after_tree, after_windows=after_windows, - duration_ms=duration_ms, dry_run=dry_run, ok=ok, - extra=extra, error=err_obj, - ) - - -def _check_confirmation(ctx: ToolContext, action_name: str, - args: Dict[str, Any], - target: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Returns an error dict to short-circuit when confirmation is required.""" - confirm = ctx.config.get("confirmation_required") or [] - if not confirm: - return None - name_to_test = "" - role_to_test = "" - # Best-effort: derive name/role from the selector tail. - sel_tail = (target.get("selector") or "").split("/")[-1] - m = re.match(r"([A-Za-z_*]\w*)", sel_tail) - if m: - role_to_test = m.group(1) - nm = re.search(r'name="([^"]*)"', sel_tail) - if nm: - name_to_test = nm.group(1) - - def _matches_rule(rule: Dict[str, Any]) -> bool: - rname = rule.get("name_regex") - rrole = rule.get("role") - if rrole and role_to_test != rrole: - return False - if rname and not re.search(rname, name_to_test or ""): - return False - return bool(rname or rrole) - - if not any(_matches_rule(r) for r in confirm): - return None - - token = args.get("confirm_token") - if not token: - return error_dict( - Code.CONFIRMATION_REQUIRED, - f"action {action_name} requires a confirm_token from propose_action", - action=action_name, target=target, - ) - sess = get_session() - ct = sess.confirms.consume(token) - if not ct: - return error_dict(Code.CONFIRMATION_INVALID, - "confirm_token expired, unknown, or already used", - token=token) - if ct.action != action_name or ct.window_uid != target["window_uid"] \ - or ct.selector != target["selector"]: - return error_dict(Code.CONFIRMATION_INVALID, - "confirm_token does not match the proposed action") - tol = (ctx.config.get("confirmation", {}) or {}).get("bbox_tolerance_px", 20) - bb = target["bounds"] - if (abs(bb["x"] - ct.bbox.get("x", 0)) > tol or - abs(bb["y"] - ct.bbox.get("y", 0)) > tol or - abs(bb["width"] - ct.bbox.get("width", 0)) > tol or - abs(bb["height"] - ct.bbox.get("height", 0)) > tol): - return error_dict(Code.CONFIRMATION_INVALID, - "element bounds drifted beyond confirmation tolerance") - return None - - -# ─── Element-targeted actions ───────────────────────────────────────────────── - -def click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - button = args.get("button", "left") - count = int(args.get("count", 1)) - - def _exec(elem: UIElement, info: WindowInfo, _args: Dict[str, Any] - ) -> Dict[str, Any]: - cx, cy = elem.bounds.center_x, elem.bounds.center_y - result = ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": cx, "y": cy, "button": button, - "double": (count >= 2)}, - hwnd=info.handle, - ) - return result - - return _do_element_action(ctx, action_name="click_element", - args=args, executor=_exec) - - -def focus_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - # Focus via center click (universal fallback). Adapter-specific - # SetFocus paths live in the platform adapters; today the universal - # click is sufficient on all three platforms. - return ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - return _do_element_action(ctx, action_name="focus_element", - args=args, executor=_exec) - - -def set_value(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - if "value" not in args: - step_id, _ = _new_step_id("set_value") - return error_dict(Code.BAD_REQUEST, "value is required", - step_id=step_id) - value = args["value"] - clear_first = bool(args.get("clear_first", True)) - - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - if clear_first: - ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) - ctx.observer.perform_action("key", value="delete", hwnd=info.handle) - return ctx.observer.perform_action("type", value=str(value), - hwnd=info.handle) - return _do_element_action(ctx, action_name="set_value", - args=args, executor=_exec) - - -def invoke_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - # No platform-specific InvokePattern surface yet; behaves like click. - return _do_element_action( - ctx, action_name="invoke_element", args=args, - executor=lambda elem, info, _a: ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ), - ) - - -def select_option(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Click the element to open the menu, then click the matching child.""" - option_name = args.get("option_name") - option_index = args.get("option_index") - if option_name is None and option_index is None: - step_id, _ = _new_step_id("select_option") - return error_dict(Code.BAD_REQUEST, - "option_name or option_index is required", - step_id=step_id) - - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - # Re-walk to see the now-visible option list. - new_tree = ctx.observer.get_element_tree(info.handle) - target: Optional[UIElement] = None - if new_tree is not None: - descendants = [] - stack = [new_tree] - while stack: - e = stack.pop() - descendants.append(e) - stack.extend(e.children) - if option_name is not None: - target = next((d for d in descendants if d.name == option_name), None) - elif option_index is not None: - idx = int(option_index) - if 0 <= idx < len(elem.children): - target = elem.children[idx] - if target is None: - return {"success": False, - "error": "option not found after opening selector"} - return ctx.observer.perform_action( - "click_at", - value={"x": target.bounds.center_x, "y": target.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - - return _do_element_action(ctx, action_name="select_option", - args=args, executor=_exec) - - -# ─── Legacy actions (now also returning ActionReceipts) ────────────────────── - -def click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("click_at") - before_windows = ctx.observer.list_windows() - started = time.time() - result = ctx.observer.perform_action("click_at", value={ - "x": args.get("x", 0), "y": args.get("y", 0), - "button": args.get("button", "left"), - "double": args.get("double", False), - }) - duration_ms = int((time.time() - started) * 1000) - after_windows = ctx.observer.list_windows() - out = annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - out.setdefault("action", "click_at") - out["duration_ms"] = duration_ms - out["new_dialogs"] = _new_dialogs(before_windows, after_windows) - return out - - -def type_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("type_text") - result = ctx.observer.perform_action("type", value=args.get("text", "")) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def press_key(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("press_key") - result = ctx.observer.perform_action("key", value=args.get("keys", "")) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def scroll(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("scroll") - result = ctx.observer.perform_action("scroll", value=args) - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_window_structure") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - serialized = tree.to_dict() - th = tree_hash(tree) - token = get_session().tree_tokens.put(info.window_uid, serialized, th) - - # P3 filtering / paging -------------------------------------------------- - roles = args.get("roles") - exclude_roles = args.get("exclude_roles") - visible_only = bool(args.get("visible_only")) - name_regex = args.get("name_regex") - max_text_len = args.get("max_text_len") - prune_empty = bool(args.get("prune_empty")) - max_nodes = args.get("max_nodes") - page_cursor = args.get("page_cursor") - - visible_regions: Optional[List[Dict[str, int]]] = None - if visible_only: - try: - visible_regions = ctx.observer.get_visible_areas(info.handle, windows) - except Exception: - visible_regions = [] - - filtered = _filter_tree( - serialized, - roles=set(roles) if roles else None, - exclude_roles=set(exclude_roles) if exclude_roles else None, - visible_regions=visible_regions, - name_regex=name_regex, - max_text_len=max_text_len, - prune_empty=prune_empty, - ) - - truncated = False - next_cursor: Optional[str] = None - node_count = _count_nodes(filtered) if filtered else 0 - if max_nodes is not None or page_cursor is not None: - filtered, truncated, next_cursor, node_count = _page_tree( - filtered, max_nodes=max_nodes, page_cursor=page_cursor, - ) - - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, - "window_uid": info.window_uid, - "element_count": len(tree.flat_list()), - "node_count": node_count, - "tree": filtered, - "tree_hash": th, - "tree_token": token, - "truncated": truncated, - "next_cursor": next_cursor, - } - - -def get_screenshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_screenshot") - windows, res = _resolve_window(ctx, args) - info = res.info - hwnd = info.handle if info else None - shot = ctx.observer.get_screenshot(hwnd) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title if info else "(full screen)", - "format": "png", "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - - -def bring_to_foreground(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("bring_to_foreground") - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - result = ctx.observer.bring_to_foreground(info.handle, windows) - result["window"] = info.title - result["window_uid"] = info.window_uid - return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) - - -def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_visible_areas") - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - areas = ctx.observer.get_visible_areas(info.handle, windows) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, - "window_uid": info.window_uid, - "visible_regions": areas, - } - - -# ─── P2: observe-with-diff, snapshots, wait_for, composites ────────────────── - -def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, - ) -> Tuple[Optional[UIElement], Dict[str, Any]]: - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return None, {"error": "no tree"} - serialized = tree.to_dict() - th = tree_hash(tree) - token = get_session().tree_tokens.put(info.window_uid, serialized, th) - return tree, { - "format": "full", - "window_uid": info.window_uid, - "window": info.title, - "tree": serialized, - "tree_hash": th, - "tree_token": token, - "base_token": None, - } - - -def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Return the current tree, optionally as a diff against a tree_token.""" - from diff import diff_custom, diff_json_patch - step_id, caused_by = _new_step_id("observe_window") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - since = args.get("since") - fmt = args.get("format", "custom") - - if not since: - _, full = _serialize_full_observation(ctx, info) - full.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "format": "full"}) - return full - - entry = get_session().tree_tokens.get(since) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id, window_uid=info.window_uid) - serialized = tree.to_dict() - th = tree_hash(tree) - new_token = get_session().tree_tokens.put(info.window_uid, serialized, th) - - if entry is None or entry.window_uid != info.window_uid: - # Token expired/wrong-window: return full tree. - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, "window": info.title, - "tree": serialized, "tree_hash": th, - "tree_token": new_token, "base_token": None, - "format": "full", - } - - if fmt == "json-patch": - changes = diff_json_patch(entry.serialized, serialized) - out_format = "json-patch" - else: - changes = diff_custom(entry.serialized, serialized) - out_format = "custom" - - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window_uid": info.window_uid, "window": info.title, - "tree_token": new_token, "base_token": since, - "format": out_format, - "changes": changes, - "unchanged": len(changes) == 0, - "tree_hash": th, - } - - -def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot") - windows = ctx.observer.list_windows() - trees: Dict[str, Dict[str, Any]] = {} - hashes: Dict[str, str] = {} - for w in windows: - try: - t = ctx.observer.get_element_tree(w.handle) - if t is not None and w.window_uid: - trees[w.window_uid] = t.to_dict() - hashes[w.window_uid] = tree_hash(t) - except Exception: - continue - snap = get_session().snapshots.put( - windows=[w.to_dict() for w in windows], - trees=trees, tree_hashes=hashes, - ) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "snapshot_id": snap.snapshot_id, - "ts": snap.ts, - "summary": {"windows": len(snap.windows), "trees": len(trees)}, - } - - -def snapshot_get(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot_get") - sid = args.get("snapshot_id") - if not sid: - return error_dict(Code.BAD_REQUEST, "snapshot_id is required", - step_id=step_id) - snap = get_session().snapshots.get(sid) - if snap is None: - return error_dict(Code.SNAPSHOT_EXPIRED, - "snapshot expired or not found", - step_id=step_id, snapshot_id=sid) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "snapshot_id": snap.snapshot_id, "ts": snap.ts, - "windows": snap.windows, - "trees": snap.trees, - "tree_hashes": snap.tree_hashes, - } - - -def snapshot_diff(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - from diff import diff_custom, diff_json_patch - step_id, caused_by = _new_step_id("snapshot_diff") - a_id = args.get("a") - b_id = args.get("b") - if not a_id or not b_id: - return error_dict(Code.BAD_REQUEST, "a and b are required", - step_id=step_id) - sess = get_session() - a = sess.snapshots.get(a_id) - b = sess.snapshots.get(b_id) - if a is None or b is None: - return error_dict(Code.SNAPSHOT_EXPIRED, - "one or both snapshots are missing", - step_id=step_id) - fmt = args.get("format", "custom") - - a_uids = {w["window_uid"] for w in a.windows} - b_uids = {w["window_uid"] for w in b.windows} - windows_added = sorted(b_uids - a_uids) - windows_removed = sorted(a_uids - b_uids) - common = sorted(a_uids & b_uids) - - per_window: Dict[str, Any] = {} - for uid in common: - if uid in a.trees and uid in b.trees: - if fmt == "json-patch": - per_window[uid] = {"format": "json-patch", - "changes": diff_json_patch(a.trees[uid], b.trees[uid])} - else: - per_window[uid] = {"format": "custom", - "changes": diff_custom(a.trees[uid], b.trees[uid])} - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "windows_added": windows_added, - "windows_removed": windows_removed, - "per_window_changes": per_window, - } - - -def snapshot_drop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("snapshot_drop") - sid = args.get("snapshot_id") - dropped = get_session().snapshots.drop(sid) if sid else False - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "dropped": dropped, - } - - -# ─── wait_for / wait_idle ───────────────────────────────────────────────────── - -def _check_condition(ctx: ToolContext, cond: Dict[str, Any], - window_uid_hint: Optional[str]) -> Tuple[bool, Dict[str, Any]]: - kind = cond.get("type") - sess = get_session() - windows = ctx.observer.list_windows() - info = ctx.observer.window_by_uid(windows, window_uid_hint) or _focused_window(windows) - - if kind == "window_appears": - rx = cond.get("title_regex", "") - for w in windows: - if re.search(rx, w.title): - return True, {"window_uid": w.window_uid, "title": w.title} - return False, {} - if kind == "window_disappears": - target = cond.get("window_uid") - for w in windows: - if w.window_uid == target: - return False, {} - return True, {"window_uid": target} - if kind == "focused_changes": - focus = next((w for w in windows if w.is_focused), None) - return (focus is not None), ({"focused_uid": focus.window_uid} if focus else {}) - if kind == "tree_changes": - token = cond.get("since") - entry = sess.tree_tokens.get(token) if token else None - if entry is None or info is None: - return False, {} - tree = ctx.observer.get_element_tree(info.handle) - return tree is not None and tree_hash(tree) != entry.tree_hash, {} - - if info is None: - return False, {} - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return False, {} - - if kind == "element_appears": - sel_text = cond.get("selector") - if not sel_text: - return False, {} - try: - res = sel.resolve(tree, sel.parse(sel_text)) - except sel.SelectorParseError: - return False, {} - if res.matches: - m = res.matches[0] - return True, {"element_id": m.element_id, "bounds": m.bounds.to_dict()} - return False, {} - if kind == "element_disappears": - sel_text = cond.get("selector") - eid = cond.get("element_id") - if sel_text: - try: - res = sel.resolve(tree, sel.parse(sel_text)) - return not res.matches, {} - except sel.SelectorParseError: - return False, {} - if eid: - return _find_by_id(tree, eid) is None, {} - return False, {} - if kind == "text_visible": - rx = cond.get("regex", "") - # Walk tree names/values. - for elem in tree.flat_list(): - joined = (elem.name or "") + " " + (elem.value or "") - if re.search(rx, joined): - return True, {"element_id": elem.element_id} - return False, {} - return False, {} - - -def wait_for(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("wait_for") - timeout_ms = int(args.get("timeout_ms", 5000)) - cap = int((ctx.config.get("wait_for", {}) or {}).get("max_timeout_ms", 60000)) - timeout_ms = min(timeout_ms, cap) - poll_ms = max(50, int(args.get("poll_ms", 200))) - conditions = args.get("any_of", []) - if not conditions: - return error_dict(Code.BAD_REQUEST, "any_of is required", - step_id=step_id) - window_uid = args.get("window_uid") - - started = time.time() - polls = 0 - while True: - polls += 1 - for i, cond in enumerate(conditions): - try: - ok, detail = _check_condition(ctx, cond, window_uid) - except Exception: - ok, detail = False, {} - if ok: - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "matched_index": i, "matched_detail": detail, - "elapsed_ms": int((time.time() - started) * 1000), - "polls": polls, - } - elapsed = (time.time() - started) * 1000 - if elapsed >= timeout_ms: - err = error_dict( - Code.TIMEOUT, f"wait_for timed out after {int(elapsed)}ms", - step_id=step_id, - ) - err.update({ - "elapsed_ms": int(elapsed), "polls": polls, - "matched_index": None, - }) - return err - time.sleep(poll_ms / 1000.0) - - -def wait_idle(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("wait_idle") - timeout_ms = int(args.get("timeout_ms", 5000)) - quiet_ms = int(args.get("quiet_ms", 750)) - poll_ms = max(50, int(args.get("poll_ms", 100))) - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - - started = time.time() - last_hash = None - last_change_at = time.time() - while (time.time() - started) * 1000 < timeout_ms: - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - time.sleep(poll_ms / 1000.0) - continue - h = tree_hash(tree) - if h != last_hash: - last_hash = h - last_change_at = time.time() - elif (time.time() - last_change_at) * 1000 >= quiet_ms: - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "elapsed_ms": int((time.time() - started) * 1000), - "tree_hash": h, - } - time.sleep(poll_ms / 1000.0) - - err = error_dict(Code.TIMEOUT, "wait_idle timed out", step_id=step_id) - err["elapsed_ms"] = int((time.time() - started) * 1000) - return err - - -# ─── Composite action+observe ───────────────────────────────────────────────── - -def _compose_observe(ctx: ToolContext, base_result: Dict[str, Any], - wait_after_ms: int, since_token: Optional[str]) -> None: - if wait_after_ms > 0: - time.sleep(wait_after_ms / 1000.0) - args: Dict[str, Any] = {} - target = base_result.get("target") or {} - if target.get("window_uid"): - args["window_uid"] = target["window_uid"] - if since_token: - args["since"] = since_token - obs = observe_window(ctx, args) - base_result["observation"] = obs - - -def click_element_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = click_element(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -def type_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = type_text(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -def press_key_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - wait_after_ms = int(args.pop("wait_after_ms", 200)) - since = args.pop("since", None) - receipt = press_key(ctx, args) - if receipt.get("ok"): - _compose_observe(ctx, receipt, wait_after_ms, since) - return receipt - - -# ─── P3: tree filtering / paging helpers ───────────────────────────────────── - -def _filter_tree(node: Dict[str, Any], *, roles: Optional[set], - exclude_roles: Optional[set], - visible_regions: Optional[List[Dict[str, int]]], - name_regex: Optional[str], - max_text_len: Optional[int], - prune_empty: bool) -> Optional[Dict[str, Any]]: - if node is None: - return None - role = node.get("role") - name = node.get("name") or "" - bounds = node.get("bounds") or {} - - # Role filter - role_keep = True - if roles is not None and role not in roles: - role_keep = False - if exclude_roles is not None and role in exclude_roles: - role_keep = False - - # Name regex - name_keep = True - if name_regex: - try: - name_keep = bool(re.search(name_regex, name)) - except re.error: - name_keep = True - - # Visibility - visible_keep = True - if visible_regions is not None: - visible_keep = _intersects_any(bounds, visible_regions) - - self_keep = role_keep and name_keep and visible_keep - - # Recurse children regardless (so we can keep ancestors if descendants match) - new_children: List[Dict[str, Any]] = [] - for c in node.get("children", []) or []: - fc = _filter_tree( - c, roles=roles, exclude_roles=exclude_roles, - visible_regions=visible_regions, name_regex=name_regex, - max_text_len=max_text_len, prune_empty=prune_empty, - ) - if fc is not None: - new_children.append(fc) - - if prune_empty and not self_keep and not new_children: - return None - - # Truncate text fields if requested. - truncated_node = dict(node) - if max_text_len is not None: - n = int(max_text_len) - if isinstance(truncated_node.get("name"), str) and len(truncated_node["name"]) > n: - truncated_node["name"] = truncated_node["name"][:n] + "…" - v = truncated_node.get("value") - if isinstance(v, str) and len(v) > n: - truncated_node["value"] = v[:n] + "…" - truncated_node["children"] = new_children - return truncated_node - - -def _intersects_any(b: Dict[str, int], regions: List[Dict[str, int]]) -> bool: - if not b: - return False - bx, by = b.get("x", 0), b.get("y", 0) - bw, bh = b.get("width", 0), b.get("height", 0) - if bw <= 0 or bh <= 0: - return False - bx2, by2 = bx + bw, by + bh - for r in regions: - rx, ry = r.get("x", 0), r.get("y", 0) - rx2, ry2 = rx + r.get("width", 0), ry + r.get("height", 0) - if bx < rx2 and bx2 > rx and by < ry2 and by2 > ry: - return True - return False - - -def _count_nodes(node: Optional[Dict[str, Any]]) -> int: - if node is None: - return 0 - return 1 + sum(_count_nodes(c) for c in (node.get("children") or [])) - - -def _page_tree(node: Optional[Dict[str, Any]], *, - max_nodes: Optional[int], - page_cursor: Optional[str] - ) -> Tuple[Optional[Dict[str, Any]], bool, Optional[str], int]: - """ - Paginated DFS walk. Returns (subtree-shaped result containing only the - page slice, truncated flag, next_cursor, node_count_in_page). - - Cursors are post-order element_ids; resuming starts from the next sibling - in the original walk. This is a best-effort pager — if the tree changed, - callers will get SnapshotExpired-shaped semantics by virtue of an unknown - cursor returning truncated:false and node_count:0. - """ - if node is None: - return None, False, None, 0 - flat: List[Dict[str, Any]] = [] - _flatten(node, flat) - - if page_cursor is not None: - for i, n in enumerate(flat): - if n.get("id") == page_cursor: - flat = flat[i + 1:] - break - else: - return None, False, None, 0 - - if max_nodes is None or max_nodes >= len(flat): - # Return full (possibly trimmed) tree starting from cursor. - if page_cursor is None: - return node, False, None, len(flat) - return _flat_to_tree(flat), False, None, len(flat) - - page = flat[:max_nodes] - truncated = True - next_cursor = page[-1].get("id") if page else None - return _flat_to_tree(page), truncated, next_cursor, len(page) - - -def _flatten(node: Dict[str, Any], out: List[Dict[str, Any]]) -> None: - out.append(node) - for c in node.get("children") or []: - _flatten(c, out) - - -def _flat_to_tree(flat: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: - """Wrap a list of nodes as children of a synthetic Window root.""" - if not flat: - return None - return { - "id": "page-root", - "name": "[paged]", - "role": "Group", - "value": None, - "bounds": {"x": 0, "y": 0, "width": 0, "height": 0}, - "enabled": True, "focused": False, - "keyboard_shortcut": None, "description": None, - "children": [dict(n, children=[]) for n in flat], - } - - -# ─── P3: cropped screenshots, region OCR, budgeted description ──────────────── - -def get_screenshot_cropped(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """get_screenshot with optional bbox / element_id / max_width / padding.""" - step_id, caused_by = _new_step_id("get_screenshot_cropped") - windows, res = _resolve_window(ctx, args) - info = res.info - hwnd = info.handle if info else None - shot = ctx.observer.get_screenshot(hwnd) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - - bbox: Optional[Dict[str, int]] = args.get("bbox") - element_id: Optional[str] = args.get("element_id") - padding = int(args.get("padding_px", 0)) - max_width: Optional[int] = args.get("max_width") - - if bbox is None and element_id and info is not None: - tree = ctx.observer.get_element_tree(info.handle) - if tree is not None: - elem = _find_by_id(tree, element_id) - if elem is not None: - # Convert to window-relative coordinates. - bbox = { - "x": max(0, elem.bounds.x - info.bounds.x), - "y": max(0, elem.bounds.y - info.bounds.y), - "width": elem.bounds.width, - "height": elem.bounds.height, - } - - if bbox or max_width: - shot, source_bbox = _apply_crop(shot, bbox, padding, max_width) - else: - source_bbox = None - - out: Dict[str, Any] = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title if info else "(full screen)", - "format": "png", "encoding": "base64", - "data": base64.b64encode(shot).decode(), - } - if source_bbox: - out["source_bbox"] = source_bbox - return out - - -def _apply_crop(png_bytes: bytes, bbox: Optional[Dict[str, int]], - padding: int, max_width: Optional[int] - ) -> Tuple[bytes, Optional[Dict[str, int]]]: - try: - import io as _io - from PIL import Image - except Exception: - return png_bytes, None - img = Image.open(_io.BytesIO(png_bytes)) - source_bbox: Optional[Dict[str, int]] = None - if bbox is not None: - x = max(0, int(bbox.get("x", 0)) - padding) - y = max(0, int(bbox.get("y", 0)) - padding) - x2 = min(img.width, int(bbox.get("x", 0)) + int(bbox.get("width", 0)) + padding) - y2 = min(img.height, int(bbox.get("y", 0)) + int(bbox.get("height", 0)) + padding) - if x2 > x and y2 > y: - img = img.crop((x, y, x2, y2)) - source_bbox = {"x": x, "y": y, "width": x2 - x, "height": y2 - y} - if max_width and img.width > int(max_width): - ratio = int(max_width) / float(img.width) - new_size = (int(max_width), max(1, int(img.height * ratio))) - img = img.resize(new_size) - buf = __import__("io").BytesIO() - img.save(buf, "PNG") - return buf.getvalue(), source_bbox - - -def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Region-scoped OCR; returns [{text, confidence, bbox}].""" - step_id, caused_by = _new_step_id("get_ocr") - try: - import io as _io - from PIL import Image - import pytesseract - from ocr_util import configure as _ocr_configure - _ocr_configure(ctx.config) - except Exception: - from ocr_util import INSTALL_HINT - return error_dict(Code.PLATFORM_UNSUPPORTED, - f"pytesseract / Pillow not installed. {INSTALL_HINT}", - step_id=step_id, hint=INSTALL_HINT) - windows, res = _resolve_window(ctx, args) - info = res.info - if info is None: - return error_dict(Code.BAD_REQUEST, - "window_uid or window_index is required", - step_id=step_id) - shot = ctx.observer.get_screenshot(info.handle) - if shot is None: - return error_dict(Code.INTERNAL, "screenshot capture failed", - step_id=step_id) - bbox = args.get("bbox") - element_id = args.get("element_id") - if element_id and not bbox: - tree = ctx.observer.get_element_tree(info.handle) - if tree is not None: - elem = _find_by_id(tree, element_id) - if elem is not None: - bbox = { - "x": max(0, elem.bounds.x - info.bounds.x), - "y": max(0, elem.bounds.y - info.bounds.y), - "width": elem.bounds.width, - "height": elem.bounds.height, - } - - img = Image.open(_io.BytesIO(shot)) - if bbox: - x = max(0, int(bbox.get("x", 0))) - y = max(0, int(bbox.get("y", 0))) - x2 = min(img.width, x + int(bbox.get("width", 0))) - y2 = min(img.height, y + int(bbox.get("height", 0))) - if x2 > x and y2 > y: - img = img.crop((x, y, x2, y2)) - - min_conf = (ctx.config.get("ocr", {}) or {}).get("min_confidence", 30) - try: - data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT) - except pytesseract.TesseractNotFoundError: - from ocr_util import diagnose as _ocr_diag, INSTALL_HINT - return error_dict( - Code.PLATFORM_UNSUPPORTED, - ("tesseract binary not found — check ocr.tesseract_cmd in " - f"config.json. {INSTALL_HINT}"), - step_id=step_id, **_ocr_diag(ctx.config), - ) - except Exception as e: - return error_dict(Code.INTERNAL, f"OCR failed: {e}", - step_id=step_id) - out_words: List[Dict[str, Any]] = [] - for i, text in enumerate(data["text"]): - text = (text or "").strip() - if not text: - continue - try: - conf = int(data["conf"][i]) - except (TypeError, ValueError): - conf = 0 - if conf < min_conf: - continue - out_words.append({ - "text": text, "confidence": conf, - "bbox": {"x": int(data["left"][i]), "y": int(data["top"][i]), - "width": int(data["width"][i]), - "height": int(data["height"][i])}, - }) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, "window_uid": info.window_uid, - "words": out_words, - } - - -def get_screen_description(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Combined description: accessibility tree + OCR + VLM, returning every available source.""" - step_id, caused_by = _new_step_id("get_screen_description") - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id) - max_tokens = args.get("max_tokens") - focus_id = args.get("focus_element") - - sub: UIElement = tree - if focus_id: - found = _find_by_id(tree, focus_id) - if found is not None: - sub = found - - parts: Dict[str, str] = {} - - # Accessibility tree — always attempted. - try: - parts["accessibility"] = ctx.describer.from_tree(sub, info) - except Exception as e: - logger.exception("[get_screen_description] accessibility failed: %s", e) - - # OCR — attempted when enabled in config. - ocr_enabled = (ctx.config.get("ocr", {}) or {}).get("enabled", True) - if ocr_enabled: - try: - shot = ctx.observer.get_screenshot(info.handle) - if shot: - parts["ocr"] = ctx.describer.from_ocr(shot) - else: - logger.warning("[get_screen_description] screenshot unavailable for OCR") - except Exception as e: - logger.exception("[get_screen_description] OCR failed: %s", e) - - # VLM — attempted when enabled in config. In multipass mode the VLM - # output is a structured envelope; the JSON-serialised form is folded - # into the concatenated body (for back-compat with the legacy text - # description) and the parsed dict is returned separately under - # ``vlm_structured`` so callers don't have to re-parse it. - vlm_structured: Any = None - vlm_enabled = (ctx.config.get("vlm", {}) or {}).get("enabled", False) - if vlm_enabled: - try: - shot = ctx.observer.get_screenshot(info.handle) - if shot: - vlm_mode = ( - (ctx.config.get("vlm", {}) or {}).get("mode") or "single" - ).lower() - if vlm_mode == "multipass": - env = ctx.describer.from_vlm_multipass( - shot, root=sub, window=info, - ) - if env is not None: - import json as _json - parts["vlm"] = _json.dumps(env, indent=2, - ensure_ascii=False) - vlm_structured = env - else: - vlm_out = ctx.describer.from_vlm( - shot, root=sub, window=info, - ) - if vlm_out is not None: - parts["vlm"] = vlm_out - else: - logger.warning("[get_screen_description] screenshot unavailable for VLM") - except Exception as e: - logger.exception("[get_screen_description] VLM failed: %s", e) - - body = "" - if parts: - body = "\n\n".join(f"[{k}]\n{v}" for k, v in parts.items()) - else: - body = "[no description available]" - - truncated = False - if max_tokens is not None: - char_cap = int(max_tokens) * 4 # rough chars-per-token - if len(body) > char_cap: - body = body[:char_cap] + "… [truncated]" - truncated = True - - result: Dict[str, Any] = { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "window": info.title, "window_uid": info.window_uid, - "effective_mode": "combined", - "description": body, - "truncated": truncated, - } - if vlm_structured is not None: - result["vlm_structured"] = vlm_structured - return result - - -# ─── Dispatcher ─────────────────────────────────────────────────────────────── - -REGISTRY: Dict[str, Callable[[ToolContext, Dict[str, Any]], Dict[str, Any]]] = { - # Read-only - "list_windows": list_windows, - "get_capabilities": get_capabilities, - "get_monitors": get_monitors, - "find_element": find_element, - "get_window_structure": get_window_structure, - "get_screenshot": get_screenshot, - "get_visible_areas": get_visible_areas, - - # Element-targeted actions - "click_element": click_element, - "focus_element": focus_element, - "set_value": set_value, - "invoke_element": invoke_element, - "select_option": select_option, - - # Legacy actions - "click_at": click_at, - "type_text": type_text, - "press_key": press_key, - "scroll": scroll, - "bring_to_foreground": bring_to_foreground, - - # P2: sync, diff, snapshots, composites - "observe_window": observe_window, - "snapshot": snapshot, - "snapshot_get": snapshot_get, - "snapshot_diff": snapshot_diff, - "snapshot_drop": snapshot_drop, - "wait_for": wait_for, - "wait_idle": wait_idle, - "click_element_and_observe": click_element_and_observe, - "type_and_observe": type_and_observe, - "press_key_and_observe": press_key_and_observe, - - # P3 - "get_screenshot_cropped": get_screenshot_cropped, - "get_ocr": get_ocr, - "get_screen_description": get_screen_description, -} - -# Forward-declared P4 entries appended after definitions (below). - - -# ─── P4: tracing, replay, scenarios, oracles ───────────────────────────────── - -def trace_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import tracing as _tracing - step_id, caused_by = _new_step_id("trace_start") - sess = get_session() - if sess.active_trace is not None and not sess.active_trace.closed: - return error_dict(Code.BAD_REQUEST, "trace already active", - step_id=step_id, - trace_id=sess.active_trace.trace_id) - handle = _tracing.start(label=args.get("label", ""), config=ctx.config) - sess.active_trace = handle - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "trace_id": handle.trace_id, - "started_at": handle.started_at, - "dir": handle.dir, - } - - -def trace_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import tracing as _tracing - step_id, caused_by = _new_step_id("trace_stop") - sess = get_session() - if sess.active_trace is None: - return error_dict(Code.BAD_REQUEST, "no active trace", - step_id=step_id) - info = _tracing.stop(sess.active_trace) - sess.active_trace = None - info.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return info - - -def trace_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("trace_status") - sess = get_session() - if sess.active_trace is None: - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "active_trace_id": None, "step_count": 0, "dir": None} - h = sess.active_trace - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "active_trace_id": h.trace_id, - "step_count": h.counter.value, - "dir": h.dir, - } - - -# ─── Replay state ──────────────────────────────────────────────────────────── - -_REPLAYS: Dict[str, Any] = {} - - -def replay_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import replay as _replay - step_id, caused_by = _new_step_id("replay_start") - path = args.get("path") - if not path: - return error_dict(Code.BAD_REQUEST, "path is required", - step_id=step_id) - mode = args.get("mode", "execute") - on_div = args.get("on_divergence", "warn") - try: - rep = _replay.load(path, mode=mode, on_divergence=on_div) - except Exception as e: - return error_dict(Code.BAD_REQUEST, f"could not load trace: {e}", - step_id=step_id, path=path) - handle_id = "rep:" + str(len(_REPLAYS) + 1) - _REPLAYS[handle_id] = rep - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": handle_id, - "total": len(rep.rows), - "mode": rep.mode, - "label": rep.label, - } - - -def replay_step(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import replay as _replay - step_id, caused_by = _new_step_id("replay_step") - rid = args.get("replay_id") - rep = _REPLAYS.get(rid) - if rep is None: - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - - def _disp(name: str, a: Dict[str, Any]) -> Dict[str, Any]: - return dispatch(ctx, name, a) - - out = _replay.step(rep, dispatch=_disp) - out.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": rid}) - return out - - -def replay_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("replay_status") - rid = args.get("replay_id") - rep = _REPLAYS.get(rid) - if rep is None: - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "replay_id": rid, - "position": rep.position, - "total": len(rep.rows), - "finished": rep.finished, - "divergences": rep.divergences, - "mode": rep.mode, - } - - -def replay_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("replay_stop") - rid = args.get("replay_id") - if rid in _REPLAYS: - _REPLAYS.pop(rid) - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "stopped": True} - return error_dict(Code.BAD_REQUEST, "unknown replay_id", - step_id=step_id, replay_id=rid) - - -# ─── Scenarios ─────────────────────────────────────────────────────────────── - -def load_scenario(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import scenarios as _scn - step_id, caused_by = _new_step_id("load_scenario") - path = args.get("path") - if not path: - return error_dict(Code.BAD_REQUEST, "path is required", - step_id=step_id) - try: - sc = _scn.load(path) - _scn.attach_to_observer(sc, ctx.observer) - except _scn.ScenarioError as e: - return error_dict(Code.SCENARIO_INVALID, str(e), - step_id=step_id, path=path) - except Exception as e: - return error_dict(Code.SCENARIO_INVALID, f"{type(e).__name__}: {e}", - step_id=step_id, path=path) - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "scenario": sc.name, - "state": sc.current_state, - "states": list(sc.states.keys()), - } - - -# ─── Oracles ───────────────────────────────────────────────────────────────── - -def assert_state(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - import oracles as _oracles - step_id, caused_by = _new_step_id("assert_state") - pred = args.get("predicate") or args.get("predicates") or [] - out = _oracles.evaluate(ctx.observer, pred, config=ctx.config) - if out.get("ok"): - out["step_id"] = step_id - out["caused_by_step_id"] = caused_by - return out - - -REGISTRY.update({ - "trace_start": trace_start, - "trace_stop": trace_stop, - "trace_status": trace_status, - "replay_start": replay_start, - "replay_step": replay_step, - "replay_status": replay_status, - "replay_stop": replay_stop, - "load_scenario": load_scenario, - "assert_state": assert_state, -}) - - -# ─── P5: budgets, propose_action, status reporters ─────────────────────────── - -def get_budget_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_budget_status") - sess = get_session() - if sess.budgets is None: - return {"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "configured": False} - out = sess.budgets.status() - out.update({"ok": True, "success": True, "configured": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return out - - -def get_redaction_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, caused_by = _new_step_id("get_redaction_status") - sess = get_session() - out = sess.redactor.status() if sess.redactor else {"enabled": False, "active": False} - out.update({"ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by}) - return out - - -def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - """Issue a single-use confirm_token for a destructive action.""" - step_id, caused_by = _new_step_id("propose_action") - action = args.get("action") - inner_args = args.get("args") or {} - if not action: - return error_dict(Code.BAD_REQUEST, "action is required", - step_id=step_id) - - windows, res = _resolve_window(ctx, inner_args) - info = res.info or _focused_window(windows) - if info is None: - return error_dict(Code.WINDOW_GONE, "no windows available", - step_id=step_id) - tree = ctx.observer.get_element_tree(info.handle) - if tree is None: - return error_dict(Code.INTERNAL, "could not retrieve element tree", - step_id=step_id) - elem, selector_str, err = _resolve_element(tree, inner_args) - if err: - return {**err, "step_id": step_id} - - sess = get_session() - bbox = elem.bounds.to_dict() - ct = sess.confirms.issue( - action=action, window_uid=info.window_uid, - selector=selector_str, bbox=bbox, args=inner_args, - ) - # Optional preview crop. - try: - shot = ctx.observer.get_screenshot(info.handle) - crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, max_width=400) - preview_b64 = base64.b64encode(crop_bytes).decode() if crop_bytes else None - except Exception: - preview_b64 = None - - return { - "ok": True, "success": True, - "step_id": step_id, "caused_by_step_id": caused_by, - "confirm_token": ct.token, - "expires_at": ct.expires_at, - "would_target": { - "window_uid": info.window_uid, - "selector": selector_str, - "bounds": bbox, - "screenshot_b64": preview_b64, - }, - } - - -REGISTRY.update({ - "get_budget_status": get_budget_status, - "get_redaction_status": get_redaction_status, - "propose_action": propose_action, -}) - - -# ─── P6: extra input verbs ──────────────────────────────────────────────────── - -def hover_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("hover_at") - x = int(args.get("x", 0)) - y = int(args.get("y", 0)) - hover_ms = int(args.get("hover_ms", 250)) - try: - import pyautogui - pyautogui.moveTo(x, y) - time.sleep(hover_ms / 1000.0) - ok = True - except Exception: - ok = False - return {"ok": ok, "success": ok, "action": "hover_at", - "step_id": step_id, "caused_by_step_id": step_id, - "x": x, "y": y, "hover_ms": hover_ms} - - -def hover_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - hover_ms = int(args.get("hover_ms", 250)) - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - try: - import pyautogui - pyautogui.moveTo(elem.bounds.center_x, elem.bounds.center_y) - time.sleep(hover_ms / 1000.0) - return {"success": True, "action": "hover", "hover_ms": hover_ms} - except Exception as e: - return {"success": False, "error": str(e)} - return _do_element_action(ctx, action_name="hover_element", - args=args, executor=_exec) - - -def right_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_at(ctx, dict(args, button="right")) - - -def right_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_element(ctx, dict(args, button="right")) - - -def double_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_at(ctx, dict(args, double=True)) - - -def double_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - return click_element(ctx, dict(args, count=2)) - - -def drag(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - step_id, _ = _new_step_id("drag") - src = args.get("from") or {} - dst = args.get("to") or {} - modifiers = list(args.get("modifiers") or []) - # Resolve element references on either end. - windows, res = _resolve_window(ctx, args) - info = res.info or _focused_window(windows) - tree = ctx.observer.get_element_tree(info.handle) if info else None - - def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: - if "x" in spec and "y" in spec: - return int(spec["x"]), int(spec["y"]) - eid = spec.get("element_id") - sel_text = spec.get("selector") - if tree is None: - return None - if sel_text: - try: - resu = sel.resolve(tree, sel.parse(sel_text)) - if resu.matches: - e = resu.matches[0] - return e.bounds.center_x, e.bounds.center_y - except sel.SelectorParseError: - return None - if eid: - e = _find_by_id(tree, eid) - if e is not None: - return e.bounds.center_x, e.bounds.center_y - return None - - p1 = _to_xy(src) - p2 = _to_xy(dst) - if p1 is None or p2 is None: - return error_dict(Code.BAD_REQUEST, - "drag requires from/to as {x,y} or {selector|element_id}", - step_id=step_id) - duration = float(args.get("duration_s", 0.5)) - path = [p1, ((p1[0] + p2[0]) // 2, (p1[1] + p2[1]) // 2), p2] - try: - import pyautogui - for k in modifiers: - pyautogui.keyDown(k) - try: - pyautogui.moveTo(*p1) - pyautogui.dragTo(*p2, duration=duration, button="left") - finally: - for k in modifiers: - pyautogui.keyUp(k) - ok = True - err = None - except Exception as e: - ok = False - err = str(e) - out = {"ok": ok, "success": ok, "action": "drag", - "step_id": step_id, "caused_by_step_id": step_id, - "from": list(p1), "to": list(p2), - "modifiers": modifiers, "path": [list(p) for p in path]} - if err: - out["error"] = {"code": Code.INTERNAL, "message": err, - "recoverable": False, "suggested_next_tool": None, - "context": {}} - return out - - -def key_into_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - keys = args.get("keys", "") - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - return ctx.observer.perform_action("key", value=keys, hwnd=info.handle) - return _do_element_action(ctx, action_name="key_into_element", - args=args, executor=_exec) - - -def clear_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: - def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] - ) -> Dict[str, Any]: - ctx.observer.perform_action( - "click_at", - element_id=elem.element_id, - value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, - "button": "left", "double": False}, - hwnd=info.handle, - ) - ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) - return ctx.observer.perform_action("key", value="delete", - hwnd=info.handle) - return _do_element_action(ctx, action_name="clear_text", - args=args, executor=_exec) - - -REGISTRY.update({ - "hover_at": hover_at, - "hover_element": hover_element, - "right_click_at": right_click_at, - "right_click_element": right_click_element, - "double_click_at": double_click_at, - "double_click_element": double_click_element, - "drag": drag, - "key_into_element": key_into_element, - "clear_text": clear_text, -}) - - -_ALLOWLIST_TOOLS = { - "get_capabilities", "get_monitors", "get_budget_status", - "get_redaction_status", "trace_status", "replay_status", - "list_windows", -} - - -def _check_allowlist(ctx: ToolContext, name: str - ) -> Optional[Dict[str, Any]]: - actions = ctx.config.get("actions") or {} - allow = set(actions.get("allow") or []) - deny = set(actions.get("deny") or []) - default = actions.get("default", "allow") - if name in _ALLOWLIST_TOOLS: - return None - if name in deny: - return error_dict(Code.PERMISSION_DENIED, - f"tool {name!r} is in actions.deny") - if allow and name not in allow and default == "deny": - return error_dict(Code.PERMISSION_DENIED, - f"tool {name!r} is not in actions.allow") - if not allow and default == "deny": - return error_dict(Code.PERMISSION_DENIED, - f"actions.default is 'deny' and no allowlist matches {name!r}") - return None - - -def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any]: - fn = REGISTRY.get(name) - if fn is None: - return error_dict(Code.BAD_REQUEST, f"unknown tool: {name}") - - blocked = _check_allowlist(ctx, name) - if blocked is not None: - sid, _ = _new_step_id(name) - blocked["step_id"] = sid - return blocked - - started = time.time() - sess = get_session() - tree_before = "" - if sess.active_trace is not None and not sess.active_trace.closed: - try: - windows0 = ctx.observer.list_windows() - focused0 = next((w for w in windows0 if w.is_focused), None) - if focused0: - t = ctx.observer.get_element_tree(focused0.handle) - if t: - tree_before = tree_hash(t) - except Exception: - pass - - try: - result = fn(ctx, args or {}) - except Exception as e: - logger.exception(f"tool {name} crashed") - result = error_dict(Code.INTERNAL, f"{type(e).__name__}: {e}") - - duration_ms = int((time.time() - started) * 1000) - - # Recurse-safety: don't trace meta tools that would recurse forever. - if name not in {"trace_start", "trace_stop", "trace_status", - "replay_start", "replay_step", "replay_status", - "replay_stop"}: - if sess.active_trace is not None and not sess.active_trace.closed: - try: - import tracing as _tracing - shot_full = ctx.observer.get_full_display_screenshot() - shot_window = None - tgt_uid = None - tgt = result.get("target") or {} - tgt_uid = tgt.get("window_uid") or args.get("window_uid") - if tgt_uid: - win = ctx.observer.window_by_uid( - ctx.observer.list_windows(), tgt_uid) - if win: - shot_window = ctx.observer.get_screenshot(win.handle) - tree_after = "" - try: - after_w = ctx.observer.list_windows() - f = next((w for w in after_w if w.is_focused), None) - if f: - t = ctx.observer.get_element_tree(f.handle) - if t: - tree_after = tree_hash(t) - except Exception: - pass - _tracing.record( - sess.active_trace, - tool=name, caller=args.get("_caller", "unknown"), - args={k: v for k, v in (args or {}).items() - if not k.startswith("_")}, - result=result, duration_ms=duration_ms, - tree_hash_before=tree_before, tree_hash_after=tree_after, - full_screenshot=shot_full, - window_screenshot=shot_window, - ) - except Exception: - logger.exception("trace.record failed") - - # Apply redaction to text-bearing read-only results. - if sess.redactor is not None: - try: - result = _apply_redaction(name, result, sess.redactor) - except Exception: - logger.exception("redaction failed") - - # Budget accounting. - if sess.budgets is not None: - try: - sess.budgets.note(name, result) - except Exception: - pass - - # Audit log. - if sess.auditor is not None: - try: - sess.auditor.record( - tool=name, caller=args.get("_caller", "unknown"), - args=args or {}, result=result, - ) - except Exception: - logger.exception("audit failed") - - return result - - -def _apply_redaction(tool: str, result: Dict[str, Any], redactor: Any - ) -> Dict[str, Any]: - if not redactor.is_active() or not isinstance(result, dict): - return result - if tool == "get_window_structure" and "tree" in result: - title = result.get("window") or "" - result["tree"] = redactor.redact_tree(result["tree"], title) - elif tool == "observe_window": - if "tree" in result and isinstance(result["tree"], dict): - result["tree"] = redactor.redact_tree(result["tree"], - result.get("window") or "") - elif tool == "get_screen_description": - if isinstance(result.get("description"), str): - result["description"] = redactor.redact_ocr_text(result["description"]) - elif tool == "get_ocr": - if isinstance(result.get("words"), list): - result["words"] = redactor.redact_ocr_words(result["words"]) - return result diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..0f34505 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,160 @@ +""" +tools — Central tool implementations (package form of the former tools.py). + +Both mcp_server.py and web_inspector.py dispatch into this package; the +MCP and REST surfaces are thin wrappers. Every tool returns a dict in +one of two shapes: + + {ok: true, step_id: …, …tool-specific fields…} + {ok: false, success: false, step_id: …, error: {code, message, …}} + +For backwards compatibility (design doc D5) success-shaped legacy fields +are preserved alongside the new `ok` / `error` object on existing tools. + +P3 decomposition: the implementation now lives in submodules (context, +receipts, actions, observe, vision, snapshots, trace_replay, meta, +dispatch). This __init__ re-exports the entire pre-split public surface +so `import tools` / `from tools import X` keep working unchanged. +""" + +from __future__ import annotations + +from tools.context import ( + ToolContext, + _find_by_id, + _focused_window, + _is_input_tool, + _new_dialogs, + _new_step_id, + _resolve_element, + _resolve_window, +) +from tools.receipts import ( + _build_receipt, + _check_confirmation, + _confirmation_rules_match, +) +from tools.observe import ( + _count_nodes, + _effective_depth, + _filter_tree, + _flat_to_tree, + _flatten, + _intersects_any, + _observe_changed_only, + _page_tree, + _perf_dict, + _serialize_full_observation, + _truncate_depth, + find_element, + get_visible_areas, + get_window_structure, + observe_window, +) +from tools.vision import ( + _apply_crop, + get_ocr, + get_screen_description, + get_screenshot, + get_screenshot_cropped, +) +from tools.actions import ( + _compose_observe, + _do_element_action, + bring_to_foreground, + clear_text, + click_at, + click_element, + click_element_and_observe, + double_click_at, + double_click_element, + drag, + focus_element, + hover_at, + hover_element, + invoke_element, + key_into_element, + press_key, + press_key_and_observe, + right_click_at, + right_click_element, + scroll, + select_option, + set_value, + type_and_observe, + type_text, +) +from tools.snapshots import ( + _check_condition, + snapshot, + snapshot_diff, + snapshot_drop, + snapshot_get, + wait_for, + wait_idle, +) +from tools.trace_replay import ( + _REPLAYS, + assert_state, + load_scenario, + replay_start, + replay_status, + replay_step, + replay_stop, + trace_start, + trace_status, + trace_stop, +) +from tools.meta import ( + get_budget_status, + get_capabilities, + get_monitors, + get_redaction_status, + list_windows, + propose_action, +) +from tools.dispatch import ( + REGISTRY, + _ALLOWLIST_TOOLS, + _apply_redaction, + _check_allowlist, + dispatch, +) + +__all__ = [ + # context + "ToolContext", "_find_by_id", "_focused_window", "_is_input_tool", + "_new_dialogs", "_new_step_id", "_resolve_element", "_resolve_window", + # receipts + "_build_receipt", "_check_confirmation", "_confirmation_rules_match", + # observe + "_count_nodes", "_effective_depth", "_filter_tree", "_flat_to_tree", + "_flatten", "_intersects_any", "_observe_changed_only", "_page_tree", + "_perf_dict", "_serialize_full_observation", "_truncate_depth", + "find_element", "get_visible_areas", "get_window_structure", + "observe_window", + # vision + "_apply_crop", "get_ocr", "get_screen_description", "get_screenshot", + "get_screenshot_cropped", + # actions + "_compose_observe", "_do_element_action", "bring_to_foreground", + "clear_text", "click_at", "click_element", "click_element_and_observe", + "double_click_at", "double_click_element", "drag", "focus_element", + "hover_at", "hover_element", "invoke_element", "key_into_element", + "press_key", "press_key_and_observe", "right_click_at", + "right_click_element", "scroll", "select_option", "set_value", + "type_and_observe", "type_text", + # snapshots / waits + "_check_condition", "snapshot", "snapshot_diff", "snapshot_drop", + "snapshot_get", "wait_for", "wait_idle", + # trace / replay / scenarios / oracles + "_REPLAYS", "assert_state", "load_scenario", "replay_start", + "replay_status", "replay_step", "replay_stop", "trace_start", + "trace_status", "trace_stop", + # meta + "get_budget_status", "get_capabilities", "get_monitors", + "get_redaction_status", "list_windows", "propose_action", + # dispatch + "REGISTRY", "_ALLOWLIST_TOOLS", "_apply_redaction", "_check_allowlist", + "dispatch", +] diff --git a/tools/actions.py b/tools/actions.py new file mode 100644 index 0000000..9b59024 --- /dev/null +++ b/tools/actions.py @@ -0,0 +1,515 @@ +""" +Element-targeted and coordinate input verbs (plus composites). + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Dict, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict, annotate_legacy_result +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_dialogs, _new_step_id, + _resolve_element, _resolve_window, +) +from tools.observe import observe_window +from tools.receipts import ( + _build_receipt, _check_confirmation, _confirmation_rules_match, +) + +logger = logging.getLogger(__name__) + + +def _do_element_action(ctx: ToolContext, *, action_name: str, args: Dict[str, Any], + executor: Callable[[UIElement, WindowInfo, Dict[str, Any]], + Dict[str, Any]]) -> Dict[str, Any]: + step_id, _ = _new_step_id(action_name) + dry_run = bool(args.get("dry_run")) + + # Budget gate (no-op until P5 plumbs budgets in). + sess = get_session() + if sess.budgets is not None: + gate = sess.budgets.gate(action_name) + if gate is not None: + return {**gate, "step_id": step_id} + + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + + elem, selector_str, err = _resolve_element(tree, args) + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} + + if not elem.enabled: + return error_dict(Code.ELEMENT_DISABLED, + f"element is disabled: {selector_str}", + step_id=step_id, selector=selector_str) + + occluded = ctx.observer.is_element_occluded(elem.bounds, info.handle, windows) + if occluded: + return error_dict(Code.ELEMENT_OCCLUDED, + f"element is occluded: {selector_str}", + step_id=step_id, selector=selector_str) + + target = { + "window_uid": info.window_uid, + "element_id": elem.element_id, + "selector": selector_str, + "bounds": elem.bounds.to_dict(), + } + + # Confirmation gate (no-op until P5 plumbs confirms in). + confirm_check = _check_confirmation(ctx, action_name, args, target) + if confirm_check is not None: + return {**confirm_check, "step_id": step_id} + + started = time.time() + executor_result: Dict[str, Any] + if dry_run: + executor_result = {"success": True, "dry_run": True} + else: + try: + executor_result = executor(elem, info, args) + except Exception as e: + logger.exception(f"{action_name} executor failed") + executor_result = {"success": False, "error": str(e)} + + duration_ms = int((time.time() - started) * 1000) + after_windows = ctx.observer.list_windows() + info_after = ctx.observer.window_by_uid(after_windows, info.window_uid) + # Post-action re-read must bypass the tree cache so the receipt's + # after-state reflects reality (the fresh capture refreshes the cache). + after_tree = (ctx.observer.get_element_tree(info_after.handle, + window_uid=info_after.window_uid, + use_cache=False) + if info_after else None) + + ok = bool(executor_result.get("success", True)) + err_obj: Optional[Dict[str, Any]] = None + if not ok: + err_obj = { + "code": executor_result.get("error_code", Code.INTERNAL), + "message": str(executor_result.get("error", "action failed")), + "recoverable": False, + "suggested_next_tool": None, + "context": {}, + } + + extra: Dict[str, Any] = {} + if "warning" in (res.warning or ""): + extra["warning"] = res.warning + if res.warning: + extra["warning"] = res.warning + + return _build_receipt( + step_id=step_id, action=action_name, target=target, + before_tree=tree, before_windows=windows, + after_tree=after_tree, after_windows=after_windows, + duration_ms=duration_ms, dry_run=dry_run, ok=ok, + extra=extra, error=err_obj, + ) + + +def click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + button = args.get("button", "left") + count = int(args.get("count", 1)) + + def _exec(elem: UIElement, info: WindowInfo, _args: Dict[str, Any] + ) -> Dict[str, Any]: + cx, cy = elem.bounds.center_x, elem.bounds.center_y + result = ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": cx, "y": cy, "button": button, + "double": (count >= 2)}, + hwnd=info.handle, + ) + return result + + return _do_element_action(ctx, action_name="click_element", + args=args, executor=_exec) + + +def focus_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + # Focus via center click (universal fallback). Adapter-specific + # SetFocus paths live in the platform adapters; today the universal + # click is sufficient on all three platforms. + return ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + return _do_element_action(ctx, action_name="focus_element", + args=args, executor=_exec) + + +def set_value(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + if "value" not in args: + step_id, _ = _new_step_id("set_value") + return error_dict(Code.BAD_REQUEST, "value is required", + step_id=step_id) + value = args["value"] + clear_first = bool(args.get("clear_first", True)) + + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + if clear_first: + ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) + ctx.observer.perform_action("key", value="delete", hwnd=info.handle) + return ctx.observer.perform_action("type", value=str(value), + hwnd=info.handle) + return _do_element_action(ctx, action_name="set_value", + args=args, executor=_exec) + + +def invoke_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + # No platform-specific InvokePattern surface yet; behaves like click. + return _do_element_action( + ctx, action_name="invoke_element", args=args, + executor=lambda elem, info, _a: ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ), + ) + + +def select_option(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Click the element to open the menu, then click the matching child.""" + option_name = args.get("option_name") + option_index = args.get("option_index") + if option_name is None and option_index is None: + step_id, _ = _new_step_id("select_option") + return error_dict(Code.BAD_REQUEST, + "option_name or option_index is required", + step_id=step_id) + + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + # Re-walk to see the now-visible option list (bypass the cache — + # the click above just changed the UI). + new_tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + target: Optional[UIElement] = None + if new_tree is not None: + descendants = [] + stack = [new_tree] + while stack: + e = stack.pop() + descendants.append(e) + stack.extend(e.children) + if option_name is not None: + target = next((d for d in descendants if d.name == option_name), None) + elif option_index is not None: + idx = int(option_index) + if 0 <= idx < len(elem.children): + target = elem.children[idx] + if target is None: + return {"success": False, + "error": "option not found after opening selector"} + return ctx.observer.perform_action( + "click_at", + value={"x": target.bounds.center_x, "y": target.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + + return _do_element_action(ctx, action_name="select_option", + args=args, executor=_exec) + + +def click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("click_at") + before_windows = ctx.observer.list_windows() + started = time.time() + result = ctx.observer.perform_action("click_at", value={ + "x": args.get("x", 0), "y": args.get("y", 0), + "button": args.get("button", "left"), + "double": args.get("double", False), + }) + duration_ms = int((time.time() - started) * 1000) + after_windows = ctx.observer.list_windows() + out = annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + out.setdefault("action", "click_at") + out["duration_ms"] = duration_ms + out["new_dialogs"] = _new_dialogs(before_windows, after_windows) + return out + + +def type_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("type_text") + result = ctx.observer.perform_action("type", value=args.get("text", "")) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def press_key(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("press_key") + result = ctx.observer.perform_action("key", value=args.get("keys", "")) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def scroll(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("scroll") + result = ctx.observer.perform_action("scroll", value=args) + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def bring_to_foreground(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("bring_to_foreground") + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + result = ctx.observer.bring_to_foreground(info.handle, windows) + result["window"] = info.title + result["window_uid"] = info.window_uid + return annotate_legacy_result(result, step_id=step_id, caused_by_step_id=step_id) + + +def _compose_observe(ctx: ToolContext, base_result: Dict[str, Any], + wait_after_ms: int, since_token: Optional[str]) -> None: + if wait_after_ms > 0: + time.sleep(wait_after_ms / 1000.0) + args: Dict[str, Any] = {} + target = base_result.get("target") or {} + if target.get("window_uid"): + args["window_uid"] = target["window_uid"] + if since_token: + args["since"] = since_token + obs = observe_window(ctx, args) + base_result["observation"] = obs + + +def click_element_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = click_element(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def type_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = type_text(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def press_key_and_observe(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + wait_after_ms = int(args.pop("wait_after_ms", 200)) + since = args.pop("since", None) + receipt = press_key(ctx, args) + if receipt.get("ok"): + _compose_observe(ctx, receipt, wait_after_ms, since) + return receipt + + +def hover_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("hover_at") + x = int(args.get("x", 0)) + y = int(args.get("y", 0)) + hover_ms = int(args.get("hover_ms", 250)) + try: + import pyautogui + pyautogui.moveTo(x, y) + time.sleep(hover_ms / 1000.0) + ok = True + except Exception: + ok = False + return {"ok": ok, "success": ok, "action": "hover_at", + "step_id": step_id, "caused_by_step_id": step_id, + "x": x, "y": y, "hover_ms": hover_ms} + + +def hover_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + hover_ms = int(args.get("hover_ms", 250)) + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + try: + import pyautogui + pyautogui.moveTo(elem.bounds.center_x, elem.bounds.center_y) + time.sleep(hover_ms / 1000.0) + return {"success": True, "action": "hover", "hover_ms": hover_ms} + except Exception as e: + return {"success": False, "error": str(e)} + return _do_element_action(ctx, action_name="hover_element", + args=args, executor=_exec) + + +def right_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_at(ctx, dict(args, button="right")) + + +def right_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_element(ctx, dict(args, button="right")) + + +def double_click_at(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_at(ctx, dict(args, double=True)) + + +def double_click_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + return click_element(ctx, dict(args, count=2)) + + +def drag(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, _ = _new_step_id("drag") + src = args.get("from") or {} + dst = args.get("to") or {} + modifiers = list(args.get("modifiers") or []) + # Resolve element references on either end. + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + tree = (ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if info else None) + + def _to_xy(spec: Dict[str, Any]) -> Optional[Tuple[int, int]]: + if "x" in spec and "y" in spec: + return int(spec["x"]), int(spec["y"]) + eid = spec.get("element_id") + sel_text = spec.get("selector") + if tree is None: + return None + if sel_text: + try: + resu = sel.resolve(tree, sel.parse(sel_text)) + if resu.matches: + e = resu.matches[0] + return e.bounds.center_x, e.bounds.center_y + except sel.SelectorParseError: + return None + if eid: + e = _find_by_id(tree, eid) + if e is not None: + return e.bounds.center_x, e.bounds.center_y + return None + + p1 = _to_xy(src) + p2 = _to_xy(dst) + if p1 is None or p2 is None: + return error_dict(Code.BAD_REQUEST, + "drag requires from/to as {x,y} or {selector|element_id}", + step_id=step_id) + + # §21 confirmation gate: drag endpoints addressed by selector/element_id + # resolve to concrete elements, so a confirmation_required rule matching + # either endpoint (e.g. a "Trash" drop target) must demand a token, just + # like the other element-targeted verbs. The first matching endpoint is + # validated (propose_action(action="drag", args={"selector": …}) issues + # the token for that endpoint's selector). + if tree is not None and info is not None: + for spec in (src, dst): + if not (spec.get("selector") or spec.get("element_id")): + continue + elem, selector_str, res_err = _resolve_element(tree, spec) + if res_err or elem is None: + continue + tgt = {"window_uid": info.window_uid, + "element_id": elem.element_id, + "selector": selector_str, + "bounds": elem.bounds.to_dict()} + if _confirmation_rules_match(ctx, tgt): + confirm_check = _check_confirmation(ctx, "drag", args, tgt) + if confirm_check is not None: + return {**confirm_check, "step_id": step_id} + break # token validated for the matching endpoint + + duration = float(args.get("duration_s", 0.5)) + path = [p1, ((p1[0] + p2[0]) // 2, (p1[1] + p2[1]) // 2), p2] + try: + import pyautogui + for k in modifiers: + pyautogui.keyDown(k) + try: + pyautogui.moveTo(*p1) + pyautogui.dragTo(*p2, duration=duration, button="left") + finally: + for k in modifiers: + pyautogui.keyUp(k) + ok = True + err = None + except Exception as e: + ok = False + err = str(e) + out = {"ok": ok, "success": ok, "action": "drag", + "step_id": step_id, "caused_by_step_id": step_id, + "from": list(p1), "to": list(p2), + "modifiers": modifiers, "path": [list(p) for p in path]} + if err: + out["error"] = {"code": Code.INTERNAL, "message": err, + "recoverable": False, "suggested_next_tool": None, + "context": {}} + return out + + +def key_into_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + keys = args.get("keys", "") + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + return ctx.observer.perform_action("key", value=keys, hwnd=info.handle) + return _do_element_action(ctx, action_name="key_into_element", + args=args, executor=_exec) + + +def clear_text(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + def _exec(elem: UIElement, info: WindowInfo, _a: Dict[str, Any] + ) -> Dict[str, Any]: + ctx.observer.perform_action( + "click_at", + element_id=elem.element_id, + value={"x": elem.bounds.center_x, "y": elem.bounds.center_y, + "button": "left", "double": False}, + hwnd=info.handle, + ) + ctx.observer.perform_action("key", value="ctrl+a", hwnd=info.handle) + return ctx.observer.perform_action("key", value="delete", + hwnd=info.handle) + return _do_element_action(ctx, action_name="clear_text", + args=args, executor=_exec) diff --git a/tools/context.py b/tools/context.py new file mode 100644 index 0000000..f09537d --- /dev/null +++ b/tools/context.py @@ -0,0 +1,142 @@ +""" +Tool execution context and window/element resolution helpers. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from observer import ScreenObserver, UIElement, WindowInfo, WindowResolution +from session import Session, get_session + +logger = logging.getLogger(__name__) + + +@dataclass +class ToolContext: + observer: ScreenObserver + renderer: Any + describer: Any + config: Dict[str, Any] + + @property + def session(self) -> Session: + return get_session() + + +def _is_input_tool(name: str) -> bool: + return name in { + "click_at", "type_text", "press_key", "scroll", "bring_to_foreground", + "click_element", "focus_element", "set_value", "invoke_element", + "select_option", "hover_at", "hover_element", + "right_click_at", "right_click_element", + "double_click_at", "double_click_element", + "drag", "key_into_element", "clear_text", + "click_element_and_observe", "type_and_observe", "press_key_and_observe", + } + + +def _new_step_id(name: str) -> Tuple[int, Optional[int]]: + return get_session().steps.next_id(is_input=_is_input_tool(name)) + + +def _resolve_window(ctx: ToolContext, args: Dict[str, Any] + ) -> Tuple[List[WindowInfo], WindowResolution]: + windows = ctx.observer.list_windows() + res = ctx.observer.resolve_window( + windows, + window_uid=args.get("window_uid"), + window_index=args.get("window_index"), + window_title=args.get("window_title"), + ) + return windows, res + + +def _focused_window(windows: List[WindowInfo]) -> Optional[WindowInfo]: + for w in windows: + if w.is_focused: + return w + return windows[0] if windows else None + + +def _resolve_element(tree: UIElement, args: Dict[str, Any] + ) -> Tuple[Optional[UIElement], Optional[str], Optional[Dict]]: + """ + Returns (element, selector_string, error_dict). Either *element* is set + or *error_dict* is. The selector string is resolved or derived. + """ + selector = args.get("selector") + element_id = args.get("element_id") + + # Try selector first, then element_id. When both are provided, element_id + # acts as a fallback so a bad/unmatched selector doesn't block the call. + selector_err = None + if selector: + try: + parsed = sel.parse(selector) + result = sel.resolve(tree, parsed) + if result.matches: + return result.matches[0], parsed.canonical(), None + selector_err = error_dict( + Code.ELEMENT_NOT_FOUND, + f"no element matches selector {selector!r}", + selector=selector, + ) + except sel.SelectorParseError as e: + selector_err = error_dict(Code.BAD_REQUEST, + f"selector parse error: {e}", + selector=selector) + + if element_id: + elem = _find_by_id(tree, element_id) + if elem is not None: + derived = sel.selector_for(tree, element_id) or "" + return elem, derived, None + # element_id didn't match any internal ID — try parsing it as a selector + # (LLMs often pass the display-format string e.g. 'TabItem "name"' as an id). + try: + parsed = sel.parse(element_id) + result = sel.resolve(tree, parsed) + if result.matches: + return result.matches[0], parsed.canonical(), None + except Exception as e: + logger.debug(f"element_id {element_id!r} not parseable as a " + f"selector either: {e}") + # Both failed — prefer selector error if we have one (more informative). + if selector_err: + return None, None, selector_err + return None, None, error_dict( + Code.ELEMENT_NOT_FOUND, + f"no element with id {element_id!r}", + element_id=element_id, + ) + + if selector_err: + return None, None, selector_err + + return None, None, error_dict(Code.BAD_REQUEST, + "either element_id or selector is required") + + +def _find_by_id(elem: UIElement, target: str) -> Optional[UIElement]: + if elem.element_id == target: + return elem + for c in elem.children: + r = _find_by_id(c, target) + if r is not None: + return r + return None + + +def _new_dialogs(before: List[WindowInfo], after: List[WindowInfo]) -> List[Dict]: + before_uids = {w.window_uid for w in before} + return [ + {"window_uid": w.window_uid, "title": w.title} + for w in after if w.window_uid and w.window_uid not in before_uids + ] diff --git a/tools/dispatch.py b/tools/dispatch.py new file mode 100644 index 0000000..4d82755 --- /dev/null +++ b/tools/dispatch.py @@ -0,0 +1,268 @@ +""" +Dispatch table, allowlist gate and cross-cutting hooks. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Dict, Optional + +from errors import Code, error_dict +from hashing import tree_hash +from redaction import mark_untrusted +from session import get_session + +from tools.context import ToolContext, _is_input_tool, _new_step_id +from tools import actions, meta, observe, snapshots, trace_replay, vision + +logger = logging.getLogger(__name__) + + +REGISTRY: Dict[str, Callable[[ToolContext, Dict[str, Any]], Dict[str, Any]]] = { + # Read-only + "list_windows": meta.list_windows, + "get_capabilities": meta.get_capabilities, + "get_monitors": meta.get_monitors, + "find_element": observe.find_element, + "get_window_structure": observe.get_window_structure, + "get_screenshot": vision.get_screenshot, + "get_visible_areas": observe.get_visible_areas, + + # Element-targeted actions + "click_element": actions.click_element, + "focus_element": actions.focus_element, + "set_value": actions.set_value, + "invoke_element": actions.invoke_element, + "select_option": actions.select_option, + + # Legacy actions + "click_at": actions.click_at, + "type_text": actions.type_text, + "press_key": actions.press_key, + "scroll": actions.scroll, + "bring_to_foreground": actions.bring_to_foreground, + + # P2: sync, diff, snapshots, composites + "observe_window": observe.observe_window, + "snapshot": snapshots.snapshot, + "snapshot_get": snapshots.snapshot_get, + "snapshot_diff": snapshots.snapshot_diff, + "snapshot_drop": snapshots.snapshot_drop, + "wait_for": snapshots.wait_for, + "wait_idle": snapshots.wait_idle, + "click_element_and_observe": actions.click_element_and_observe, + "type_and_observe": actions.type_and_observe, + "press_key_and_observe": actions.press_key_and_observe, + + # P3 + "get_screenshot_cropped": vision.get_screenshot_cropped, + "get_ocr": vision.get_ocr, + "get_screen_description": vision.get_screen_description, + + # P4: tracing, replay, scenarios, oracles + "trace_start": trace_replay.trace_start, + "trace_stop": trace_replay.trace_stop, + "trace_status": trace_replay.trace_status, + "replay_start": trace_replay.replay_start, + "replay_step": trace_replay.replay_step, + "replay_status": trace_replay.replay_status, + "replay_stop": trace_replay.replay_stop, + "load_scenario": trace_replay.load_scenario, + "assert_state": trace_replay.assert_state, + + # P5: budgets, propose_action, status reporters + "get_budget_status": meta.get_budget_status, + "get_redaction_status": meta.get_redaction_status, + "propose_action": meta.propose_action, + + # P6: extra input verbs + "hover_at": actions.hover_at, + "hover_element": actions.hover_element, + "right_click_at": actions.right_click_at, + "right_click_element": actions.right_click_element, + "double_click_at": actions.double_click_at, + "double_click_element": actions.double_click_element, + "drag": actions.drag, + "key_into_element": actions.key_into_element, + "clear_text": actions.clear_text, +} + + +_ALLOWLIST_TOOLS = { + "get_capabilities", "get_monitors", "get_budget_status", + "get_redaction_status", "trace_status", "replay_status", + "list_windows", +} + + +def _check_allowlist(ctx: ToolContext, name: str + ) -> Optional[Dict[str, Any]]: + actions = ctx.config.get("actions") or {} + allow = set(actions.get("allow") or []) + deny = set(actions.get("deny") or []) + default = actions.get("default", "allow") + if name in _ALLOWLIST_TOOLS: + return None + if name in deny: + return error_dict(Code.PERMISSION_DENIED, + f"tool {name!r} is in actions.deny") + if allow and name not in allow and default == "deny": + return error_dict(Code.PERMISSION_DENIED, + f"tool {name!r} is not in actions.allow") + if not allow and default == "deny": + return error_dict(Code.PERMISSION_DENIED, + f"actions.default is 'deny' and no allowlist matches {name!r}") + return None + + +def dispatch(ctx: ToolContext, name: str, args: Dict[str, Any]) -> Dict[str, Any]: + fn = REGISTRY.get(name) + if fn is None: + return error_dict(Code.BAD_REQUEST, f"unknown tool: {name}") + + blocked = _check_allowlist(ctx, name) + if blocked is not None: + sid, _ = _new_step_id(name) + blocked["step_id"] = sid + return blocked + + started = time.time() + sess = get_session() + tree_before = "" + if sess.active_trace is not None and not sess.active_trace.closed: + try: + windows0 = ctx.observer.list_windows() + focused0 = next((w for w in windows0 if w.is_focused), None) + if focused0: + t = ctx.observer.get_element_tree( + focused0.handle, window_uid=focused0.window_uid) + if t: + tree_before = tree_hash(t) + except Exception as e: + logger.debug(f"trace: pre-call tree hash unavailable: {e}") + + try: + result = fn(ctx, args or {}) + except Exception as e: + logger.exception(f"tool {name} crashed") + result = error_dict(Code.INTERNAL, f"{type(e).__name__}: {e}") + + duration_ms = int((time.time() - started) * 1000) + + # P1 tree cache invalidation — the single choke point. After any input + # tool (including bring_to_foreground) the affected window's cached tree + # is stale; drop it so subsequent reads re-walk. When the target window + # cannot be determined (legacy coordinate tools), drop everything. + if _is_input_tool(name): + try: + uid = None + if isinstance(result, dict): + uid = ((result.get("target") or {}).get("window_uid") + or result.get("window_uid")) + uid = uid or (args or {}).get("window_uid") + if uid: + sess.tree_cache.invalidate(uid) + else: + sess.tree_cache.invalidate_all() + except Exception: + logger.exception("tree cache invalidation failed") + + # Recurse-safety: don't trace meta tools that would recurse forever. + if name not in {"trace_start", "trace_stop", "trace_status", + "replay_start", "replay_step", "replay_status", + "replay_stop"}: + if sess.active_trace is not None and not sess.active_trace.closed: + try: + import tracing as _tracing + shot_full = ctx.observer.get_full_display_screenshot() + shot_window = None + tgt_uid = None + tgt = result.get("target") or {} + tgt_uid = tgt.get("window_uid") or args.get("window_uid") + if tgt_uid: + win = ctx.observer.window_by_uid( + ctx.observer.list_windows(), tgt_uid) + if win: + shot_window = ctx.observer.get_screenshot(win.handle) + tree_after = "" + try: + after_w = ctx.observer.list_windows() + f = next((w for w in after_w if w.is_focused), None) + if f: + t = ctx.observer.get_element_tree( + f.handle, window_uid=f.window_uid, + use_cache=False) + if t: + tree_after = tree_hash(t) + except Exception as e: + logger.debug(f"trace: post-call tree hash " + f"unavailable: {e}") + _tracing.record( + sess.active_trace, + tool=name, caller=args.get("_caller", "unknown"), + args={k: v for k, v in (args or {}).items() + if not k.startswith("_")}, + result=result, duration_ms=duration_ms, + tree_hash_before=tree_before, tree_hash_after=tree_after, + full_screenshot=shot_full, + window_screenshot=shot_window, + ) + except Exception: + logger.exception("trace.record failed") + + # Apply redaction to text-bearing read-only results. + if sess.redactor is not None: + try: + result = _apply_redaction(name, result, sess.redactor) + except Exception: + logger.exception("redaction failed") + + # Untrusted-content marking (always on): screen-derived text is + # attacker-influenced data (prompt injection), never instructions. + # Flag it and strip ANSI/control characters. + try: + result = mark_untrusted(name, result) + except Exception: + logger.exception("untrusted-content marking failed") + + # Budget accounting. + if sess.budgets is not None: + try: + sess.budgets.note(name, result) + except Exception: + logger.exception("budget accounting failed") + + # Audit log. + if sess.auditor is not None: + try: + sess.auditor.record( + tool=name, caller=args.get("_caller", "unknown"), + args=args or {}, result=result, + ) + except Exception: + logger.exception("audit failed") + + return result + + +def _apply_redaction(tool: str, result: Dict[str, Any], redactor: Any + ) -> Dict[str, Any]: + if not redactor.is_active() or not isinstance(result, dict): + return result + if tool == "get_window_structure" and "tree" in result: + title = result.get("window") or "" + result["tree"] = redactor.redact_tree(result["tree"], title) + elif tool == "observe_window": + if "tree" in result and isinstance(result["tree"], dict): + result["tree"] = redactor.redact_tree(result["tree"], + result.get("window") or "") + elif tool == "get_screen_description": + if isinstance(result.get("description"), str): + result["description"] = redactor.redact_ocr_text(result["description"]) + elif tool == "get_ocr": + if isinstance(result.get("words"), list): + result["words"] = redactor.redact_ocr_words(result["words"]) + return result diff --git a/tools/meta.py b/tools/meta.py new file mode 100644 index 0000000..5ebb532 --- /dev/null +++ b/tools/meta.py @@ -0,0 +1,133 @@ +""" +Introspection tools: windows, capabilities, status, proposals. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +from typing import Any, Dict + +from errors import Code, error_dict +from session import get_session + +from tools.context import ( + ToolContext, _focused_window, _new_step_id, _resolve_element, + _resolve_window, +) +from tools.vision import _apply_crop + +logger = logging.getLogger(__name__) + + +def list_windows(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("list_windows") + try: + windows = ctx.observer.list_windows() + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "is_mock": ctx.observer.is_mock, + "count": len(windows), + "windows": [{"index": i, **w.to_dict()} for i, w in enumerate(windows)], + } + except Exception as e: + logger.exception("list_windows failed") + return error_dict(Code.INTERNAL, str(e), step_id=step_id) + + +def get_capabilities(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_capabilities") + out = ctx.observer.get_capabilities() + out.update({"step_id": step_id, "caused_by_step_id": caused_by, "success": True}) + return out + + +def get_monitors(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_monitors") + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "monitors": ctx.observer.get_monitors(), + } + + +def get_budget_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_budget_status") + sess = get_session() + if sess.budgets is None: + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "configured": False} + out = sess.budgets.status() + out.update({"ok": True, "success": True, "configured": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return out + + +def get_redaction_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_redaction_status") + sess = get_session() + out = sess.redactor.status() if sess.redactor else {"enabled": False, "active": False} + out.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return out + + +def propose_action(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Issue a single-use confirm_token for a destructive action.""" + step_id, caused_by = _new_step_id("propose_action") + action = args.get("action") + inner_args = args.get("args") or {} + if not action: + return error_dict(Code.BAD_REQUEST, "action is required", + step_id=step_id) + + windows, res = _resolve_window(ctx, inner_args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id) + elem, selector_str, err = _resolve_element(tree, inner_args) + if err or elem is None: + return {**(err or error_dict(Code.ELEMENT_NOT_FOUND, + "element resolution failed")), + "step_id": step_id} + + sess = get_session() + bbox = elem.bounds.to_dict() + ct = sess.confirms.issue( + action=action, window_uid=info.window_uid, + selector=selector_str or "", bbox=bbox, args=inner_args, + ) + # Optional preview crop. + preview_b64 = None + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot is not None: + crop_bytes, _ = _apply_crop(shot, bbox=bbox, padding=8, + max_width=400) + if crop_bytes: + preview_b64 = base64.b64encode(crop_bytes).decode() + except Exception as e: + logger.debug(f"propose_action: preview crop failed: {e}") + + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "confirm_token": ct.token, + "expires_at": ct.expires_at, + "would_target": { + "window_uid": info.window_uid, + "selector": selector_str, + "bounds": bbox, + "screenshot_b64": preview_b64, + }, + } diff --git a/tools/observe.py b/tools/observe.py new file mode 100644 index 0000000..b3d5d3f --- /dev/null +++ b/tools/observe.py @@ -0,0 +1,543 @@ +""" +Tree observation: structure, diffs, filtering/paging helpers. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +import time +from typing import Any, Dict, List, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from hashing import tree_hash +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ( + ToolContext, _focused_window, _new_step_id, _resolve_window, +) + + +def find_element(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("find_element") + selector_text = args.get("selector") + if not selector_text: + return error_dict(Code.BAD_REQUEST, "selector is required", + step_id=step_id) + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + try: + parsed = sel.parse(selector_text) + except sel.SelectorParseError as e: + return error_dict(Code.BAD_REQUEST, f"selector parse error: {e}", + step_id=step_id) + result = sel.resolve(tree, parsed) + if not result.matches: + return error_dict(Code.ELEMENT_NOT_FOUND, + f"no element matches {selector_text!r}", + step_id=step_id, selector=selector_text, + window_uid=info.window_uid) + first = result.matches[0] + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, + "element_id": first.element_id, + "selector": parsed.canonical(), + "bounds": first.bounds.to_dict(), + "ambiguous_matches": len(result.matches), + "all_matches": [ + {"element_id": m.element_id, "bounds": m.bounds.to_dict(), + "name": m.name, "role": m.role} + for m in result.matches + ], + } + + +def _effective_depth(ctx: ToolContext, args: Dict[str, Any]) -> int: + """Depth to return: caller's depth= (clamped to tree.max_depth) or + tree.default_depth when the caller passes none.""" + tree_cfg = ctx.config.get("tree", {}) or {} + hard_cap = int(tree_cfg.get("max_depth", 8)) + requested = args.get("depth") + if requested is None: + depth = int(tree_cfg.get("default_depth", 5)) + else: + try: + depth = int(requested) + except (TypeError, ValueError): + depth = int(tree_cfg.get("default_depth", 5)) + return max(0, min(depth, hard_cap)) + + +def _truncate_depth(node: Optional[Dict[str, Any]], max_depth: int, + _depth: int = 0) -> Tuple[Optional[Dict[str, Any]], bool]: + """Copy *node* limited to *max_depth* levels below it. + + Nodes whose children were dropped are marked ``truncated: true`` with a + ``child_count`` so the caller knows to drill in (via scope=/depth=). + Returns (tree, any_node_truncated). The input dict is not mutated.""" + if node is None: + return None, False + out = dict(node) + children = node.get("children") or [] + if _depth >= max_depth and children: + out["children"] = [] + out["truncated"] = True + out["child_count"] = len(children) + return out, True + truncated_any = False + new_children: List[Dict[str, Any]] = [] + for c in children: + nc, t = _truncate_depth(c, max_depth, _depth + 1) + if nc is not None: + new_children.append(nc) + truncated_any = truncated_any or t + out["children"] = new_children + return out, truncated_any + + +def get_window_structure(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_window_structure") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + + depth = _effective_depth(ctx, args) + scope = args.get("scope") + token = None + + if scope: + # Drill into one branch only (element-id path, e.g. 'root.3.2'). + ttl = float((ctx.config.get("tree", {}) or {}).get("cache_ttl_s", 2.0)) + had_fresh_entry = (get_session().tree_cache.get( + info.window_uid, ttl_s=ttl) is not None) + started = time.time() + tree = ctx.observer.get_element_subtree( + info.handle, scope, max_depth=depth, + window_uid=info.window_uid) + capture_ms = int((time.time() - started) * 1000) + if tree is None: + return error_dict(Code.ELEMENT_NOT_FOUND, + f"no element with id {scope!r} to scope to", + step_id=step_id, scope=scope, + window_uid=info.window_uid) + perf = {"capture_ms": capture_ms, + "node_count": len(tree.flat_list()), + "cache": "hit" if had_fresh_entry else "miss", + "depth_used": depth} + # scoped captures are not valid diff baselines → no tree_token + else: + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + perf = {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} + full_serialized = tree.to_dict() + th = tree_hash(tree) + if not scope: + token = get_session().tree_tokens.put(info.window_uid, + full_serialized, th) + serialized, depth_truncated = _truncate_depth(full_serialized, depth) + + # P3 filtering / paging -------------------------------------------------- + roles = args.get("roles") + exclude_roles = args.get("exclude_roles") + visible_only = bool(args.get("visible_only")) + name_regex = args.get("name_regex") + max_text_len = args.get("max_text_len") + prune_empty = bool(args.get("prune_empty")) + max_nodes = args.get("max_nodes") + page_cursor = args.get("page_cursor") + + visible_regions: Optional[List[Dict[str, int]]] = None + if visible_only: + try: + visible_regions = ctx.observer.get_visible_areas(info.handle, windows) + except Exception: + visible_regions = [] + + filtered = _filter_tree( + serialized or {}, + roles=set(roles) if roles else None, + exclude_roles=set(exclude_roles) if exclude_roles else None, + visible_regions=visible_regions, + name_regex=name_regex, + max_text_len=max_text_len, + prune_empty=prune_empty, + ) + + truncated = False + next_cursor: Optional[str] = None + node_count = _count_nodes(filtered) if filtered else 0 + if max_nodes is not None or page_cursor is not None: + filtered, truncated, next_cursor, node_count = _page_tree( + filtered, max_nodes=max_nodes, page_cursor=page_cursor, + ) + + out = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, + "window_uid": info.window_uid, + "element_count": len(tree.flat_list()), + "node_count": node_count, + "tree": filtered, + "tree_hash": th, + "tree_token": token, + "truncated": truncated, + "next_cursor": next_cursor, + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": perf, + } + if scope: + out["scope"] = scope + else: + # Degradation signal: accessibility-dark windows (games, custom + # renderers) produce sparse trees — steer the agent to pixel-based + # fallbacks instead of letting it act on a near-empty tree. + named = sum(1 for e in tree.flat_list()[1:] if (e.name or "").strip()) + threshold = int((ctx.config.get("tree", {}) or {}) + .get("sparse_threshold", 5)) + if named < threshold: + out["degraded"] = { + "reason": "sparse_accessibility_tree", + "named_node_count": named, + "threshold": threshold, + "suggested_fallbacks": ["get_ocr", "get_screen_description"], + } + return out + + +def get_visible_areas(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_visible_areas") + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + areas = ctx.observer.get_visible_areas(info.handle, windows) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, + "window_uid": info.window_uid, + "visible_regions": areas, + } + + +def _perf_dict(meta: Dict[str, Any], depth: Optional[int]) -> Dict[str, Any]: + return {"capture_ms": meta["capture_ms"], + "node_count": meta["node_count"], + "cache": meta["cache"], + "depth_used": depth} + + +def _serialize_full_observation(ctx: ToolContext, info: WindowInfo, + depth: Optional[int] = None, + ) -> Tuple[Optional[UIElement], Dict[str, Any]]: + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return None, {"error": "no tree"} + serialized = tree.to_dict() + th = tree_hash(tree) + # Diff baselines keep the full capture; only the returned tree is + # depth-bounded (with truncated-node markers). + token = get_session().tree_tokens.put(info.window_uid, serialized, th) + out_tree: 'Optional[Dict[str, Any]]' = serialized + depth_truncated = False + if depth is not None: + out_tree, depth_truncated = _truncate_depth(serialized, depth) + return tree, { + "format": "full", + "window_uid": info.window_uid, + "window": info.title, + "tree": out_tree, + "tree_hash": th, + "tree_token": token, + "base_token": None, + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), + } + + +def observe_window(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Return the current tree, optionally as a diff against a tree_token.""" + from diff import diff_custom, diff_json_patch + step_id, caused_by = _new_step_id("observe_window") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + since = args.get("since") + fmt = args.get("format", "custom") + depth = _effective_depth(ctx, args) + changed_only = bool(args.get("changed_only")) + + if not since and changed_only: + return _observe_changed_only(ctx, info, depth=depth, + step_id=step_id, caused_by=caused_by) + + if not since: + _, full = _serialize_full_observation(ctx, info, depth=depth) + full.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "format": "full"}) + return full + + entry = get_session().tree_tokens.get(since) + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + serialized = tree.to_dict() + th = tree_hash(tree) + new_token = get_session().tree_tokens.put(info.window_uid, serialized, th) + + if entry is None or entry.window_uid != info.window_uid: + # Token expired/wrong-window: return full tree (depth-bounded). + out_tree, depth_truncated = _truncate_depth(serialized, depth) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree": out_tree, "tree_hash": th, + "tree_token": new_token, "base_token": None, + "format": "full", + "depth_used": depth, + "depth_truncated": depth_truncated, + "perf": _perf_dict(meta, depth), + } + + if fmt == "json-patch": + changes = diff_json_patch(entry.serialized, serialized) + out_format = "json-patch" + else: + changes = diff_custom(entry.serialized, serialized) + out_format = "custom" + + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree_token": new_token, "base_token": since, + "format": out_format, + "changes": changes, + "unchanged": len(changes) == 0, + "tree_hash": th, + "perf": _perf_dict(meta, depth), + } + + +def _observe_changed_only(ctx: ToolContext, info: WindowInfo, *, depth: int, + step_id: int, caused_by: Optional[int] + ) -> Dict[str, Any]: + """observe_window changed_only=true: compare a fresh capture against the + last cached capture of the window. Unchanged → a tiny + {unchanged: true, tree_hash} response; changed → a custom diff instead + of the full tree; no baseline → full tree.""" + from diff import diff_custom + sess = get_session() + # Baseline: the most recent capture, regardless of cache TTL. + baseline = sess.tree_cache.peek(info.window_uid) + + # Fresh capture (bypass the cache — the whole point is to detect drift). + tree, meta = ctx.observer.get_element_tree_with_meta( + info.handle, window_uid=info.window_uid, use_cache=False) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id, window_uid=info.window_uid) + serialized = tree.to_dict() + th = tree_hash(tree) + new_token = sess.tree_tokens.put(info.window_uid, serialized, th) + perf = _perf_dict(meta, depth) + + base: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window_uid": info.window_uid, "window": info.title, + "tree_hash": th, "tree_token": new_token, + "changed_only": True, + "perf": perf, + } + + if baseline is None: + # Nothing to compare against — return the (depth-bounded) full tree. + out_tree, depth_truncated = _truncate_depth(serialized, depth) + base.update({"format": "full", "tree": out_tree, "base_token": None, + "depth_used": depth, "depth_truncated": depth_truncated}) + return base + + if baseline.tree_hash == th: + base["unchanged"] = True + return base + + changes = diff_custom(baseline.serialized, serialized) + base.update({"format": "custom", "changes": changes, "unchanged": False}) + return base + + +def _filter_tree(node: Dict[str, Any], *, roles: Optional[set], + exclude_roles: Optional[set], + visible_regions: Optional[List[Dict[str, int]]], + name_regex: Optional[str], + max_text_len: Optional[int], + prune_empty: bool) -> Optional[Dict[str, Any]]: + if node is None: + return None + role = node.get("role") + name = node.get("name") or "" + bounds = node.get("bounds") or {} + + # Role filter + role_keep = True + if roles is not None and role not in roles: + role_keep = False + if exclude_roles is not None and role in exclude_roles: + role_keep = False + + # Name regex + name_keep = True + if name_regex: + try: + name_keep = bool(re.search(name_regex, name)) + except re.error: + name_keep = True + + # Visibility + visible_keep = True + if visible_regions is not None: + visible_keep = _intersects_any(bounds, visible_regions) + + self_keep = role_keep and name_keep and visible_keep + + # Recurse children regardless (so we can keep ancestors if descendants match) + new_children: List[Dict[str, Any]] = [] + for c in node.get("children", []) or []: + fc = _filter_tree( + c, roles=roles, exclude_roles=exclude_roles, + visible_regions=visible_regions, name_regex=name_regex, + max_text_len=max_text_len, prune_empty=prune_empty, + ) + if fc is not None: + new_children.append(fc) + + if prune_empty and not self_keep and not new_children: + return None + + # Truncate text fields if requested. + truncated_node = dict(node) + if max_text_len is not None: + n = int(max_text_len) + if isinstance(truncated_node.get("name"), str) and len(truncated_node["name"]) > n: + truncated_node["name"] = truncated_node["name"][:n] + "…" + v = truncated_node.get("value") + if isinstance(v, str) and len(v) > n: + truncated_node["value"] = v[:n] + "…" + truncated_node["children"] = new_children + return truncated_node + + +def _intersects_any(b: Dict[str, int], regions: List[Dict[str, int]]) -> bool: + if not b: + return False + bx, by = b.get("x", 0), b.get("y", 0) + bw, bh = b.get("width", 0), b.get("height", 0) + if bw <= 0 or bh <= 0: + return False + bx2, by2 = bx + bw, by + bh + for r in regions: + rx, ry = r.get("x", 0), r.get("y", 0) + rx2, ry2 = rx + r.get("width", 0), ry + r.get("height", 0) + if bx < rx2 and bx2 > rx and by < ry2 and by2 > ry: + return True + return False + + +def _count_nodes(node: Optional[Dict[str, Any]]) -> int: + if node is None: + return 0 + return 1 + sum(_count_nodes(c) for c in (node.get("children") or [])) + + +def _page_tree(node: Optional[Dict[str, Any]], *, + max_nodes: Optional[int], + page_cursor: Optional[str] + ) -> Tuple[Optional[Dict[str, Any]], bool, Optional[str], int]: + """ + Paginated DFS walk. Returns (subtree-shaped result containing only the + page slice, truncated flag, next_cursor, node_count_in_page). + + Cursors are post-order element_ids; resuming starts from the next sibling + in the original walk. This is a best-effort pager — if the tree changed, + callers will get SnapshotExpired-shaped semantics by virtue of an unknown + cursor returning truncated:false and node_count:0. + """ + if node is None: + return None, False, None, 0 + flat: List[Dict[str, Any]] = [] + _flatten(node, flat) + + if page_cursor is not None: + for i, n in enumerate(flat): + if n.get("id") == page_cursor: + flat = flat[i + 1:] + break + else: + return None, False, None, 0 + + if max_nodes is None or max_nodes >= len(flat): + # Return full (possibly trimmed) tree starting from cursor. + if page_cursor is None: + return node, False, None, len(flat) + return _flat_to_tree(flat), False, None, len(flat) + + page = flat[:max_nodes] + truncated = True + next_cursor = page[-1].get("id") if page else None + return _flat_to_tree(page), truncated, next_cursor, len(page) + + +def _flatten(node: Dict[str, Any], out: List[Dict[str, Any]]) -> None: + out.append(node) + for c in node.get("children") or []: + _flatten(c, out) + + +def _flat_to_tree(flat: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Wrap a list of nodes as children of a synthetic Window root.""" + if not flat: + return None + return { + "id": "page-root", + "name": "[paged]", + "role": "Group", + "value": None, + "bounds": {"x": 0, "y": 0, "width": 0, "height": 0}, + "enabled": True, "focused": False, + "keyboard_shortcut": None, "description": None, + "children": [dict(n, children=[]) for n in flat], + } diff --git a/tools/receipts.py b/tools/receipts.py new file mode 100644 index 0000000..ae08b69 --- /dev/null +++ b/tools/receipts.py @@ -0,0 +1,117 @@ +""" +Action receipts and confirmation gating. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Optional + +from errors import Code, error_dict +from hashing import focused_selector, tree_hash +from observer import UIElement, WindowInfo +from session import get_session + +from tools.context import ToolContext, _new_dialogs + + +def _build_receipt(*, step_id: int, action: str, target: Dict[str, Any], + before_tree: Optional[UIElement], + before_windows: List[WindowInfo], + after_tree: Optional[UIElement], + after_windows: List[WindowInfo], + duration_ms: int, dry_run: bool, ok: bool, + extra: Optional[Dict[str, Any]] = None, + error: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + before_hash = tree_hash(before_tree) if before_tree else "" + after_hash = tree_hash(after_tree) if after_tree else "" + receipt: Dict[str, Any] = { + "ok": ok, "success": ok, + "step_id": step_id, "caused_by_step_id": step_id, + "action": action, + "dry_run": dry_run, + "target": target, + "before": { + "tree_hash": before_hash, + "focused_selector": focused_selector(before_tree) if before_tree else "", + }, + "after": { + "tree_hash": after_hash, + "focused_selector": focused_selector(after_tree) if after_tree else "", + }, + "changed": (before_hash != after_hash) and not dry_run, + "new_dialogs": _new_dialogs(before_windows, after_windows), + "duration_ms": duration_ms, + } + if extra: + receipt.update(extra) + if error: + receipt["error"] = error + return receipt + + +def _confirmation_rules_match(ctx: ToolContext, + target: Dict[str, Any]) -> bool: + """True when any configured confirmation_required rule matches the + target's role/name (derived from the selector tail).""" + confirm = ctx.config.get("confirmation_required") or [] + if not confirm: + return False + name_to_test = "" + role_to_test = "" + # Best-effort: derive name/role from the selector tail. + sel_tail = (target.get("selector") or "").split("/")[-1] + m = re.match(r"([A-Za-z_*]\w*)", sel_tail) + if m: + role_to_test = m.group(1) + nm = re.search(r'name="([^"]*)"', sel_tail) + if nm: + name_to_test = nm.group(1) + + def _matches_rule(rule: Dict[str, Any]) -> bool: + rname = rule.get("name_regex") + rrole = rule.get("role") + if rrole and role_to_test != rrole: + return False + if rname and not re.search(rname, name_to_test or ""): + return False + return bool(rname or rrole) + + return any(_matches_rule(r) for r in confirm) + + +def _check_confirmation(ctx: ToolContext, action_name: str, + args: Dict[str, Any], + target: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Returns an error dict to short-circuit when confirmation is required.""" + if not _confirmation_rules_match(ctx, target): + return None + + token = args.get("confirm_token") + if not token: + return error_dict( + Code.CONFIRMATION_REQUIRED, + f"action {action_name} requires a confirm_token from propose_action", + action=action_name, target=target, + ) + sess = get_session() + ct = sess.confirms.consume(token) + if not ct: + return error_dict(Code.CONFIRMATION_INVALID, + "confirm_token expired, unknown, or already used", + token=token) + if ct.action != action_name or ct.window_uid != target["window_uid"] \ + or ct.selector != target["selector"]: + return error_dict(Code.CONFIRMATION_INVALID, + "confirm_token does not match the proposed action") + tol = (ctx.config.get("confirmation", {}) or {}).get("bbox_tolerance_px", 20) + bb = target["bounds"] + if (abs(bb["x"] - ct.bbox.get("x", 0)) > tol or + abs(bb["y"] - ct.bbox.get("y", 0)) > tol or + abs(bb["width"] - ct.bbox.get("width", 0)) > tol or + abs(bb["height"] - ct.bbox.get("height", 0)) > tol): + return error_dict(Code.CONFIRMATION_INVALID, + "element bounds drifted beyond confirmation tolerance") + return None diff --git a/tools/snapshots.py b/tools/snapshots.py new file mode 100644 index 0000000..af6ae1a --- /dev/null +++ b/tools/snapshots.py @@ -0,0 +1,278 @@ +""" +Snapshots and wait_for / wait_idle condition polling. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import re +import time +from typing import Any, Dict, Optional, Tuple + +import element_selectors as sel +from errors import Code, error_dict +from hashing import tree_hash +from session import get_session + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_step_id, _resolve_window, +) + + +def snapshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot") + windows = ctx.observer.list_windows() + trees: Dict[str, Dict[str, Any]] = {} + hashes: Dict[str, str] = {} + for w in windows: + try: + t = ctx.observer.get_element_tree(w.handle, + window_uid=w.window_uid) + if t is not None and w.window_uid: + trees[w.window_uid] = t.to_dict() + hashes[w.window_uid] = tree_hash(t) + except Exception: + continue + snap = get_session().snapshots.put( + windows=[w.to_dict() for w in windows], + trees=trees, tree_hashes=hashes, + ) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "snapshot_id": snap.snapshot_id, + "ts": snap.ts, + "summary": {"windows": len(snap.windows), "trees": len(trees)}, + } + + +def snapshot_get(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot_get") + sid = args.get("snapshot_id") + if not sid: + return error_dict(Code.BAD_REQUEST, "snapshot_id is required", + step_id=step_id) + snap = get_session().snapshots.get(sid) + if snap is None: + return error_dict(Code.SNAPSHOT_EXPIRED, + "snapshot expired or not found", + step_id=step_id, snapshot_id=sid) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "snapshot_id": snap.snapshot_id, "ts": snap.ts, + "windows": snap.windows, + "trees": snap.trees, + "tree_hashes": snap.tree_hashes, + } + + +def snapshot_diff(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + from diff import diff_custom, diff_json_patch + step_id, caused_by = _new_step_id("snapshot_diff") + a_id = args.get("a") + b_id = args.get("b") + if not a_id or not b_id: + return error_dict(Code.BAD_REQUEST, "a and b are required", + step_id=step_id) + sess = get_session() + a = sess.snapshots.get(a_id) + b = sess.snapshots.get(b_id) + if a is None or b is None: + return error_dict(Code.SNAPSHOT_EXPIRED, + "one or both snapshots are missing", + step_id=step_id) + fmt = args.get("format", "custom") + + a_uids = {w["window_uid"] for w in a.windows} + b_uids = {w["window_uid"] for w in b.windows} + windows_added = sorted(b_uids - a_uids) + windows_removed = sorted(a_uids - b_uids) + common = sorted(a_uids & b_uids) + + per_window: Dict[str, Any] = {} + for uid in common: + if uid in a.trees and uid in b.trees: + if fmt == "json-patch": + per_window[uid] = {"format": "json-patch", + "changes": diff_json_patch(a.trees[uid], b.trees[uid])} + else: + per_window[uid] = {"format": "custom", + "changes": diff_custom(a.trees[uid], b.trees[uid])} + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "windows_added": windows_added, + "windows_removed": windows_removed, + "per_window_changes": per_window, + } + + +def snapshot_drop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("snapshot_drop") + sid = args.get("snapshot_id") + dropped = get_session().snapshots.drop(sid) if sid else False + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "dropped": dropped, + } + + +def _check_condition(ctx: ToolContext, cond: Dict[str, Any], + window_uid_hint: Optional[str]) -> Tuple[bool, Dict[str, Any]]: + kind = cond.get("type") + sess = get_session() + windows = ctx.observer.list_windows() + info = ctx.observer.window_by_uid(windows, window_uid_hint) or _focused_window(windows) + + if kind == "window_appears": + rx = cond.get("title_regex", "") + for w in windows: + if re.search(rx, w.title): + return True, {"window_uid": w.window_uid, "title": w.title} + return False, {} + if kind == "window_disappears": + target = cond.get("window_uid") + for w in windows: + if w.window_uid == target: + return False, {} + return True, {"window_uid": target} + if kind == "focused_changes": + focus = next((w for w in windows if w.is_focused), None) + return (focus is not None), ({"focused_uid": focus.window_uid} if focus else {}) + if kind == "tree_changes": + token = cond.get("since") + entry = sess.tree_tokens.get(token) if token else None + if entry is None or info is None: + return False, {} + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + return tree is not None and tree_hash(tree) != entry.tree_hash, {} + + if info is None: + return False, {} + # Polling must observe fresh state — bypass the tree cache. + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + if tree is None: + return False, {} + + if kind == "element_appears": + sel_text = cond.get("selector") + if not sel_text: + return False, {} + try: + res = sel.resolve(tree, sel.parse(sel_text)) + except sel.SelectorParseError: + return False, {} + if res.matches: + m = res.matches[0] + return True, {"element_id": m.element_id, "bounds": m.bounds.to_dict()} + return False, {} + if kind == "element_disappears": + sel_text = cond.get("selector") + eid = cond.get("element_id") + if sel_text: + try: + res = sel.resolve(tree, sel.parse(sel_text)) + return not res.matches, {} + except sel.SelectorParseError: + return False, {} + if eid: + return _find_by_id(tree, eid) is None, {} + return False, {} + if kind == "text_visible": + rx = cond.get("regex", "") + # Walk tree names/values. + for elem in tree.flat_list(): + joined = (elem.name or "") + " " + (elem.value or "") + if re.search(rx, joined): + return True, {"element_id": elem.element_id} + return False, {} + return False, {} + + +def wait_for(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("wait_for") + timeout_ms = int(args.get("timeout_ms", 5000)) + cap = int((ctx.config.get("wait_for", {}) or {}).get("max_timeout_ms", 60000)) + timeout_ms = min(timeout_ms, cap) + poll_ms = max(50, int(args.get("poll_ms", 200))) + conditions = args.get("any_of", []) + if not conditions: + return error_dict(Code.BAD_REQUEST, "any_of is required", + step_id=step_id) + window_uid = args.get("window_uid") + + started = time.time() + polls = 0 + while True: + polls += 1 + for i, cond in enumerate(conditions): + try: + ok, detail = _check_condition(ctx, cond, window_uid) + except Exception: + ok, detail = False, {} + if ok: + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "matched_index": i, "matched_detail": detail, + "elapsed_ms": int((time.time() - started) * 1000), + "polls": polls, + } + elapsed = (time.time() - started) * 1000 + if elapsed >= timeout_ms: + err = error_dict( + Code.TIMEOUT, f"wait_for timed out after {int(elapsed)}ms", + step_id=step_id, + ) + err.update({ + "elapsed_ms": int(elapsed), "polls": polls, + "matched_index": None, + }) + return err + time.sleep(poll_ms / 1000.0) + + +def wait_idle(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("wait_idle") + timeout_ms = int(args.get("timeout_ms", 5000)) + quiet_ms = int(args.get("quiet_ms", 750)) + poll_ms = max(50, int(args.get("poll_ms", 100))) + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + + started = time.time() + last_hash = None + last_change_at = time.time() + while (time.time() - started) * 1000 < timeout_ms: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid, + use_cache=False) + if tree is None: + time.sleep(poll_ms / 1000.0) + continue + h = tree_hash(tree) + if h != last_hash: + last_hash = h + last_change_at = time.time() + elif (time.time() - last_change_at) * 1000 >= quiet_ms: + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "elapsed_ms": int((time.time() - started) * 1000), + "tree_hash": h, + } + time.sleep(poll_ms / 1000.0) + + err = error_dict(Code.TIMEOUT, "wait_idle timed out", step_id=step_id) + err["elapsed_ms"] = int((time.time() - started) * 1000) + return err diff --git a/tools/trace_replay.py b/tools/trace_replay.py new file mode 100644 index 0000000..606b4ff --- /dev/null +++ b/tools/trace_replay.py @@ -0,0 +1,183 @@ +""" +Tracing, replay, scenario loading and state oracles. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +from typing import Any, Dict + +from errors import Code, error_dict +from session import get_session + +from tools.context import ToolContext, _new_step_id + + +def trace_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import tracing as _tracing + step_id, caused_by = _new_step_id("trace_start") + sess = get_session() + if sess.active_trace is not None and not sess.active_trace.closed: + return error_dict(Code.BAD_REQUEST, "trace already active", + step_id=step_id, + trace_id=sess.active_trace.trace_id) + handle = _tracing.start(label=args.get("label", ""), config=ctx.config) + sess.active_trace = handle + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "trace_id": handle.trace_id, + "started_at": handle.started_at, + "dir": handle.dir, + } + + +def trace_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import tracing as _tracing + step_id, caused_by = _new_step_id("trace_stop") + sess = get_session() + if sess.active_trace is None: + return error_dict(Code.BAD_REQUEST, "no active trace", + step_id=step_id) + info = _tracing.stop(sess.active_trace) + sess.active_trace = None + info.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by}) + return info + + +def trace_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("trace_status") + sess = get_session() + if sess.active_trace is None: + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "active_trace_id": None, "step_count": 0, "dir": None} + h = sess.active_trace + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "active_trace_id": h.trace_id, + "step_count": h.counter.value, + "dir": h.dir, + } + + +_REPLAYS: Dict[str, Any] = {} + + +def replay_start(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import replay as _replay + step_id, caused_by = _new_step_id("replay_start") + path = args.get("path") + if not path: + return error_dict(Code.BAD_REQUEST, "path is required", + step_id=step_id) + mode = args.get("mode", "execute") + on_div = args.get("on_divergence", "warn") + try: + rep = _replay.load(path, mode=mode, on_divergence=on_div) + except Exception as e: + return error_dict(Code.BAD_REQUEST, f"could not load trace: {e}", + step_id=step_id, path=path) + handle_id = "rep:" + str(len(_REPLAYS) + 1) + _REPLAYS[handle_id] = rep + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": handle_id, + "total": len(rep.rows), + "mode": rep.mode, + "label": rep.label, + } + + +def replay_step(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import replay as _replay + step_id, caused_by = _new_step_id("replay_step") + rid = args.get("replay_id") or "" + rep = _REPLAYS.get(rid) + if rep is None: + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + + def _disp(name: str, a: Dict[str, Any]) -> Dict[str, Any]: + # Local import: tools.dispatch imports this module to build REGISTRY. + from tools.dispatch import dispatch + return dispatch(ctx, name, a) + + out = _replay.step(rep, dispatch=_disp) + out.update({"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": rid}) + return out + + +def replay_status(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("replay_status") + rid = args.get("replay_id") or "" + rep = _REPLAYS.get(rid) + if rep is None: + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "replay_id": rid, + "position": rep.position, + "total": len(rep.rows), + "finished": rep.finished, + "divergences": rep.divergences, + "mode": rep.mode, + } + + +def replay_stop(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("replay_stop") + rid = args.get("replay_id") + if rid in _REPLAYS: + _REPLAYS.pop(rid) + return {"ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "stopped": True} + return error_dict(Code.BAD_REQUEST, "unknown replay_id", + step_id=step_id, replay_id=rid) + + +def load_scenario(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import scenarios as _scn + step_id, caused_by = _new_step_id("load_scenario") + path = args.get("path") + if not path: + return error_dict(Code.BAD_REQUEST, "path is required", + step_id=step_id) + try: + sc = _scn.load(path) + _scn.attach_to_observer(sc, ctx.observer) + # The scenario replaces the mock world — cached trees are stale. + get_session().tree_cache.invalidate_all() + except _scn.ScenarioError as e: + return error_dict(Code.SCENARIO_INVALID, str(e), + step_id=step_id, path=path) + except Exception as e: + return error_dict(Code.SCENARIO_INVALID, f"{type(e).__name__}: {e}", + step_id=step_id, path=path) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "scenario": sc.name, + "state": sc.current_state, + "states": list(sc.states.keys()), + } + + +def assert_state(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + import oracles as _oracles + step_id, caused_by = _new_step_id("assert_state") + pred = args.get("predicate") or args.get("predicates") or [] + out = _oracles.evaluate(ctx.observer, pred, config=ctx.config) + if out.get("ok"): + out["step_id"] = step_id + out["caused_by_step_id"] = caused_by + return out diff --git a/tools/vision.py b/tools/vision.py new file mode 100644 index 0000000..150b933 --- /dev/null +++ b/tools/vision.py @@ -0,0 +1,301 @@ +""" +Pixel-derived observations: screenshots, OCR, descriptions. + +Split out of tools.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +from typing import Any, Dict, List, Optional, Tuple + +from errors import Code, error_dict +from observer import UIElement + +from tools.context import ( + ToolContext, _find_by_id, _focused_window, _new_step_id, _resolve_window, +) + +logger = logging.getLogger(__name__) + + +def get_screenshot(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + step_id, caused_by = _new_step_id("get_screenshot") + windows, res = _resolve_window(ctx, args) + info = res.info + hwnd = info.handle if info else None + shot = ctx.observer.get_screenshot(hwnd) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title if info else "(full screen)", + "format": "png", "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + + +def get_screenshot_cropped(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """get_screenshot with optional bbox / element_id / max_width / padding.""" + step_id, caused_by = _new_step_id("get_screenshot_cropped") + windows, res = _resolve_window(ctx, args) + info = res.info + hwnd = info.handle if info else None + shot = ctx.observer.get_screenshot(hwnd) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + + bbox: Optional[Dict[str, int]] = args.get("bbox") + element_id: Optional[str] = args.get("element_id") + padding = int(args.get("padding_px", 0)) + max_width: Optional[int] = args.get("max_width") + + if bbox is None and element_id and info is not None: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is not None: + elem = _find_by_id(tree, element_id) + if elem is not None: + # Convert to window-relative coordinates. + bbox = { + "x": max(0, elem.bounds.x - info.bounds.x), + "y": max(0, elem.bounds.y - info.bounds.y), + "width": elem.bounds.width, + "height": elem.bounds.height, + } + + if bbox or max_width: + shot, source_bbox = _apply_crop(shot, bbox, padding, max_width) + else: + source_bbox = None + + out: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title if info else "(full screen)", + "format": "png", "encoding": "base64", + "data": base64.b64encode(shot).decode(), + } + if source_bbox: + out["source_bbox"] = source_bbox + return out + + +def _apply_crop(png_bytes: bytes, bbox: Optional[Dict[str, int]], + padding: int, max_width: Optional[int] + ) -> Tuple[bytes, Optional[Dict[str, int]]]: + try: + import io as _io + from PIL import Image + except Exception: + return png_bytes, None + img: "Image.Image" = Image.open(_io.BytesIO(png_bytes)) + source_bbox: Optional[Dict[str, int]] = None + if bbox is not None: + x = max(0, int(bbox.get("x", 0)) - padding) + y = max(0, int(bbox.get("y", 0)) - padding) + x2 = min(img.width, int(bbox.get("x", 0)) + int(bbox.get("width", 0)) + padding) + y2 = min(img.height, int(bbox.get("y", 0)) + int(bbox.get("height", 0)) + padding) + if x2 > x and y2 > y: + img = img.crop((x, y, x2, y2)) + source_bbox = {"x": x, "y": y, "width": x2 - x, "height": y2 - y} + if max_width and img.width > int(max_width): + ratio = int(max_width) / float(img.width) + new_size = (int(max_width), max(1, int(img.height * ratio))) + img = img.resize(new_size) + buf = __import__("io").BytesIO() + img.save(buf, "PNG") + return buf.getvalue(), source_bbox + + +def get_ocr(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Region-scoped OCR; returns [{text, confidence, bbox}].""" + step_id, caused_by = _new_step_id("get_ocr") + try: + import io as _io + from PIL import Image + import pytesseract + from ocr_util import configure as _ocr_configure + _ocr_configure(ctx.config) + except Exception: + from ocr_util import INSTALL_HINT + return error_dict(Code.PLATFORM_UNSUPPORTED, + f"pytesseract / Pillow not installed. {INSTALL_HINT}", + step_id=step_id, hint=INSTALL_HINT) + windows, res = _resolve_window(ctx, args) + info = res.info + if info is None: + return error_dict(Code.BAD_REQUEST, + "window_uid or window_index is required", + step_id=step_id) + shot = ctx.observer.get_screenshot(info.handle) + if shot is None: + return error_dict(Code.INTERNAL, "screenshot capture failed", + step_id=step_id) + bbox = args.get("bbox") + element_id = args.get("element_id") + if element_id and not bbox: + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is not None: + elem = _find_by_id(tree, element_id) + if elem is not None: + bbox = { + "x": max(0, elem.bounds.x - info.bounds.x), + "y": max(0, elem.bounds.y - info.bounds.y), + "width": elem.bounds.width, + "height": elem.bounds.height, + } + + img: "Image.Image" = Image.open(_io.BytesIO(shot)) + if bbox: + x = max(0, int(bbox.get("x", 0))) + y = max(0, int(bbox.get("y", 0))) + x2 = min(img.width, x + int(bbox.get("width", 0))) + y2 = min(img.height, y + int(bbox.get("height", 0))) + if x2 > x and y2 > y: + img = img.crop((x, y, x2, y2)) + + min_conf = (ctx.config.get("ocr", {}) or {}).get("min_confidence", 30) + try: + data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT) + except pytesseract.TesseractNotFoundError: + from ocr_util import diagnose as _ocr_diag, INSTALL_HINT + return error_dict( + Code.PLATFORM_UNSUPPORTED, + ("tesseract binary not found — check ocr.tesseract_cmd in " + f"config.json. {INSTALL_HINT}"), + step_id=step_id, **_ocr_diag(ctx.config), + ) + except Exception as e: + return error_dict(Code.INTERNAL, f"OCR failed: {e}", + step_id=step_id) + out_words: List[Dict[str, Any]] = [] + for i, text in enumerate(data["text"]): + text = (text or "").strip() + if not text: + continue + try: + conf = int(data["conf"][i]) + except (TypeError, ValueError): + conf = 0 + if conf < min_conf: + continue + out_words.append({ + "text": text, "confidence": conf, + "bbox": {"x": int(data["left"][i]), "y": int(data["top"][i]), + "width": int(data["width"][i]), + "height": int(data["height"][i])}, + }) + return { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, "window_uid": info.window_uid, + "words": out_words, + } + + +def get_screen_description(ctx: ToolContext, args: Dict[str, Any]) -> Dict[str, Any]: + """Combined description: accessibility tree + OCR + VLM, returning every available source.""" + step_id, caused_by = _new_step_id("get_screen_description") + windows, res = _resolve_window(ctx, args) + info = res.info or _focused_window(windows) + if info is None: + return error_dict(Code.WINDOW_GONE, "no windows available", + step_id=step_id) + tree = ctx.observer.get_element_tree(info.handle, + window_uid=info.window_uid) + if tree is None: + return error_dict(Code.INTERNAL, "could not retrieve element tree", + step_id=step_id) + max_tokens = args.get("max_tokens") + focus_id = args.get("focus_element") + + sub: UIElement = tree + if focus_id: + found = _find_by_id(tree, focus_id) + if found is not None: + sub = found + + parts: Dict[str, str] = {} + + # Accessibility tree — always attempted. + try: + parts["accessibility"] = ctx.describer.from_tree(sub, info) + except Exception as e: + logger.exception("[get_screen_description] accessibility failed: %s", e) + + # OCR — attempted when enabled in config. + ocr_enabled = (ctx.config.get("ocr", {}) or {}).get("enabled", True) + if ocr_enabled: + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot: + parts["ocr"] = ctx.describer.from_ocr(shot) + else: + logger.warning("[get_screen_description] screenshot unavailable for OCR") + except Exception as e: + logger.exception("[get_screen_description] OCR failed: %s", e) + + # VLM — attempted when enabled in config. In multipass mode the VLM + # output is a structured envelope; the JSON-serialised form is folded + # into the concatenated body (for back-compat with the legacy text + # description) and the parsed dict is returned separately under + # ``vlm_structured`` so callers don't have to re-parse it. + vlm_structured: Any = None + vlm_enabled = (ctx.config.get("vlm", {}) or {}).get("enabled", False) + if vlm_enabled: + try: + shot = ctx.observer.get_screenshot(info.handle) + if shot: + vlm_mode = ( + (ctx.config.get("vlm", {}) or {}).get("mode") or "single" + ).lower() + if vlm_mode == "multipass": + env = ctx.describer.from_vlm_multipass( + shot, root=sub, window=info, + ) + if env is not None: + import json as _json + parts["vlm"] = _json.dumps(env, indent=2, + ensure_ascii=False) + vlm_structured = env + else: + vlm_out = ctx.describer.from_vlm( + shot, root=sub, window=info, + ) + if vlm_out is not None: + parts["vlm"] = vlm_out + else: + logger.warning("[get_screen_description] screenshot unavailable for VLM") + except Exception as e: + logger.exception("[get_screen_description] VLM failed: %s", e) + + body = "" + if parts: + body = "\n\n".join(f"[{k}]\n{v}" for k, v in parts.items()) + else: + body = "[no description available]" + + truncated = False + if max_tokens is not None: + char_cap = int(max_tokens) * 4 # rough chars-per-token + if len(body) > char_cap: + body = body[:char_cap] + "… [truncated]" + truncated = True + + result: Dict[str, Any] = { + "ok": True, "success": True, + "step_id": step_id, "caused_by_step_id": caused_by, + "window": info.title, "window_uid": info.window_uid, + "effective_mode": "combined", + "description": body, + "truncated": truncated, + } + if vlm_structured is not None: + result["vlm_structured"] = vlm_structured + return result diff --git a/tree_cache.py b/tree_cache.py new file mode 100644 index 0000000..eafa0fb --- /dev/null +++ b/tree_cache.py @@ -0,0 +1,180 @@ +""" +tree_cache.py — Per-window accessibility-tree cache (perception performance P1). + +Walking the accessibility tree is by far the most expensive observation +primitive (multi-second on complex Windows apps). Historically every tool +call re-walked the full tree; this module caches the most recent capture per +``window_uid`` so read-only tools within a short TTL reuse it. + +Coherence model +--------------- + - Entries expire after ``tree.cache_ttl_s`` seconds (default 2.0). + - ``tools.dispatch()`` invalidates a window's entry after any input tool + (clicks, typing, bring_to_foreground, …) executes, so post-action reads + never observe pre-action state. + - Post-action re-reads (ActionReceipt ``after`` captures) bypass the cache + explicitly via ``use_cache=False``. + +The cache also remembers lightweight per-window last-capture statistics +(node counts) which survive invalidation; ``get_capabilities`` surfaces them +so agents can detect accessibility-dark windows. +""" + +from __future__ import annotations + +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any, Dict, Optional + +# Defaults (overridable via config / constructor). +DEFAULT_TTL_S = 2.0 +DEFAULT_MAX_WINDOWS = 8 +_MAX_STATS = 64 # per-window last-capture stats retained + + +@dataclass +class TreeCacheEntry: + window_uid: str + tree: Any # UIElement root of the capture + serialized: Dict[str, Any] # tree.to_dict() of the same capture + tree_hash: str + captured_at: float + max_depth: int # depth cap in effect when captured + capture_ms: int = 0 + node_count: int = 0 + named_node_count: int = 0 + + def age_s(self, now: Optional[float] = None) -> float: + return (now if now is not None else time.time()) - self.captured_at + + +class TreeCache: + """Thread-safe TTL + LRU cache of per-window accessibility trees.""" + + def __init__(self, ttl_s: float = DEFAULT_TTL_S, + max_windows: int = DEFAULT_MAX_WINDOWS) -> None: + self.ttl_s = float(ttl_s) + self.max_windows = int(max_windows) + self._lock = threading.RLock() + self._entries: "OrderedDict[str, TreeCacheEntry]" = OrderedDict() + # Last-capture stats per window; kept across invalidation so + # capability reporting works even when the cache is cold. + self._stats: "OrderedDict[str, Dict[str, Any]]" = OrderedDict() + # Telemetry counters (P2): cache effectiveness + capture latency. + self._hits = 0 + self._misses = 0 + self._capture_count = 0 + self._capture_ms_total = 0 + self._capture_ms_max = 0 + + # ── Read ───────────────────────────────────────────────────────────────── + + def get(self, window_uid: str, + ttl_s: Optional[float] = None) -> Optional[TreeCacheEntry]: + """Return a fresh entry for *window_uid*, or None (expired entries + are dropped). *ttl_s* overrides the instance TTL for this lookup.""" + limit = self.ttl_s if ttl_s is None else float(ttl_s) + with self._lock: + entry = self._entries.get(window_uid) + if entry is None: + self._misses += 1 + return None + if entry.age_s() > limit: + self._entries.pop(window_uid, None) + self._misses += 1 + return None + self._entries.move_to_end(window_uid) + self._hits += 1 + return entry + + def peek(self, window_uid: str) -> Optional[TreeCacheEntry]: + """Return the last capture regardless of TTL (no LRU touch). + Used as the baseline for changed_only comparisons.""" + with self._lock: + return self._entries.get(window_uid) + + # ── Write ──────────────────────────────────────────────────────────────── + + def put(self, window_uid: str, *, tree: Any, serialized: Dict[str, Any], + tree_hash: str, max_depth: int, capture_ms: int = 0, + node_count: int = 0, named_node_count: int = 0) -> TreeCacheEntry: + entry = TreeCacheEntry( + window_uid=window_uid, tree=tree, serialized=serialized, + tree_hash=tree_hash, captured_at=time.time(), + max_depth=int(max_depth), capture_ms=int(capture_ms), + node_count=int(node_count), + named_node_count=int(named_node_count), + ) + with self._lock: + self._capture_count += 1 + self._capture_ms_total += entry.capture_ms + if entry.capture_ms > self._capture_ms_max: + self._capture_ms_max = entry.capture_ms + self._entries[window_uid] = entry + self._entries.move_to_end(window_uid) + while len(self._entries) > self.max_windows: + self._entries.popitem(last=False) + self._stats[window_uid] = { + "captured_at": entry.captured_at, + "capture_ms": entry.capture_ms, + "node_count": entry.node_count, + "named_node_count": entry.named_node_count, + } + self._stats.move_to_end(window_uid) + while len(self._stats) > _MAX_STATS: + self._stats.popitem(last=False) + return entry + + # ── Invalidation ───────────────────────────────────────────────────────── + + def invalidate(self, window_uid: str) -> bool: + """Drop the entry for one window. Returns True when one existed.""" + with self._lock: + return self._entries.pop(window_uid, None) is not None + + def invalidate_all(self) -> None: + with self._lock: + self._entries.clear() + + # ── Introspection ──────────────────────────────────────────────────────── + + def __len__(self) -> int: + with self._lock: + return len(self._entries) + + def __contains__(self, window_uid: str) -> bool: + with self._lock: + return window_uid in self._entries + + def stats(self) -> Dict[str, Dict[str, Any]]: + """Per-window last-capture statistics (survive invalidation).""" + with self._lock: + return {uid: dict(s) for uid, s in self._stats.items()} + + def counters(self) -> Dict[str, Any]: + """Session-cumulative telemetry: cache hit/miss counts and a + capture-latency summary (count / total / max / mean ms). Surfaced + by /api/healthz and /api/metrics.""" + with self._lock: + mean = (self._capture_ms_total / self._capture_count + if self._capture_count else 0.0) + return { + "hits": self._hits, + "misses": self._misses, + "entries": len(self._entries), + "capture_count": self._capture_count, + "capture_ms_total": self._capture_ms_total, + "capture_ms_max": self._capture_ms_max, + "capture_ms_mean": round(mean, 2), + } + + +# Convenience default factory used by session.Session. +def default_tree_cache() -> TreeCache: + return TreeCache() + + +__all__ = ["TreeCache", "TreeCacheEntry", "default_tree_cache", + "DEFAULT_TTL_S", "DEFAULT_MAX_WINDOWS"] diff --git a/web_inspector/__init__.py b/web_inspector/__init__.py new file mode 100644 index 0000000..c2c0abf --- /dev/null +++ b/web_inspector/__init__.py @@ -0,0 +1,32 @@ +""" +web_inspector — Human-facing inspection interface (package form of the +former web_inspector.py). + +Exposes a Flask HTTP server on localhost:5001 (configurable) with: + + GET / — Single-page inspection UI + GET /api/windows — List windows + GET /api/structure — Accessibility tree JSON + GET /api/description — Textual description (combined: accessibility + OCR + VLM) + GET /api/sketch — ASCII layout sketch + GET /api/screenshot — Screenshot as base64 PNG + POST /api/action — Execute an input action + +The HTML/CSS/JS is inlined as a template string (assets.py) so the entire +server is importable with no external static files. + +CORS policy: no CORS headers are emitted unless web_ui.cors_origins is set +in config (default [] = same-origin only). See create_web_app(). + +P3 decomposition: the implementation now lives in submodules (assets, +views, server). This __init__ re-exports the pre-split surface so +`from web_inspector import create_web_app` keeps working unchanged. +""" + +from __future__ import annotations + +from web_inspector.assets import _HTML +from web_inspector.server import create_web_app +from web_inspector.views import register_routes + +__all__ = ["_HTML", "create_web_app", "register_routes"] diff --git a/web_inspector.py b/web_inspector/assets.py old mode 100755 new mode 100644 similarity index 70% rename from web_inspector.py rename to web_inspector/assets.py index caaa872..42a796e --- a/web_inspector.py +++ b/web_inspector/assets.py @@ -1,40 +1,10 @@ """ -web_inspector.py — Human-facing inspection interface. +Inline single-page inspection UI (HTML/CSS/JS template). -Exposes a Flask HTTP server on localhost:5001 (configurable) with: - - GET / — Single-page inspection UI - GET /api/windows — List windows - GET /api/structure — Accessibility tree JSON - GET /api/description — Textual description (combined: accessibility + OCR + VLM) - GET /api/sketch — ASCII layout sketch - GET /api/screenshot — Screenshot as base64 PNG - POST /api/action — Execute an input action - -The HTML/CSS/JS is inlined as a template string so the entire server is a -single importable Python module with no external static files. +Split out of web_inspector.py (P3); behavior is unchanged. """ -import base64 -import logging -import traceback -from typing import Optional - -from flask import Flask, jsonify, request -from flask_cors import CORS - -from ascii_renderer import ASCIIRenderer -from description import DescriptionGenerator -from observer import ScreenObserver -import tools as _tools -from errors import http_status_for - -logger = logging.getLogger(__name__) - - -# ───────────────────────────────────────────────────────────────────────────── -# HTML template (single-page application, no external dependencies) -# ───────────────────────────────────────────────────────────────────────────── +from __future__ import annotations _HTML = r""" @@ -1168,546 +1138,3 @@ """ - - -# ───────────────────────────────────────────────────────────────────────────── -# Flask application factory -# ───────────────────────────────────────────────────────────────────────────── - -def create_web_app( - observer: ScreenObserver, - renderer: ASCIIRenderer, - describer: DescriptionGenerator, - config: dict, -) -> Flask: - """ - Create and configure the Flask inspection application. - - All routes are defined inside this factory so they close over the - shared observer/renderer/describer instances. - """ - app = Flask(__name__) - CORS(app) - - ctx = _tools.ToolContext(observer=observer, renderer=renderer, - describer=describer, config=config) - - def _tool_response(name: str, args: dict): - result = _tools.dispatch(ctx, name, args) - if not result.get("ok", True): - code = (result.get("error") or {}).get("code", "Internal") - return jsonify(result), http_status_for(code) - return jsonify(result) - - def _merge_query(extra: Optional[dict] = None) -> dict: - out = {k: v for k, v in request.args.items()} - if "window_index" in out: - try: - out["window_index"] = int(out["window_index"]) - except (TypeError, ValueError): - pass - if extra: - out.update(extra) - return out - - # ── UI ──────────────────────────────────────────────────────────────────── - - @app.route("/") - def index(): - return _HTML, 200, {"Content-Type": "text/html; charset=utf-8"} - - # ── API helpers ─────────────────────────────────────────────────────────── - - def _window_from_args(): - """Resolve window_uid, window_index, or window_title → (WindowInfo, hwnd, windows). - - Priority: window_uid > window_index > window_title (substring match). - """ - windows = observer.list_windows() - res = observer.resolve_window( - windows, - window_uid=request.args.get("window_uid"), - window_index=int(request.args["window_index"]) if "window_index" in request.args else None, - window_title=request.args.get("window_title"), - ) - info = res.info - hwnd = info.handle if info else None - return info, hwnd, windows - - # ── /api/windows ────────────────────────────────────────────────────────── - - @app.route("/api/windows") - def api_windows(): - return _tool_response("list_windows", {}) - - # ── /api/structure ──────────────────────────────────────────────────────── - - @app.route("/api/structure") - def api_structure(): - # Forwards through tools.dispatch so callers can use roles=, - # name_regex=, prune_empty=, max_nodes=, page_cursor= filters. - args = _merge_query() - for key in ("roles", "exclude_roles"): - if key in args and isinstance(args[key], str): - args[key] = [s for s in args[key].split(",") if s] - for bool_key in ("visible_only", "prune_empty"): - if bool_key in args: - args[bool_key] = str(args[bool_key]).lower() in ("1", "true", "yes") - for int_key in ("max_text_len", "max_nodes"): - if int_key in args: - try: - args[int_key] = int(args[int_key]) - except (TypeError, ValueError): - args.pop(int_key, None) - return _tool_response("get_window_structure", args) - - # ── /api/description ────────────────────────────────────────────────────── - - @app.route("/api/description") - def api_description(): - args = _merge_query() - if "max_tokens" in args: - try: - args["max_tokens"] = int(args["max_tokens"]) - except (TypeError, ValueError): - args.pop("max_tokens", None) - return _tool_response("get_screen_description", args) - - # ── /api/sketch ─────────────────────────────────────────────────────────── - - @app.route("/api/sketch") - def api_sketch(): - try: - info, hwnd, _ = _window_from_args() - tree = observer.get_element_tree(hwnd) - if tree is None: - return jsonify({"error": "Could not retrieve element tree"}), 500 - - gw = request.args.get("grid_width", type=int) - gh = request.args.get("grid_height", type=int) - ref = info.bounds if info else tree.bounds - - # Optional OCR overlay: pass ?ocr=1 to enable Tesseract text overlay. - # Requires pytesseract + tesseract on PATH; silently skipped otherwise. - shot_bytes: Optional[bytes] = None - if request.args.get("ocr", "").strip() in ("1", "true", "yes"): - shot_bytes = observer.get_screenshot(hwnd) - - want_structured = request.args.get("structured", "").strip() in ( - "1", "true", "yes", - ) - result = renderer.render_structured( - root = tree, - screen_bounds = ref, - grid_width = gw, - grid_height = gh, - screenshot_bytes = shot_bytes, - ) - payload = { - "window": info.title if info else "(focused)", - "grid_width": gw or renderer.default_width, - "grid_height": gh or renderer.default_height, - "ocr_overlay": shot_bytes is not None, - "sketch": result["sketch"], - } - if want_structured: - payload["elements"] = result["elements"] - payload["legend"] = result["legend"] - return jsonify(payload) - except Exception as e: - print(f"[web_inspector:/api/sketch] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/screenshot ─────────────────────────────────────────────────────── - - @app.route("/api/screenshot") - def api_screenshot(): - try: - info, hwnd, _ = _window_from_args() - shot = observer.get_screenshot(hwnd) - if shot is None: - return jsonify({"error": "Screenshot capture failed"}), 500 - return jsonify({ - "window": info.title if info else "(full screen)", - "format": "png", - "encoding": "base64", - "data": base64.b64encode(shot).decode(), - }) - except Exception as e: - print(f"[web_inspector:/api/screenshot] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/full_screenshot ────────────────────────────────────────────────── - - @app.route("/api/full_screenshot") - def api_full_screenshot(): - """All-monitor screenshot + optional ASCII sketch in one call.""" - try: - info, hwnd, _ = _window_from_args() - # Always capture the full virtual desktop (all monitors combined) - shot = observer.get_full_display_screenshot() - if shot is None: - return jsonify({"error": "Screenshot capture failed"}), 500 - - sketch: Optional[str] = None - tree = observer.get_element_tree(hwnd) if hwnd is not None else None - if tree is not None: - gw = request.args.get("grid_width", type=int) - gh = request.args.get("grid_height", type=int) - ref = info.bounds if info else observer.get_screen_bounds() - # Crop the full-display PNG to the window's bounds so that OCR - # word coordinates (which are window-relative in ascii_renderer) - # align correctly with the sketch grid. - ocr_bytes = shot - if info is not None: - try: - import io as _io2 - from PIL import Image as _Image2 - full_img = _Image2.open(_io2.BytesIO(shot)) - screen_b = observer.get_screen_bounds() - crop_box = ( - info.bounds.x - screen_b.x, - info.bounds.y - screen_b.y, - info.bounds.right - screen_b.x, - info.bounds.bottom - screen_b.y, - ) - crop_box = ( - max(0, crop_box[0]), - max(0, crop_box[1]), - min(full_img.width, crop_box[2]), - min(full_img.height, crop_box[3]), - ) - buf2 = _io2.BytesIO() - full_img.crop(crop_box).save(buf2, format="PNG") - ocr_bytes = buf2.getvalue() - except Exception: - pass - sketch = renderer.render( - root = tree, - screen_bounds = ref, - grid_width = gw, - grid_height = gh, - screenshot_bytes = ocr_bytes, - ) - - try: - import io as _io - from PIL import Image as _Image - _img = _Image.open(_io.BytesIO(shot)) - img_w, img_h = _img.size - except Exception: - img_w = img_h = None - - return jsonify({ - "window": info.title if info else "(full screen)", - "screenshot_scope": "full_display", - "format": "png", - "encoding": "base64", - "width": img_w, - "height": img_h, - "data": base64.b64encode(shot).decode(), - "sketch": sketch, - }) - except Exception as e: - print(f"[web_inspector:/api/full_screenshot] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/visible_areas ──────────────────────────────────────────────────── - - @app.route("/api/visible_areas") - def api_visible_areas(): - """Visible (non-occluded, on-screen) bounding boxes for a window.""" - try: - info, hwnd, windows = _window_from_args() - if hwnd is None: - return jsonify({"error": "window_uid or window_index is required"}), 400 - areas = observer.get_visible_areas(hwnd, windows) - return jsonify({ - "window": info.title if info else "(unknown)", - "visible_regions": areas, - }) - except Exception as e: - print(f"[web_inspector:/api/visible_areas] {e}") - traceback.print_exc() - return jsonify({"error": str(e)}), 500 - - # ── /api/bring_to_foreground ────────────────────────────────────────────── - - @app.route("/api/bring_to_foreground") - def api_bring_to_foreground(): - """Click the title bar of a window to bring it to the foreground.""" - try: - info, hwnd, windows = _window_from_args() - if hwnd is None: - return jsonify({"success": False, - "error": "window_uid or window_index is required"}), 400 - result = observer.bring_to_foreground(hwnd, windows) - result["window"] = info.title if info else "(unknown)" - result["window_uid"] = info.window_uid if info else None - return jsonify(result) - except Exception as e: - print(f"[web_inspector:/api/bring_to_foreground] {e}") - traceback.print_exc() - return jsonify({"success": False, "error": str(e)}), 500 - - # ── /api/action ─────────────────────────────────────────────────────────── - - @app.route("/api/action", methods=["POST"]) - def api_action(): - body = request.get_json(force=True) or {} - action = body.get("action", "") - if action == "click_at": - return _tool_response("click_at", { - "x": body.get("x", 0), "y": body.get("y", 0), - "button": body.get("button", "left"), - "double": body.get("double", False), - }) - if action == "type": - return _tool_response("type_text", {"text": body.get("value", "")}) - if action == "key": - return _tool_response("press_key", {"keys": body.get("value", "")}) - if action == "scroll": - return _tool_response("scroll", body) - return jsonify({"success": False, "ok": False, - "error": f"Unknown action: {action}"}), 400 - - # ── P1: identity, capabilities, element-targeted actions ───────────────── - - @app.route("/api/capabilities") - def api_capabilities(): - return _tool_response("get_capabilities", {}) - - @app.route("/api/monitors") - def api_monitors(): - return _tool_response("get_monitors", {}) - - @app.route("/api/find_element") - def api_find_element(): - return _tool_response("find_element", _merge_query()) - - @app.route("/api/element/click", methods=["POST"]) - def api_element_click(): - return _tool_response("click_element", request.get_json(force=True) or {}) - - @app.route("/api/element/focus", methods=["POST"]) - def api_element_focus(): - return _tool_response("focus_element", request.get_json(force=True) or {}) - - @app.route("/api/element/set_value", methods=["POST"]) - def api_element_set_value(): - return _tool_response("set_value", request.get_json(force=True) or {}) - - @app.route("/api/element/invoke", methods=["POST"]) - def api_element_invoke(): - return _tool_response("invoke_element", request.get_json(force=True) or {}) - - @app.route("/api/element/select", methods=["POST"]) - def api_element_select(): - return _tool_response("select_option", request.get_json(force=True) or {}) - - # ── P2: observe-with-diff, snapshots, wait_for ────────────────────────── - - @app.route("/api/observe") - def api_observe(): - return _tool_response("observe_window", _merge_query()) - - @app.route("/api/snapshot", methods=["POST"]) - def api_snapshot(): - return _tool_response("snapshot", request.get_json(silent=True) or {}) - - @app.route("/api/snapshot/") - def api_snapshot_get(sid: str): - return _tool_response("snapshot_get", {"snapshot_id": sid}) - - @app.route("/api/snapshot/diff", methods=["POST"]) - def api_snapshot_diff(): - return _tool_response("snapshot_diff", request.get_json(force=True) or {}) - - @app.route("/api/snapshot/", methods=["DELETE"]) - def api_snapshot_drop(sid: str): - return _tool_response("snapshot_drop", {"snapshot_id": sid}) - - @app.route("/api/wait_for", methods=["POST"]) - def api_wait_for(): - return _tool_response("wait_for", request.get_json(force=True) or {}) - - @app.route("/api/wait_idle", methods=["POST"]) - def api_wait_idle(): - return _tool_response("wait_idle", request.get_json(force=True) or {}) - - @app.route("/api/element/click_and_observe", methods=["POST"]) - def api_element_click_observe(): - return _tool_response("click_element_and_observe", - request.get_json(force=True) or {}) - - @app.route("/api/type_and_observe", methods=["POST"]) - def api_type_observe(): - return _tool_response("type_and_observe", request.get_json(force=True) or {}) - - @app.route("/api/key_and_observe", methods=["POST"]) - def api_key_observe(): - return _tool_response("press_key_and_observe", request.get_json(force=True) or {}) - - # ── P3: filtering, cropping, region OCR, budgeted description ─────────── - - @app.route("/api/screenshot/cropped") - def api_screenshot_cropped(): - return _tool_response("get_screenshot_cropped", _merge_query()) - - @app.route("/api/ocr") - def api_ocr(): - return _tool_response("get_ocr", _merge_query()) - - # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── - - @app.route("/api/trace/start", methods=["POST"]) - def api_trace_start(): - return _tool_response("trace_start", request.get_json(silent=True) or {}) - - @app.route("/api/trace/stop", methods=["POST"]) - def api_trace_stop(): - return _tool_response("trace_stop", {}) - - @app.route("/api/trace/status") - def api_trace_status(): - return _tool_response("trace_status", {}) - - @app.route("/api/replay/start", methods=["POST"]) - def api_replay_start(): - return _tool_response("replay_start", request.get_json(force=True) or {}) - - @app.route("/api/replay/step", methods=["POST"]) - def api_replay_step(): - return _tool_response("replay_step", request.get_json(force=True) or {}) - - @app.route("/api/replay/status", methods=["POST"]) - def api_replay_status(): - return _tool_response("replay_status", request.get_json(force=True) or {}) - - @app.route("/api/replay/stop", methods=["POST"]) - def api_replay_stop(): - return _tool_response("replay_stop", request.get_json(force=True) or {}) - - @app.route("/api/scenario/load", methods=["POST"]) - def api_scenario_load(): - return _tool_response("load_scenario", request.get_json(force=True) or {}) - - @app.route("/api/assert_state", methods=["POST"]) - def api_assert_state(): - return _tool_response("assert_state", request.get_json(force=True) or {}) - - # ── P5: budgets / redaction status / propose ──────────────────────────── - - @app.route("/api/budget_status") - def api_budget_status(): - return _tool_response("get_budget_status", {}) - - @app.route("/api/redaction_status") - def api_redaction_status(): - return _tool_response("get_redaction_status", {}) - - @app.route("/api/propose_action", methods=["POST"]) - def api_propose(): - return _tool_response("propose_action", request.get_json(force=True) or {}) - - # ── P6: extra input verbs ──────────────────────────────────────────────── - - @app.route("/api/hover", methods=["POST"]) - def api_hover(): - body = request.get_json(force=True) or {} - if "x" in body and "y" in body and not body.get("selector") and not body.get("element_id"): - return _tool_response("hover_at", body) - return _tool_response("hover_element", body) - - @app.route("/api/element/right_click", methods=["POST"]) - def api_right_click(): - return _tool_response("right_click_element", request.get_json(force=True) or {}) - - @app.route("/api/element/double_click", methods=["POST"]) - def api_double_click(): - return _tool_response("double_click_element", request.get_json(force=True) or {}) - - @app.route("/api/drag", methods=["POST"]) - def api_drag(): - return _tool_response("drag", request.get_json(force=True) or {}) - - @app.route("/api/element/key", methods=["POST"]) - def api_key_into(): - return _tool_response("key_into_element", request.get_json(force=True) or {}) - - @app.route("/api/element/clear_text", methods=["POST"]) - def api_clear_text(): - return _tool_response("clear_text", request.get_json(force=True) or {}) - - # ── Telemetry: metrics in Prometheus format ───────────────────────────── - - @app.route("/api/metrics") - def api_metrics(): - from session import get_session - s = get_session() - lines = [ - "# HELP oso_step_count Total tool calls processed", - "# TYPE oso_step_count counter", - f"oso_step_count {s.steps.count}", - "# HELP oso_uptime_seconds Process uptime", - "# TYPE oso_uptime_seconds gauge", - f"oso_uptime_seconds {int(s.steps.uptime_s)}", - ] - if s.budgets is not None: - st = s.budgets.status() - lines += [ - "# TYPE oso_actions_used counter", - f"oso_actions_used {st['actions']['used']}", - "# TYPE oso_screenshots_used counter", - f"oso_screenshots_used {st['screenshots']['used']}", - ] - if s.active_trace is not None: - lines.append("oso_active_trace 1") - else: - lines.append("oso_active_trace 0") - body = "\n".join(lines) + "\n" - return body, 200, {"Content-Type": "text/plain; version=0.0.4"} - - # ── Generic tool console ──────────────────────────────────────────────── - - @app.route("/api/tools") - def api_tools_list(): - return jsonify({"ok": True, - "tools": sorted(_tools.REGISTRY.keys())}) - - @app.route("/api/tool/", methods=["GET", "POST"]) - def api_tool_run(name: str): - if request.method == "POST": - args = request.get_json(silent=True) or {} - else: - args = _merge_query() - return _tool_response(name, args) - - @app.route("/api/healthz") - def api_healthz(): - from session import get_session - s = get_session() - out = { - "ok": True, - "uptime_s": int(s.steps.uptime_s), - "step_count": s.steps.count, - "adapter": type(observer._adapter).__name__, - "version": (config.get("mcp", {}) or {}).get("version", "0.2.0"), - } - # Surface common misconfigurations. - try: - from main import config_load_status - out.update(config_load_status()) - except Exception: - pass - try: - from ocr_util import diagnose as _ocr_diag - out["ocr"] = _ocr_diag(config) - except Exception: - pass - return jsonify(out) - - return app diff --git a/web_inspector/server.py b/web_inspector/server.py new file mode 100644 index 0000000..fb9b3b9 --- /dev/null +++ b/web_inspector/server.py @@ -0,0 +1,55 @@ +""" +Flask application factory for the inspection server. + +Split out of web_inspector.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import logging + +from flask import Flask +from flask_cors import CORS + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from observer import ScreenObserver +import tools as _tools + +from web_inspector.views import register_routes + +logger = logging.getLogger(__name__) + + +def create_web_app( + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: dict, +) -> Flask: + """ + Create and configure the Flask inspection application. + + All routes are defined inside this factory so they close over the + shared observer/renderer/describer instances. + """ + app = Flask(__name__) + + # CORS is opt-in (web_ui.cors_origins, default []). With the default no + # Access-Control-Allow-Origin header is ever sent, so browsers enforce + # same-origin — the bundled inspector UI at "/" is served same-origin and + # keeps working. Operators can list explicit origins, or ["*"] for + # Docker/testing scenarios where cross-origin dashboards need access. + # Never enable "*" together with a non-loopback bind outside an isolated + # environment: /api/action is unauthenticated and destructive. + cors_origins = list((config.get("web_ui") or {}).get("cors_origins") or []) + if cors_origins: + CORS(app, origins=cors_origins) + logger.warning(f"CORS enabled for origins: {cors_origins}") + + ctx = _tools.ToolContext(observer=observer, renderer=renderer, + describer=describer, config=config) + + register_routes(app, observer=observer, renderer=renderer, + describer=describer, config=config, ctx=ctx) + return app diff --git a/web_inspector/views.py b/web_inspector/views.py new file mode 100644 index 0000000..b2dda9c --- /dev/null +++ b/web_inspector/views.py @@ -0,0 +1,598 @@ +""" +Flask route registration for the inspection API + UI. + +Split out of web_inspector.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import base64 +import logging +import traceback +from typing import Optional + +from flask import Flask, jsonify, request + +from ascii_renderer import ASCIIRenderer +from description import DescriptionGenerator +from errors import http_status_for +from observer import ScreenObserver +import tools as _tools + +from web_inspector.assets import _HTML + +logger = logging.getLogger(__name__) + + +def register_routes( + app: Flask, + *, + observer: ScreenObserver, + renderer: ASCIIRenderer, + describer: DescriptionGenerator, + config: dict, + ctx: _tools.ToolContext, +) -> None: + """Define all routes on *app*, closing over the shared + observer/renderer/describer instances (moved verbatim out of + create_web_app).""" + def _tool_response(name: str, args: dict): + result = _tools.dispatch(ctx, name, args) + if not result.get("ok", True): + code = (result.get("error") or {}).get("code", "Internal") + return jsonify(result), http_status_for(code) + return jsonify(result) + + def _merge_query(extra: Optional[dict] = None) -> dict: + out: dict = {k: v for k, v in request.args.items()} + if "window_index" in out: + try: + out["window_index"] = int(out["window_index"]) + except (TypeError, ValueError): + pass + if extra: + out.update(extra) + return out + + # ── UI ──────────────────────────────────────────────────────────────────── + + @app.route("/") + def index(): + return _HTML, 200, {"Content-Type": "text/html; charset=utf-8"} + + # ── API helpers ─────────────────────────────────────────────────────────── + + def _window_from_args(): + """Resolve window_uid, window_index, or window_title → (WindowInfo, hwnd, windows). + + Priority: window_uid > window_index > window_title (substring match). + """ + windows = observer.list_windows() + res = observer.resolve_window( + windows, + window_uid=request.args.get("window_uid"), + window_index=int(request.args["window_index"]) if "window_index" in request.args else None, + window_title=request.args.get("window_title"), + ) + info = res.info + hwnd = info.handle if info else None + return info, hwnd, windows + + # ── /api/windows ────────────────────────────────────────────────────────── + + @app.route("/api/windows") + def api_windows(): + return _tool_response("list_windows", {}) + + # ── /api/structure ──────────────────────────────────────────────────────── + + @app.route("/api/structure") + def api_structure(): + # Forwards through tools.dispatch so callers can use roles=, + # name_regex=, prune_empty=, max_nodes=, page_cursor= filters. + args = _merge_query() + for key in ("roles", "exclude_roles"): + if key in args and isinstance(args[key], str): + args[key] = [s for s in args[key].split(",") if s] + for bool_key in ("visible_only", "prune_empty"): + if bool_key in args: + args[bool_key] = str(args[bool_key]).lower() in ("1", "true", "yes") + for int_key in ("max_text_len", "max_nodes", "depth"): + if int_key in args: + try: + args[int_key] = int(args[int_key]) + except (TypeError, ValueError): + args.pop(int_key, None) + return _tool_response("get_window_structure", args) + + # ── /api/description ────────────────────────────────────────────────────── + + @app.route("/api/description") + def api_description(): + args = _merge_query() + if "max_tokens" in args: + try: + args["max_tokens"] = int(args["max_tokens"]) + except (TypeError, ValueError): + args.pop("max_tokens", None) + return _tool_response("get_screen_description", args) + + # ── /api/sketch ─────────────────────────────────────────────────────────── + + @app.route("/api/sketch") + def api_sketch(): + try: + info, hwnd, _ = _window_from_args() + tree = observer.get_element_tree(hwnd) + if tree is None: + return jsonify({"error": "Could not retrieve element tree"}), 500 + + gw = request.args.get("grid_width", type=int) + gh = request.args.get("grid_height", type=int) + ref = info.bounds if info else tree.bounds + + # Optional OCR overlay: pass ?ocr=1 to enable Tesseract text overlay. + # Requires pytesseract + tesseract on PATH; silently skipped otherwise. + shot_bytes: Optional[bytes] = None + if request.args.get("ocr", "").strip() in ("1", "true", "yes"): + shot_bytes = observer.get_screenshot(hwnd) + + want_structured = request.args.get("structured", "").strip() in ( + "1", "true", "yes", + ) + result = renderer.render_structured( + root = tree, + screen_bounds = ref, + grid_width = gw, + grid_height = gh, + screenshot_bytes = shot_bytes, + ) + payload = { + "window": info.title if info else "(focused)", + "grid_width": gw or renderer.default_width, + "grid_height": gh or renderer.default_height, + "ocr_overlay": shot_bytes is not None, + "sketch": result["sketch"], + } + if want_structured: + payload["elements"] = result["elements"] + payload["legend"] = result["legend"] + return jsonify(payload) + except Exception as e: + print(f"[web_inspector:/api/sketch] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/screenshot ─────────────────────────────────────────────────────── + + @app.route("/api/screenshot") + def api_screenshot(): + try: + info, hwnd, _ = _window_from_args() + shot = observer.get_screenshot(hwnd) + if shot is None: + return jsonify({"error": "Screenshot capture failed"}), 500 + return jsonify({ + "window": info.title if info else "(full screen)", + "format": "png", + "encoding": "base64", + "data": base64.b64encode(shot).decode(), + }) + except Exception as e: + print(f"[web_inspector:/api/screenshot] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/full_screenshot ────────────────────────────────────────────────── + + @app.route("/api/full_screenshot") + def api_full_screenshot(): + """All-monitor screenshot + optional ASCII sketch in one call.""" + try: + info, hwnd, _ = _window_from_args() + # Always capture the full virtual desktop (all monitors combined) + shot = observer.get_full_display_screenshot() + if shot is None: + return jsonify({"error": "Screenshot capture failed"}), 500 + + sketch: Optional[str] = None + tree = observer.get_element_tree(hwnd) if hwnd is not None else None + if tree is not None: + gw = request.args.get("grid_width", type=int) + gh = request.args.get("grid_height", type=int) + ref = info.bounds if info else observer.get_screen_bounds() + # Crop the full-display PNG to the window's bounds so that OCR + # word coordinates (which are window-relative in ascii_renderer) + # align correctly with the sketch grid. + ocr_bytes = shot + if info is not None: + try: + import io as _io2 + from PIL import Image as _Image2 + full_img = _Image2.open(_io2.BytesIO(shot)) + screen_b = observer.get_screen_bounds() + crop_box = ( + info.bounds.x - screen_b.x, + info.bounds.y - screen_b.y, + info.bounds.right - screen_b.x, + info.bounds.bottom - screen_b.y, + ) + crop_box = ( + max(0, crop_box[0]), + max(0, crop_box[1]), + min(full_img.width, crop_box[2]), + min(full_img.height, crop_box[3]), + ) + buf2 = _io2.BytesIO() + full_img.crop(crop_box).save(buf2, format="PNG") + ocr_bytes = buf2.getvalue() + except Exception as e: + logger.debug( + f"[/api/full_screenshot] window crop for OCR " + f"overlay failed; using full image: {e}") + sketch = renderer.render( + root = tree, + screen_bounds = ref, + grid_width = gw, + grid_height = gh, + screenshot_bytes = ocr_bytes, + ) + + try: + import io as _io + from PIL import Image as _Image + _img = _Image.open(_io.BytesIO(shot)) + img_w, img_h = _img.size + except Exception: + img_w = img_h = None + + return jsonify({ + "window": info.title if info else "(full screen)", + "screenshot_scope": "full_display", + "format": "png", + "encoding": "base64", + "width": img_w, + "height": img_h, + "data": base64.b64encode(shot).decode(), + "sketch": sketch, + }) + except Exception as e: + print(f"[web_inspector:/api/full_screenshot] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/visible_areas ──────────────────────────────────────────────────── + + @app.route("/api/visible_areas") + def api_visible_areas(): + """Visible (non-occluded, on-screen) bounding boxes for a window.""" + try: + info, hwnd, windows = _window_from_args() + if hwnd is None: + return jsonify({"error": "window_uid or window_index is required"}), 400 + areas = observer.get_visible_areas(hwnd, windows) + return jsonify({ + "window": info.title if info else "(unknown)", + "visible_regions": areas, + }) + except Exception as e: + print(f"[web_inspector:/api/visible_areas] {e}") + traceback.print_exc() + return jsonify({"error": str(e)}), 500 + + # ── /api/bring_to_foreground ────────────────────────────────────────────── + + @app.route("/api/bring_to_foreground") + def api_bring_to_foreground(): + """Click the title bar of a window to bring it to the foreground.""" + try: + info, hwnd, windows = _window_from_args() + if hwnd is None: + return jsonify({"success": False, + "error": "window_uid or window_index is required"}), 400 + result = observer.bring_to_foreground(hwnd, windows) + result["window"] = info.title if info else "(unknown)" + result["window_uid"] = info.window_uid if info else None + return jsonify(result) + except Exception as e: + print(f"[web_inspector:/api/bring_to_foreground] {e}") + traceback.print_exc() + return jsonify({"success": False, "error": str(e)}), 500 + + # ── /api/action ─────────────────────────────────────────────────────────── + + @app.route("/api/action", methods=["POST"]) + def api_action(): + body = request.get_json(force=True) or {} + action = body.get("action", "") + if action == "click_at": + return _tool_response("click_at", { + "x": body.get("x", 0), "y": body.get("y", 0), + "button": body.get("button", "left"), + "double": body.get("double", False), + }) + if action == "type": + return _tool_response("type_text", {"text": body.get("value", "")}) + if action == "key": + return _tool_response("press_key", {"keys": body.get("value", "")}) + if action == "scroll": + return _tool_response("scroll", body) + return jsonify({"success": False, "ok": False, + "error": f"Unknown action: {action}"}), 400 + + # ── P1: identity, capabilities, element-targeted actions ───────────────── + + @app.route("/api/capabilities") + def api_capabilities(): + return _tool_response("get_capabilities", {}) + + @app.route("/api/monitors") + def api_monitors(): + return _tool_response("get_monitors", {}) + + @app.route("/api/find_element") + def api_find_element(): + return _tool_response("find_element", _merge_query()) + + @app.route("/api/element/click", methods=["POST"]) + def api_element_click(): + return _tool_response("click_element", request.get_json(force=True) or {}) + + @app.route("/api/element/focus", methods=["POST"]) + def api_element_focus(): + return _tool_response("focus_element", request.get_json(force=True) or {}) + + @app.route("/api/element/set_value", methods=["POST"]) + def api_element_set_value(): + return _tool_response("set_value", request.get_json(force=True) or {}) + + @app.route("/api/element/invoke", methods=["POST"]) + def api_element_invoke(): + return _tool_response("invoke_element", request.get_json(force=True) or {}) + + @app.route("/api/element/select", methods=["POST"]) + def api_element_select(): + return _tool_response("select_option", request.get_json(force=True) or {}) + + # ── P2: observe-with-diff, snapshots, wait_for ────────────────────────── + + @app.route("/api/observe") + def api_observe(): + args = _merge_query() + if "depth" in args: + try: + args["depth"] = int(args["depth"]) + except (TypeError, ValueError): + args.pop("depth", None) + if "changed_only" in args: + args["changed_only"] = str(args["changed_only"]).lower() in ( + "1", "true", "yes") + return _tool_response("observe_window", args) + + @app.route("/api/snapshot", methods=["POST"]) + def api_snapshot(): + return _tool_response("snapshot", request.get_json(silent=True) or {}) + + @app.route("/api/snapshot/") + def api_snapshot_get(sid: str): + return _tool_response("snapshot_get", {"snapshot_id": sid}) + + @app.route("/api/snapshot/diff", methods=["POST"]) + def api_snapshot_diff(): + return _tool_response("snapshot_diff", request.get_json(force=True) or {}) + + @app.route("/api/snapshot/", methods=["DELETE"]) + def api_snapshot_drop(sid: str): + return _tool_response("snapshot_drop", {"snapshot_id": sid}) + + @app.route("/api/wait_for", methods=["POST"]) + def api_wait_for(): + return _tool_response("wait_for", request.get_json(force=True) or {}) + + @app.route("/api/wait_idle", methods=["POST"]) + def api_wait_idle(): + return _tool_response("wait_idle", request.get_json(force=True) or {}) + + @app.route("/api/element/click_and_observe", methods=["POST"]) + def api_element_click_observe(): + return _tool_response("click_element_and_observe", + request.get_json(force=True) or {}) + + @app.route("/api/type_and_observe", methods=["POST"]) + def api_type_observe(): + return _tool_response("type_and_observe", request.get_json(force=True) or {}) + + @app.route("/api/key_and_observe", methods=["POST"]) + def api_key_observe(): + return _tool_response("press_key_and_observe", request.get_json(force=True) or {}) + + # ── P3: filtering, cropping, region OCR, budgeted description ─────────── + + @app.route("/api/screenshot/cropped") + def api_screenshot_cropped(): + return _tool_response("get_screenshot_cropped", _merge_query()) + + @app.route("/api/ocr") + def api_ocr(): + return _tool_response("get_ocr", _merge_query()) + + # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── + + @app.route("/api/trace/start", methods=["POST"]) + def api_trace_start(): + return _tool_response("trace_start", request.get_json(silent=True) or {}) + + @app.route("/api/trace/stop", methods=["POST"]) + def api_trace_stop(): + return _tool_response("trace_stop", {}) + + @app.route("/api/trace/status") + def api_trace_status(): + return _tool_response("trace_status", {}) + + @app.route("/api/replay/start", methods=["POST"]) + def api_replay_start(): + return _tool_response("replay_start", request.get_json(force=True) or {}) + + @app.route("/api/replay/step", methods=["POST"]) + def api_replay_step(): + return _tool_response("replay_step", request.get_json(force=True) or {}) + + @app.route("/api/replay/status", methods=["POST"]) + def api_replay_status(): + return _tool_response("replay_status", request.get_json(force=True) or {}) + + @app.route("/api/replay/stop", methods=["POST"]) + def api_replay_stop(): + return _tool_response("replay_stop", request.get_json(force=True) or {}) + + @app.route("/api/scenario/load", methods=["POST"]) + def api_scenario_load(): + return _tool_response("load_scenario", request.get_json(force=True) or {}) + + @app.route("/api/assert_state", methods=["POST"]) + def api_assert_state(): + return _tool_response("assert_state", request.get_json(force=True) or {}) + + # ── P5: budgets / redaction status / propose ──────────────────────────── + + @app.route("/api/budget_status") + def api_budget_status(): + return _tool_response("get_budget_status", {}) + + @app.route("/api/redaction_status") + def api_redaction_status(): + return _tool_response("get_redaction_status", {}) + + @app.route("/api/propose_action", methods=["POST"]) + def api_propose(): + return _tool_response("propose_action", request.get_json(force=True) or {}) + + # ── P6: extra input verbs ──────────────────────────────────────────────── + + @app.route("/api/hover", methods=["POST"]) + def api_hover(): + body = request.get_json(force=True) or {} + if "x" in body and "y" in body and not body.get("selector") and not body.get("element_id"): + return _tool_response("hover_at", body) + return _tool_response("hover_element", body) + + @app.route("/api/element/right_click", methods=["POST"]) + def api_right_click(): + return _tool_response("right_click_element", request.get_json(force=True) or {}) + + @app.route("/api/element/double_click", methods=["POST"]) + def api_double_click(): + return _tool_response("double_click_element", request.get_json(force=True) or {}) + + @app.route("/api/drag", methods=["POST"]) + def api_drag(): + return _tool_response("drag", request.get_json(force=True) or {}) + + @app.route("/api/element/key", methods=["POST"]) + def api_key_into(): + return _tool_response("key_into_element", request.get_json(force=True) or {}) + + @app.route("/api/element/clear_text", methods=["POST"]) + def api_clear_text(): + return _tool_response("clear_text", request.get_json(force=True) or {}) + + # ── Telemetry: metrics in Prometheus format ───────────────────────────── + + @app.route("/api/metrics") + def api_metrics(): + from session import get_session + s = get_session() + tc = s.tree_cache.counters() + lines = [ + "# HELP oso_step_count Total tool calls processed", + "# TYPE oso_step_count counter", + f"oso_step_count {s.steps.count}", + "# HELP oso_uptime_seconds Process uptime", + "# TYPE oso_uptime_seconds gauge", + f"oso_uptime_seconds {int(s.steps.uptime_s)}", + # ── Tree-cache effectiveness (P2) ───────────────────────────── + "# HELP oso_tree_cache_hits_total Tree-cache lookups served " + "from a fresh cached capture", + "# TYPE oso_tree_cache_hits_total counter", + f"oso_tree_cache_hits_total {tc['hits']}", + "# HELP oso_tree_cache_misses_total Tree-cache lookups that " + "required a fresh accessibility-tree walk", + "# TYPE oso_tree_cache_misses_total counter", + f"oso_tree_cache_misses_total {tc['misses']}", + "# HELP oso_tree_cache_entries Windows currently cached", + "# TYPE oso_tree_cache_entries gauge", + f"oso_tree_cache_entries {tc['entries']}", + # ── Capture-latency summary (P2) ────────────────────────────── + "# HELP oso_tree_capture_ms Accessibility-tree capture latency " + "summary (milliseconds)", + "# TYPE oso_tree_capture_ms summary", + f"oso_tree_capture_ms_sum {tc['capture_ms_total']}", + f"oso_tree_capture_ms_count {tc['capture_count']}", + "# HELP oso_tree_capture_ms_max Slowest tree capture this " + "session (milliseconds)", + "# TYPE oso_tree_capture_ms_max gauge", + f"oso_tree_capture_ms_max {tc['capture_ms_max']}", + ] + if s.budgets is not None: + st = s.budgets.status() + lines += [ + "# TYPE oso_actions_used counter", + f"oso_actions_used {st['actions']['used']}", + "# TYPE oso_screenshots_used counter", + f"oso_screenshots_used {st['screenshots']['used']}", + ] + if s.active_trace is not None: + lines.append("oso_active_trace 1") + else: + lines.append("oso_active_trace 0") + body = "\n".join(lines) + "\n" + return body, 200, {"Content-Type": "text/plain; version=0.0.4"} + + # ── Generic tool console ──────────────────────────────────────────────── + + @app.route("/api/tools") + def api_tools_list(): + return jsonify({"ok": True, + "tools": sorted(_tools.REGISTRY.keys())}) + + @app.route("/api/tool/", methods=["GET", "POST"]) + def api_tool_run(name: str): + if request.method == "POST": + args = request.get_json(silent=True) or {} + else: + args = _merge_query() + return _tool_response(name, args) + + # /api/healthz must stay cheap enough to poll: the OCR diagnostic + # spawns a `tesseract --version` subprocess, so it is computed once + # per process and reused (binary availability doesn't change mid-run). + _healthz_ocr_cache: dict = {} + + @app.route("/api/healthz") + def api_healthz(): + from session import get_session + s = get_session() + out = { + "ok": True, + "uptime_s": int(s.steps.uptime_s), + "step_count": s.steps.count, + "adapter": type(observer._adapter).__name__, + "version": (config.get("mcp", {}) or {}).get("version", "0.2.0"), + # P2 telemetry: cache effectiveness + capture-latency summary. + "tree_cache": s.tree_cache.counters(), + } + # Surface common misconfigurations. + try: + from main import config_load_status + out.update(config_load_status()) + except Exception as e: + logger.debug(f"healthz: config_load_status unavailable: {e}") + try: + if "ocr" not in _healthz_ocr_cache: + from ocr_util import diagnose as _ocr_diag + _healthz_ocr_cache["ocr"] = _ocr_diag(config) + out["ocr"] = _healthz_ocr_cache["ocr"] + except Exception as e: + logger.debug(f"healthz: OCR diagnose failed: {e}") + return jsonify(out) diff --git a/window_agent.py b/window_agent.py deleted file mode 100644 index b67a331..0000000 --- a/window_agent.py +++ /dev/null @@ -1,1978 +0,0 @@ -#!/usr/bin/env python3 -""" -window_agent.py — Interactive window inspection and LLM agent for OSScreenObserver. - -Usage: - python window_agent.py [--rest http://127.0.0.1:5001] - -At startup you are prompted for: - OpenWebUI base URL (e.g. http://localhost:3000) - OpenWebUI API key - Model name (e.g. llama3.2:3b or mistral) - -The program then: - 1. Polls the REST server until it responds. - 2. Lists all open windows; you choose one. - 3. Displays the ASCII sketch + accessibility description. - 4. Opens an interactive prompt where you can type a task for the LLM. - 5. The LLM runs an agentic loop, calling screen tools (observe, click, - type, press keys, get OCR, etc.) until the task is done or it stops. - -No API credentials are saved to disk. -""" - -import argparse -import json -import sys -import time -import traceback -import urllib.error -import urllib.parse -import urllib.request -from typing import Any, Dict, List, Optional, Tuple - -# ─── ANSI colour helpers ────────────────────────────────────────────────────── - -_NO_COLOR = not sys.stdout.isatty() - -def _c(text: str, *codes: str) -> str: - if _NO_COLOR: - return text - _MAP = { - "bold": "\033[1m", "dim": "\033[2m", "reset": "\033[0m", - "red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", - "blue": "\033[34m", "magenta": "\033[35m", "cyan": "\033[36m", - "white": "\033[37m", "bright_white": "\033[97m", - } - return "".join(_MAP.get(c, "") for c in codes) + text + "\033[0m" - -# ─── HTTP helpers (stdlib only, no extra deps) ──────────────────────────────── - -def _get(base: str, path: str, params: Optional[Dict] = None, timeout: int = 30) -> Any: - url = base.rstrip("/") + path - if params: - filtered = {k: v for k, v in params.items() if v is not None} - if filtered: - url += "?" + urllib.parse.urlencode(filtered) - with urllib.request.urlopen(url, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Raise on any redirect so a POST is never silently converted to GET.""" - def redirect_request(self, req, fp, code, msg, headers, newurl): - raise urllib.error.HTTPError(newurl, code, - f"Redirect to {newurl} — update your base URL", - headers, fp) - -_NO_REDIRECT_OPENER = urllib.request.build_opener(_NoRedirectHandler) - - -def _post(base: str, path: str, data: Any, headers: Optional[Dict] = None, - timeout: int = 60) -> Any: - body = json.dumps(data).encode() - req = urllib.request.Request( - base.rstrip("/") + path, - data=body, - headers={"Content-Type": "application/json", **(headers or {})}, - method="POST", - ) - with _NO_REDIRECT_OPENER.open(req, timeout=timeout) as resp: - return json.loads(resp.read().decode()) - -# ─── REST server poller ─────────────────────────────────────────────────────── - -def wait_for_server(rest_base: str, retries: int = 40, delay: float = 0.5) -> bool: - print(_c(" Waiting for REST server …", "dim"), end="", flush=True) - for _ in range(retries): - try: - _get(rest_base, "/api/windows", timeout=2) - print(_c(" ready.", "green")) - return True - except Exception: - print(".", end="", flush=True) - time.sleep(delay) - print(_c(" timed out.", "red")) - return False - -# ─── REST API wrappers ──────────────────────────────────────────────────────── - -def _win_params(uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict[str, Any]: - """Return the minimal window-selector dict: uid > index > title substring.""" - if uid: - return {"window_uid": uid} - if index is not None: - return {"window_index": index} - if title: - return {"window_title": title} - return {} - - -def api_list_windows(rest: str) -> Dict: - return _get(rest, "/api/windows") - - -def api_observe(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - params = _win_params(uid, index, title) - sketch = _get(rest, "/api/sketch", params) - desc = _get(rest, "/api/description", params) - return { - "window": sketch.get("window", "unknown"), - "sketch": sketch.get("sketch", ""), - "description": desc.get("description", ""), - } - - -def api_element_tree(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/structure", _win_params(uid, index, title)) - - -def api_description(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/description", _win_params(uid, index, title)) - - -def api_sketch(rest: str, uid: Optional[str], index: Optional[int], - grid_width: Optional[int] = None, grid_height: Optional[int] = None, - ocr: bool = False, title: Optional[str] = None) -> Dict: - params: Dict[str, Any] = _win_params(uid, index, title) - if grid_width is not None: - params["grid_width"] = grid_width - if grid_height is not None: - params["grid_height"] = grid_height - if ocr: - params["ocr"] = "1" - return _get(rest, "/api/sketch", params) - - -def api_screenshot(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/screenshot", _win_params(uid, index, title)) - - -def api_full_screenshot(rest: str, uid: Optional[str], index: Optional[int], - grid_width: Optional[int] = None, - grid_height: Optional[int] = None, - title: Optional[str] = None) -> Dict: - params: Dict[str, Any] = _win_params(uid, index, title) - if grid_width is not None: - params["grid_width"] = grid_width - if grid_height is not None: - params["grid_height"] = grid_height - return _get(rest, "/api/full_screenshot", params) - - -def api_visible_areas(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/visible_areas", _win_params(uid, index, title)) - - -def api_bring_to_foreground(rest: str, uid: Optional[str], index: Optional[int], - title: Optional[str] = None) -> Dict: - return _get(rest, "/api/bring_to_foreground", _win_params(uid, index, title)) - - -def api_action(rest: str, payload: Dict) -> Dict: - return _post(rest, "/api/action", payload) - -# ─── Tool dispatcher (maps LLM tool names → REST calls) ────────────────────── - -def dispatch_tool(tool_name: str, args: Dict, rest: str, - default_uid: Optional[str] = None, - default_index: Optional[int] = None) -> Any: - """ - Route a tool call from the LLM to the appropriate REST endpoint. - Returns a Python object (will be JSON-serialised before sending back to LLM). - """ - uid: Optional[str] = args.get("window_uid") - wi: Optional[int] = args.get("window_index") if "window_index" in args else None - title: Optional[str] = args.get("window_title") - # Apply defaults only when the LLM specified none of uid / index / title. - if uid is None and wi is None and title is None: - uid = default_uid - wi = default_index - - if tool_name == "list_windows": - return api_list_windows(rest) - - elif tool_name == "observe_window": - return api_observe(rest, uid, wi, title) - - elif tool_name == "get_element_tree": - return api_element_tree(rest, uid, wi, title) - - elif tool_name == "get_screen_description": - return api_description(rest, uid, wi, title) - - elif tool_name == "get_screen_sketch": - raw_ocr = args.get("ocr", False) - if isinstance(raw_ocr, str): - ocr = raw_ocr.strip().lower() not in ("", "false", "0", "no") - else: - ocr = bool(raw_ocr) - return api_sketch(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), ocr=ocr, title=title) - - elif tool_name == "get_screenshot": - result = api_screenshot(rest, uid, wi, title) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - result["note"] = ( - "Screenshot captured (base64 data omitted from tool result). " - "Use get_screen_description for text content (OCR + VLM when available)." - ) - return result - - elif tool_name == "get_full_screenshot": - result = api_full_screenshot(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), title=title) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - note = "Screenshot captured (base64 data omitted)." - if result.get("sketch"): - note += " Sketch included above." - result["note"] = note - return result - - elif tool_name == "get_visible_areas": - if uid is None and wi is None and title is None: - return {"error": "get_visible_areas requires a selected window (window_uid, window_index, or window_title)"} - return api_visible_areas(rest, uid, wi, title) - - elif tool_name == "bring_to_foreground": - if uid is None and wi is None and title is None: - return {"error": "bring_to_foreground requires a selected window (window_uid, window_index, or window_title)"} - return api_bring_to_foreground(rest, uid, wi, title) - - elif tool_name == "click_at": - payload = { - "action": "click_at", - "x": args["x"], - "y": args["y"], - } - if "button" in args: - payload["button"] = args["button"] - if "double" in args: - payload["double"] = args["double"] - return api_action(rest, payload) - - elif tool_name == "type_text": - return api_action(rest, {"action": "type", "value": args["text"]}) - - elif tool_name == "press_key": - return api_action(rest, {"action": "key", "value": args["keys"]}) - - elif tool_name == "scroll": - payload = {"action": "scroll"} - for k in ("x", "y", "clicks"): - if k in args: - payload[k] = args[k] - return api_action(rest, payload) - - # ── P1: identity / discovery ──────────────────────────────────────────── - elif tool_name == "get_capabilities": - return _get(rest, "/api/capabilities") - - elif tool_name == "get_monitors": - return _get(rest, "/api/monitors") - - elif tool_name == "find_element": - params: Dict[str, Any] = {"selector": args["selector"]} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - return _get(rest, "/api/find_element", params) - - # ── P1 / P6: element-targeted actions ─────────────────────────────────── - elif tool_name in ("click_element", "focus_element", "invoke_element", - "set_value", "select_option", - "right_click_element", "double_click_element", - "hover_element", "clear_text", "key_into_element"): - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - path = { - "click_element": "/api/element/click", - "focus_element": "/api/element/focus", - "invoke_element": "/api/element/invoke", - "set_value": "/api/element/set_value", - "select_option": "/api/element/select", - "right_click_element": "/api/element/right_click", - "double_click_element": "/api/element/double_click", - "hover_element": "/api/hover", - "clear_text": "/api/element/clear_text", - "key_into_element": "/api/element/key", - }[tool_name] - return _post(rest, path, body) - - elif tool_name == "hover_at": - return _post(rest, "/api/hover", {k: args[k] for k in args - if k in ("x", "y", "hover_ms")}) - - elif tool_name == "drag": - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, "/api/drag", body) - - # ── P2: synchronisation and observation ───────────────────────────────── - elif tool_name == "wait_for": - body = dict(args) - return _post(rest, "/api/wait_for", body) - - elif tool_name == "wait_idle": - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, "/api/wait_idle", body) - - elif tool_name == "observe_window_diff": - params: Dict[str, Any] = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - if "since" in args: - params["since"] = args["since"] - if "format" in args: - params["format"] = args["format"] - return _get(rest, "/api/observe", params) - - elif tool_name in ("click_element_and_observe", "type_and_observe", - "press_key_and_observe"): - path = { - "click_element_and_observe": "/api/element/click_and_observe", - "type_and_observe": "/api/type_and_observe", - "press_key_and_observe": "/api/key_and_observe", - }[tool_name] - body = dict(args) - if "window_index" not in body and "window_uid" not in body and wi is not None: - body["window_index"] = wi - return _post(rest, path, body) - - # ── P2: snapshots ─────────────────────────────────────────────────────── - elif tool_name == "snapshot": - return _post(rest, "/api/snapshot", {}) - - elif tool_name == "snapshot_get": - return _get(rest, f"/api/snapshot/{args['snapshot_id']}") - - elif tool_name == "snapshot_diff": - return _post(rest, "/api/snapshot/diff", - {k: args[k] for k in args if k in ("a", "b", "format")}) - - elif tool_name == "snapshot_drop": - sid = args["snapshot_id"] - url = rest.rstrip("/") + f"/api/snapshot/{sid}" - req = urllib.request.Request(url, method="DELETE") - with _NO_REDIRECT_OPENER.open(req, timeout=10) as resp: - return json.loads(resp.read().decode()) - - # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── - elif tool_name == "trace_start": - return _post(rest, "/api/trace/start", - {"label": args.get("label", "")}) - elif tool_name == "trace_stop": - return _post(rest, "/api/trace/stop", {}) - elif tool_name == "trace_status": - return _get(rest, "/api/trace/status") - - elif tool_name == "replay_start": - return _post(rest, "/api/replay/start", - {k: args[k] for k in args - if k in ("path", "mode", "on_divergence")}) - elif tool_name == "replay_step": - return _post(rest, "/api/replay/step", - {"replay_id": args["replay_id"]}) - elif tool_name == "replay_status": - return _post(rest, "/api/replay/status", - {"replay_id": args["replay_id"]}) - elif tool_name == "replay_stop": - return _post(rest, "/api/replay/stop", - {"replay_id": args["replay_id"]}) - - elif tool_name == "load_scenario": - return _post(rest, "/api/scenario/load", {"path": args["path"]}) - - elif tool_name == "assert_state": - return _post(rest, "/api/assert_state", - {"predicate": args.get("predicate", args.get("predicates", []))}) - - # ── P5: safety & status ───────────────────────────────────────────────── - elif tool_name == "get_budget_status": - return _get(rest, "/api/budget_status") - - elif tool_name == "get_redaction_status": - return _get(rest, "/api/redaction_status") - - elif tool_name == "propose_action": - return _post(rest, "/api/propose_action", - {"action": args["action"], "args": args.get("args", {})}) - - # ── P3 extras ─────────────────────────────────────────────────────────── - elif tool_name == "get_screenshot_cropped": - # Pixel data is huge — same omission policy as get_screenshot. - params: Dict[str, Any] = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - for k in ("element_id", "padding_px", "max_width"): - if k in args: - params[k] = args[k] - result = _get(rest, "/api/screenshot/cropped", params) - if "data" in result: - result = {k: v for k, v in result.items() if k != "data"} - result["note"] = "Screenshot captured (base64 data omitted)." - return result - - elif tool_name == "get_ocr": - params: Dict[str, Any] = {} - if "window_uid" in args: - params["window_uid"] = args["window_uid"] - elif wi is not None: - params["window_index"] = wi - if "element_id" in args: - params["element_id"] = args["element_id"] - return _get(rest, "/api/ocr", params) - - # ── Catch-all: drive any registered tool by name ──────────────────────── - elif tool_name == "call_tool": - name = args["name"] - body = args.get("args", {}) or {} - return _post(rest, f"/api/tool/{name}", body) - - else: - return {"error": f"Unknown tool: {tool_name}"} - -# ─── Tool definitions (OpenAI / OpenWebUI format) ───────────────────────────── - -# ─── Full tool catalogue (trimmed descriptions) ────────────────────────────── -# Each entry is the schema sent to the LLM. Tiers and keyword groups are in -# _TOOL_TIER / _KEYWORD_GROUPS below — not in the dict itself so the JSON -# payload stays clean. - -SCREEN_TOOLS: List[Dict] = [ - # ── Core observation ───────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "list_windows", - "description": "List visible desktop windows. Returns index, title, process, PID, geometry, and window_uid. Prefer window_uid over index — index changes after every focus change.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "observe_window", - "description": "ASCII sketch + accessibility description of a window. Call before and after every action.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer", "description": "From list_windows. Omit for focused window."}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_element_tree", - "description": "Accessibility tree as JSON. Each node has id, name, role, value, enabled, focused, bounds {x,y,width,height}.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screen_description", - "description": "Describe a window using all available sources: accessibility tree, OCR, and VLM (when enabled). Returns everything available in one call.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screen_sketch", - "description": "ASCII spatial layout of a window. ocr=true overlays Tesseract text.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "grid_width": {"type": "integer"}, - "grid_height": {"type": "integer"}, - "ocr": {"type": "boolean"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_screenshot", - "description": "Capture a window screenshot. Pixel data omitted; use get_screen_description mode=ocr for text.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_full_screenshot", - "description": "Full-display screenshot + ASCII sketch with OCR for a window.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "grid_width": {"type": "integer"}, - "grid_height": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_visible_areas", - "description": "Non-occluded bounding boxes {x,y,width,height} for a window.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": ["window_index"], - }, - }, - }, - - # ── Core actions ───────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "click_at", - "description": "Click at absolute screen coordinates. Derive from get_element_tree: cx=x+w//2, cy=y+h//2.", - "parameters": { - "type": "object", - "properties": { - "x": {"type": "integer"}, - "y": {"type": "integer"}, - "button": {"type": "string", "enum": ["left", "right", "middle"]}, - "double": {"type": "boolean"}, - }, - "required": ["x", "y"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "type_text", - "description": "Type text into the focused element. Click the field first.", - "parameters": { - "type": "object", - "properties": {"text": {"type": "string"}}, - "required": ["text"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "press_key", - "description": "Press a key or chord. Examples: enter, tab, escape, ctrl+s, alt+f4.", - "parameters": { - "type": "object", - "properties": {"keys": {"type": "string"}}, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "scroll", - "description": "Scroll at (x,y). clicks>0 scrolls up, clicks<0 scrolls down.", - "parameters": { - "type": "object", - "properties": { - "clicks": {"type": "integer"}, - "x": {"type": "integer"}, - "y": {"type": "integer"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "bring_to_foreground", - "description": "Raise a window to the foreground by clicking its title bar. Window indices change after this call — always call list_windows again before using any index.", - "parameters": { - "type": "object", - "properties": {"window_index": {"type": "integer"}}, - "required": ["window_index"], - }, - }, - }, - - # ── Element-targeted actions ────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "find_element", - "description": 'Resolve selector to element_id + bounds. XPath: Window/Pane/Button[name="OK"]. CSS: Window > Button.', - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": ["selector"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "click_element", - "description": "Click by selector or element_id. Returns ActionReceipt (changed, new_dialogs, before/after hashes).", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "button": {"type": "string", "enum": ["left", "right", "middle"]}, - "count": {"type": "integer"}, - "dry_run": {"type": "boolean"}, - "confirm_token": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "click_element_and_observe", - "description": "Click element then return post-click observation diff.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "button": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "type_and_observe", - "description": "type_text + observe_window in one round-trip.", - "parameters": { - "type": "object", - "properties": { - "text": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - "window_uid": {"type": "string"}, - }, - "required": ["text"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "press_key_and_observe", - "description": "press_key + observe_window in one round-trip.", - "parameters": { - "type": "object", - "properties": { - "keys": {"type": "string"}, - "wait_after_ms": {"type": "integer"}, - "since": {"type": "string"}, - "window_uid": {"type": "string"}, - }, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "right_click_element", - "description": "Right-click an element by selector or element_id.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "double_click_element", - "description": "Double-click an element by selector or element_id.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "select_option", - "description": "Open a combo-box / list and select an option by name or zero-based index.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "option_name": {"type": "string"}, - "option_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "set_value", - "description": "Click an editable element, optionally clear it, then type a value.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "value": {"type": "string"}, - "clear_first": {"type": "boolean"}, - }, - "required": ["value"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "focus_element", - "description": "Give keyboard focus to an element.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "invoke_element", - "description": "Invoke an element's primary action (UIA InvokePattern or click fallback).", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "clear_text", - "description": "Click an editable element then select-all + delete.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "key_into_element", - "description": "Click an element then press a key combination atomically.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "keys": {"type": "string"}, - }, - "required": ["keys"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "hover_at", - "description": "Move mouse to (x, y) and hold hover_ms ms.", - "parameters": { - "type": "object", - "properties": { - "x": {"type": "integer"}, "y": {"type": "integer"}, - "hover_ms": {"type": "integer"}, - }, - "required": ["x", "y"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "hover_element", - "description": "Move mouse to an element's centre and hold hover_ms ms.", - "parameters": { - "type": "object", - "properties": { - "selector": {"type": "string"}, - "element_id": {"type": "string"}, - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "hover_ms": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "drag", - "description": "Drag from one point/element to another. Each endpoint: {x,y} or {selector} or {element_id}.", - "parameters": { - "type": "object", - "properties": { - "from": {"type": "object"}, - "to": {"type": "object"}, - "modifiers": {"type": "array", "items": {"type": "string"}}, - "duration_s": {"type": "number"}, - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - }, - "required": ["from", "to"], - }, - }, - }, - - # ── Synchronisation ─────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "wait_for", - "description": "Block until a condition matches or timeout. Conditions: element_appears, element_disappears, text_visible, window_appears, tree_changes.", - "parameters": { - "type": "object", - "properties": { - "any_of": {"type": "array"}, - "window_uid": {"type": "string"}, - "timeout_ms": {"type": "integer"}, - "poll_ms": {"type": "integer"}, - }, - "required": ["any_of"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "wait_idle", - "description": "Block until the accessibility tree is stable for quiet_ms.", - "parameters": { - "type": "object", - "properties": { - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "quiet_ms": {"type": "integer"}, - "timeout_ms": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "observe_window_diff", - "description": "observe_window but returns only the diff since a tree_token.", - "parameters": { - "type": "object", - "properties": { - "window_uid": {"type": "string"}, - "window_index": {"type": "integer"}, - "since": {"type": "string"}, - "format": {"type": "string", "enum": ["custom", "json-patch"]}, - }, - "required": [], - }, - }, - }, - - # ── Snapshots ───────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "snapshot", - "description": "Capture all windows + trees. Returns snapshot_id.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_get", - "description": "Retrieve a saved snapshot.", - "parameters": { - "type": "object", - "properties": {"snapshot_id": {"type": "string"}}, - "required": ["snapshot_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_diff", - "description": "Diff two snapshots (added/removed windows + per-window tree diff).", - "parameters": { - "type": "object", - "properties": { - "a": {"type": "string"}, "b": {"type": "string"}, - "format": {"type": "string", "enum": ["custom", "json-patch"]}, - }, - "required": ["a", "b"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "snapshot_drop", - "description": "Free a snapshot before its TTL expires.", - "parameters": { - "type": "object", - "properties": {"snapshot_id": {"type": "string"}}, - "required": ["snapshot_id"], - }, - }, - }, - - # ── Tracing / replay ────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "trace_start", - "description": "Start recording tool calls to a JSONL trace file.", - "parameters": { - "type": "object", - "properties": {"label": {"type": "string"}}, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "trace_stop", - "description": "Close the active trace.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "trace_status", - "description": "Report whether a trace is active.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "replay_start", - "description": "Load a trace for replay (execute or verify mode).", - "parameters": { - "type": "object", - "properties": { - "path": {"type": "string"}, - "mode": {"type": "string", "enum": ["execute", "verify"]}, - "on_divergence": {"type": "string", "enum": ["stop", "warn", "resume"]}, - }, - "required": ["path"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_step", - "description": "Advance one row of the active replay.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_status", - "description": "Report replay position, total, finished, divergences.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "replay_stop", - "description": "Free a replay handle.", - "parameters": { - "type": "object", - "properties": {"replay_id": {"type": "string"}}, - "required": ["replay_id"], - }, - }, - }, - - # ── Testing / harness ───────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "load_scenario", - "description": "Load a YAML scenario file into the mock adapter.", - "parameters": { - "type": "object", - "properties": {"path": {"type": "string"}}, - "required": ["path"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "assert_state", - "description": "Evaluate predicates against current state. Returns all_passed. Predicates: element_exists, value_equals, text_visible, window_focused, screenshot_similar, …", - "parameters": { - "type": "object", - "properties": { - "predicate": {"type": "array"}, - "predicates": {"type": "array"}, - }, - "required": [], - }, - }, - }, - - # ── Safety / budget ─────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_budget_status", - "description": "Remaining budget: actions, screenshots, vlm_tokens, …", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "get_redaction_status", - "description": "Redaction enabled state and applied count.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "propose_action", - "description": "Issue a confirm_token for a destructive action. Pass the token back to the action to proceed.", - "parameters": { - "type": "object", - "properties": { - "action": {"type": "string"}, - "args": {"type": "object"}, - }, - "required": ["action"], - }, - }, - }, - - # ── Discovery ───────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_capabilities", - "description": "Features the server supports on this host.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "get_monitors", - "description": "Monitors with bounds and scale factor.", - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - - # ── OCR extras ──────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "get_screenshot_cropped", - "description": "Cropped screenshot around an element or bbox, with optional max_width downscale.", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - "element_id": {"type": "string"}, - "padding_px": {"type": "integer"}, - "max_width": {"type": "integer"}, - }, - "required": [], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_ocr", - "description": "Region-scoped OCR. Returns [{text, confidence, bbox}].", - "parameters": { - "type": "object", - "properties": { - "window_index": {"type": "integer"}, - "window_uid": {"type": "string"}, - "element_id": {"type": "string"}, - }, - "required": [], - }, - }, - }, - - # ── Escape hatch ────────────────────────────────────────────────────────── - { - "type": "function", - "function": { - "name": "call_tool", - "description": "Call any server tool by name with arbitrary JSON args.", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "args": {"type": "object"}, - }, - "required": ["name"], - }, - }, - }, -] - -# ─── Tool index, tiers, and keyword groups ──────────────────────────────────── - -_TOOL_BY_NAME: Dict[str, Dict] = { - t["function"]["name"]: t for t in SCREEN_TOOLS -} - -# "always" — sent every turn regardless of task -# "usually" — sent by default (covers the vast majority of tasks) -# "on_demand"— omitted unless the task keywords or request_tools activates them -_TOOL_TIER: Dict[str, str] = { - # always - "list_windows": "always", - "observe_window": "always", - "get_element_tree": "always", - "find_element": "always", - "click_element": "always", - "type_text": "always", - "press_key": "always", - "scroll": "always", - "click_element_and_observe": "always", - # usually - "bring_to_foreground": "usually", - "right_click_element": "usually", - "double_click_element": "usually", - "select_option": "usually", - "set_value": "usually", - "wait_idle": "usually", - "type_and_observe": "usually", - "press_key_and_observe": "usually", - "get_screen_description": "usually", - # on_demand (everything else) - "get_screen_sketch": "on_demand", - "get_screenshot": "on_demand", - "get_full_screenshot": "on_demand", - "get_visible_areas": "on_demand", - "get_screenshot_cropped": "on_demand", - "get_ocr": "on_demand", - "hover_at": "on_demand", - "hover_element": "on_demand", - "drag": "on_demand", - "key_into_element": "on_demand", - "clear_text": "on_demand", - "focus_element": "on_demand", - "invoke_element": "on_demand", - "click_at": "on_demand", - "wait_for": "on_demand", - "observe_window_diff": "on_demand", - "snapshot": "on_demand", - "snapshot_get": "on_demand", - "snapshot_diff": "on_demand", - "snapshot_drop": "on_demand", - "trace_start": "on_demand", - "trace_stop": "on_demand", - "trace_status": "on_demand", - "replay_start": "on_demand", - "replay_step": "on_demand", - "replay_status": "on_demand", - "replay_stop": "on_demand", - "load_scenario": "on_demand", - "assert_state": "on_demand", - "get_budget_status": "on_demand", - "get_redaction_status": "on_demand", - "propose_action": "on_demand", - "get_capabilities": "on_demand", - "get_monitors": "on_demand", - "call_tool": "on_demand", -} - -# Task keywords that unlock specific on_demand tools without a request_tools call. -_KEYWORD_GROUPS: Dict[str, List[str]] = { - "drag": ["drag"], - "drop": ["drag"], - "move": ["drag"], - "hover": ["hover_at", "hover_element"], - "tooltip": ["hover_at", "hover_element"], - "screenshot": ["get_full_screenshot", "get_screenshot"], - "capture": ["get_full_screenshot"], - "sketch": ["get_screen_sketch"], - "ocr": ["get_ocr", "get_screen_description"], - "read text": ["get_ocr"], - "cropped": ["get_screenshot_cropped"], - "visible area": ["get_visible_areas"], - "occluded": ["get_visible_areas"], - "snapshot": ["snapshot", "snapshot_get", "snapshot_diff", "snapshot_drop"], - "compare": ["snapshot", "snapshot_diff"], - "diff": ["observe_window_diff", "snapshot_diff"], - "wait for": ["wait_for"], - "appear": ["wait_for"], - "loading": ["wait_for"], - "coordinates": ["click_at"], - "pixel": ["click_at"], - "clear": ["clear_text"], - "erase": ["clear_text"], - "focus": ["focus_element"], - "invoke": ["invoke_element"], - "trace": ["trace_start", "trace_stop", "trace_status"], - "record": ["trace_start", "trace_stop", "trace_status"], - "replay": ["replay_start", "replay_step", "replay_status", "replay_stop"], - "playback": ["replay_start", "replay_step", "replay_status", "replay_stop"], - "assert": ["assert_state"], - "verify state": ["assert_state"], - "budget": ["get_budget_status"], - "redact": ["get_redaction_status"], - "capabilities": ["get_capabilities"], - "monitors": ["get_monitors"], -} - -# Meta-tools always included — handled locally in the agent loop, never sent to REST. -_META_TOOLS: List[Dict] = [ - { - "type": "function", - "function": { - "name": "list_available_tools", - "description": ( - "List tools not yet active in this session, with one-line descriptions. " - "Call request_tools with the names you need to activate them." - ), - "parameters": {"type": "object", "properties": {}, "required": []}, - }, - }, - { - "type": "function", - "function": { - "name": "request_tools", - "description": ( - "Activate additional tools by name for the rest of this session. " - "Use list_available_tools first to see what is available." - ), - "parameters": { - "type": "object", - "properties": { - "names": { - "type": "array", - "items": {"type": "string"}, - "description": "Tool names to activate.", - } - }, - "required": ["names"], - }, - }, - }, -] - - -def _initial_active_tools(task: str) -> set: - """Return the set of tool names to send on turn 1 based on tier + keywords.""" - active = {name for name, tier in _TOOL_TIER.items() if tier in ("always", "usually")} - task_lower = task.lower() - for kw, names in _KEYWORD_GROUPS.items(): - if kw in task_lower: - active.update(names) - return active - - -def _tool_defs_for(active: set) -> List[Dict]: - """Build the tools list to send to the LLM from the active name set.""" - defs = [_TOOL_BY_NAME[n] for n in active if n in _TOOL_BY_NAME] - return defs + _META_TOOLS - - -SYSTEM_PROMPT = """\ -You are a GUI automation agent operating on a live desktop. -You observe screen state through accessibility tools and execute mouse and keyboard actions. - -COORDINATE RULE -All x, y values must come from get_element_tree bounds — never estimate or recall coordinates. -To click the centre of an element with bounds {x, y, width, height}: - click_x = x + width // 2 - click_y = y + height // 2 - -OBSERVATION RULE -You are blind to the screen unless you call observe_window or get_screen_description. -Call observe_window before deciding where to act, and after every action to confirm the result. -Always read the window title from the observe_window result and confirm it matches the window -you intend to act on before proceeding. - -FINDING ELEMENTS — IMPORTANT -The accessibility tree may be incomplete for web pages and some applications. -If a selector or element search fails with NOT FOUND: - 1. Call get_screen_description to get accessibility + OCR + visual text all at once. - 2. Use get_screenshot and inspect the image to identify element positions visually. - 3. Fall back to click_at with coordinates derived from element bounds in the sketch/screenshot. -Do not give up after one NOT FOUND — always try the screenshot/OCR path before reporting failure. - -WINDOW INDEX INSTABILITY — CRITICAL -window_index values change every time a window is raised, minimised, or closed. -• Every window tool call returns window_uid in its response — capture it immediately and use it - on all subsequent calls for that window instead of window_index. -• When you must call by window_index, the server auto-resolves it to the uid and returns - window_uid in the result — read that value and switch to uid= from that point on. -• Never assume the same index still refers to the same window between tool calls. - -BROWSER TAB SWITCHING -Browser tab bars appear as TabItem elements in the accessibility tree. -To switch to a different tab: observe_window, then click_element on the correct TabItem. -The window title updates to reflect the active tab after the click. - -TASK COMPLETION -Complete every part of the user's task before stopping. -Do not ask for clarification or next steps mid-task when the task is unambiguous. -Only report done when all sub-tasks are finished. - -WORKFLOW -1. list_windows — note window_uid for the target window; use uid on all future calls -2. bring_to_foreground(window_uid=…) — raise the window (result includes window_uid if you used index) -3. observe_window(window_uid=…) — verify window title matches; understand current state -4. get_element_tree(window_uid=…) — get exact coordinates when needed -5. Execute one action -6. observe_window — verify title still matches and the change occurred -7. Repeat until ALL sub-tasks are complete - -TOOL AVAILABILITY -Only a subset of tools is active at session start to keep context short. - • list_available_tools() — see what else exists - • request_tools(names=[…]) — activate specific tools for this session - -If an action does not produce the expected result, re-observe and try an alternative approach. -""" - -# OpenWebUI OpenAI-compatible API prefix (centralised so both endpoints stay in sync) -_OWU_PREFIX = "/api/v1" - -# ─── LLM client (OpenAI-compatible, OpenWebUI) ──────────────────────────────── - -class LLMClient: - def __init__(self, base_url: str, api_key: str, model: str): - self.base_url = base_url.rstrip("/") - self.api_key = api_key - self.model = model - - def chat(self, messages: List[Dict], tools: Optional[List[Dict]] = None) -> Dict: - payload: Dict[str, Any] = { - "model": self.model, - "messages": messages, - "temperature": 0.2, - } - if tools: - payload["tools"] = tools - headers = {} - if self.api_key: - headers["Authorization"] = f"Bearer {self.api_key}" - return _post(self.base_url, f"{_OWU_PREFIX}/chat/completions", payload, headers, timeout=240) - -# ─── Agentic loop ───────────────────────────────────────────────────────────── - -MAX_TURNS = 30 -_LLM_MAX_RETRIES = 3 # retries for transient network errors per turn -_LLM_RETRY_DELAY = 5.0 # seconds before first retry (doubles each attempt) - -def run_agent( - llm: LLMClient, - rest: str, - user_task: str, - default_uid: Optional[str], - history: List[Dict], - default_index: Optional[int] = None, -) -> List[Dict]: - """ - Run the agentic tool-calling loop. - - Appends messages to *history* in place and returns the updated history. - Prints progress to stdout using ANSI colours. - """ - history.append({"role": "user", "content": user_task}) - print() - print(_c(f" User: {user_task}", "cyan")) - - active_tools: set = _initial_active_tools(user_task) - n_keyword = len(active_tools) - sum( - 1 for t, tier in _TOOL_TIER.items() if tier in ("always", "usually") - ) - print(_c( - f" [Tools: {len(active_tools)} active" - + (f" (+{n_keyword} from keywords)" if n_keyword > 0 else "") - + f" / {len(SCREEN_TOOLS)} total]", - "dim", - )) - - for turn in range(MAX_TURNS): - tool_defs = _tool_defs_for(active_tools) - resp = None - for attempt in range(_LLM_MAX_RETRIES + 1): - try: - resp = llm.chat(history, tools=tool_defs) - break - except (TimeoutError, ConnectionError, OSError) as e: - if attempt < _LLM_MAX_RETRIES: - delay = _LLM_RETRY_DELAY * (2 ** attempt) - print(_c(f"\n [LLM timeout/connection error — retrying in {delay:.0f}s: {e}]", "yellow")) - time.sleep(delay) - else: - print(_c(f"\n [LLM request failed after {_LLM_MAX_RETRIES + 1} attempts: {e}]", "red")) - return history - except urllib.error.URLError as e: - # URLError wraps OSError/TimeoutError — retry those too. - cause = e.reason if hasattr(e, "reason") else e - if isinstance(cause, (TimeoutError, ConnectionError, OSError)) and attempt < _LLM_MAX_RETRIES: - delay = _LLM_RETRY_DELAY * (2 ** attempt) - print(_c(f"\n [LLM network error — retrying in {delay:.0f}s: {e}]", "yellow")) - time.sleep(delay) - else: - print(_c(f"\n [LLM request failed: {e}]", "red")) - return history - except Exception as e: - print(_c(f"\n [LLM error: {e}]", "red")) - traceback.print_exc() - return history - if resp is None: - return history - - choices = resp.get("choices", []) - if not choices: - print(_c(" [Empty response from LLM]", "red")) - break - - choice = choices[0] - message = choice.get("message", {}) - reason = choice.get("finish_reason", "stop") - - # Accumulate assistant message into history - history.append({"role": "assistant", **{k: v for k, v in message.items() - if k not in ("role",)}}) - - # Print any text content the LLM produced - content = message.get("content") or "" - if content and content.strip(): - print() - print(_c(" Assistant:", "green", "bold")) - for line in content.strip().splitlines(): - print(_c(f" {line}", "white")) - - if reason != "tool_calls": - # LLM is done - break - - tool_calls = message.get("tool_calls") or [] - if not tool_calls: - print(_c(" [finish_reason=tool_calls but no tool_calls in message]", "yellow")) - break - - tool_results: List[Dict] = [] - for tc in tool_calls: - fn_name = tc.get("function", {}).get("name", "unknown") - raw_args = tc.get("function", {}).get("arguments", "{}") - try: - fn_args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args - except json.JSONDecodeError: - fn_args = {} - - # Pretty-print the call - arg_str = ", ".join(f"{k}={v!r}" for k, v in fn_args.items()) if fn_args else "" - print() - print(_c(f" → {fn_name}({arg_str})", "yellow", "bold")) - - # Handle meta-tools locally; dispatch everything else to REST. - if fn_name == "list_available_tools": - inactive = [ - {"name": n, "description": _TOOL_BY_NAME[n]["function"]["description"]} - for n in _TOOL_BY_NAME - if n not in active_tools - ] - result = {"available": inactive, "count": len(inactive)} - elif fn_name == "request_tools": - requested = fn_args.get("names", []) - added, unknown = [], [] - for name in requested: - if name in _TOOL_BY_NAME and name not in active_tools: - active_tools.add(name) - added.append(name) - elif name not in _TOOL_BY_NAME: - unknown.append(name) - result = { - "ok": True, - "added": added, - "unknown": unknown, - "note": ( - f"Activated {len(added)} tool(s). " - "They are available in your tool list from the next turn." - + (f" Unknown: {unknown}" if unknown else "") - ), - } - if added: - print(_c(f" ↳ activated: {', '.join(added)}", "dim")) - else: - try: - result = dispatch_tool(fn_name, fn_args, rest, default_uid, default_index) - except Exception as e: - result = {"error": str(e)} - - # Print a short summary of the result - _print_tool_result(fn_name, result) - - tool_results.append({ - "role": "tool", - "tool_call_id": tc.get("id", ""), - "content": json.dumps(result), - }) - - # Add all tool results as a batch before the next LLM call - history.extend(tool_results) - - else: - print(_c(f"\n [Reached maximum {MAX_TURNS} turns]", "yellow")) - - return history - - -def _print_tool_result(tool_name: str, result: Any) -> None: - """Print a concise, human-readable summary of a tool result.""" - if isinstance(result, dict): - if result.get("error"): - print(_c(f" ✗ error: {result['error']}", "red")) - return - - if tool_name == "list_windows": - windows = result.get("windows", []) - print(_c(f" ← {len(windows)} window(s):", "dim")) - for w in windows[:8]: - flag = " [FOCUSED]" if w.get("focused") else "" - uid = f" uid={w['window_uid']}" if w.get("window_uid") else "" - print(_c(f" [{w['index']}] {w['title']}{flag}{uid}", "dim")) - if len(windows) > 8: - print(_c(f" … and {len(windows) - 8} more", "dim")) - - elif tool_name in ("observe_window",): - sketch = result.get("sketch", "") - if sketch: - first_lines = sketch.splitlines()[:5] - for ln in first_lines: - print(_c(f" │ {ln}", "dim")) - extra = len(sketch.splitlines()) - 5 - if extra > 0: - print(_c(f" │ … ({extra} more lines)", "dim")) - - elif tool_name == "get_screen_sketch": - sketch = result.get("sketch", "") - lines = sketch.splitlines() - for ln in lines[:6]: - print(_c(f" │ {ln}", "dim")) - if len(lines) > 6: - print(_c(f" │ … ({len(lines) - 6} more lines)", "dim")) - - elif tool_name in ("click_at", "type_text", "press_key", "scroll"): - ok = result.get("success", False) - sym = "✓" if ok else "✗" - color = "green" if ok else "red" - note = f" ({result['note']})" if "note" in result else "" - err = f" — {result['error']}" if "error" in result else "" - print(_c(f" ← {sym} {tool_name}{note}{err}", color)) - - elif tool_name == "get_element_tree": - count = result.get("element_count", "?") - window = result.get("window", "") - print(_c(f" ← {count} elements in '{window}'", "dim")) - - elif tool_name == "get_screen_description": - desc = result.get("description", "") - preview = (desc[:120] + "…") if len(desc) > 120 else desc - print(_c(f" ← {preview}", "dim")) - - else: - # Generic: print first 200 chars of JSON - raw = json.dumps(result) - print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) - else: - raw = str(result) - print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) - -# ─── Display helpers ────────────────────────────────────────────────────────── - -_BANNER = r""" - ___ ____ ____ ___ _ - / _ \/ ___/ ___| ___ _ __ ___ ___ _ __ / _ \| |__ ___ ___ _ ____ _____ _ __ -| | | \___ \___ \ / __| '__/ _ \/ _ \ '_ \| | | | '_ \/ __|/ _ \ '__\ \ / / _ \ '__| -| |_| |___) |__) | (__| | | __/ __/ | | | |_| | |_) \__ \ __/ | \ V / __/ | - \___/|____/____/ \___|_| \___|\___|_| |_|\___/|_.__/|___/\___|_| \_/ \___|_| -""" - -def print_banner(): - print(_c(_BANNER.strip("\n"), "cyan", "bold")) - print(_c(" Window Inspection + LLM Agent • OSScreenObserver REST API\n", "dim")) - - -def print_window_list(windows: List[Dict]) -> None: - print(_c(f"\n {'#':>3} {'Title':<50} {'Process':<20} {'Size'}", "bold")) - print(" " + "─" * 90) - for w in windows: - b = w.get("bounds", {}) - size = f"{b.get('width', 0)}×{b.get('height', 0)}" - focused = _c(" ◀", "green") if w.get("focused") else "" - idx_s = _c(f"{w['index']:>3}", "yellow") - title = w.get("title", "")[:50] - proc = w.get("process", "")[:20] - print(f" {idx_s} {title:<50} {proc:<20} {size}{focused}") - print() - - -def print_window_view(data: Dict) -> None: - """Print the sketch and description for a selected window.""" - window = data.get("window", "?") - sketch = data.get("sketch", "") - desc = data.get("description", "") - - width = max((len(ln) for ln in sketch.splitlines()), default=0) + 4 - width = max(width, 60) - header = f" SKETCH — {window} " - pad = max(0, width - len(header) - 2) - print() - print(_c("┌" + header + "─" * pad + "┐", "blue")) - for line in sketch.splitlines(): - print(_c("│ ", "blue") + line) - print(_c("└" + "─" * (width - 2) + "┘", "blue")) - - if desc: - print() - print(_c(" ACCESSIBILITY DESCRIPTION", "bold")) - print(" " + "─" * 50) - for line in desc.splitlines(): - print(" " + line) - print() - -# ─── Interactive prompts ────────────────────────────────────────────────────── - -def prompt(msg: str, default: str = "", secret: bool = False) -> str: - if default: - display = f"{msg} [{default}]: " - else: - display = f"{msg}: " - if secret: - import getpass - val = getpass.getpass(display) - else: - val = input(display).strip() - return val if val else default - - -def ask_connection() -> Tuple[str, str]: - """Interactively collect OpenWebUI connection parameters (model chosen after fetch).""" - print(_c("\n ── OpenWebUI / LLM Connection ──────────────────────────────\n", "bold")) - base_url = prompt(" OpenWebUI base URL", "http://localhost:3000") - api_key = prompt(" API key (leave blank if none)", secret=True) - return base_url, api_key - - -def pick_model(models: List[str]) -> str: - """Display a numbered menu of *models* and return the chosen model id.""" - default = models[0] - print(_c("\n Available models:\n", "bold")) - for i, m in enumerate(models): - print(_c(f" {i + 1:>3}. ", "yellow") + m) - print() - while True: - raw = input(_c(f" Select model (number or name) [{default}]: ", "bold", "cyan")).strip() - if not raw: - return default - if raw.isdigit(): - idx = int(raw) - 1 - if 0 <= idx < len(models): - return models[idx] - print(_c(f" Please enter a number between 1 and {len(models)}.", "red")) - elif raw in models: - return raw - else: - confirm = input(_c(f" '{raw}' not in list — use it anyway? [y/N] ", "yellow")).strip().lower() - if confirm == "y": - return raw - - -def list_models(llm_base: str, api_key: str) -> Tuple[List[str], Optional[str]]: - """Fetch model list from /api/v1/models. Returns (models, error_message).""" - try: - headers: Dict[str, str] = {} - if api_key: - headers["Authorization"] = f"Bearer {api_key}" - req = urllib.request.Request( - llm_base.rstrip("/") + f"{_OWU_PREFIX}/models", - headers=headers, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - data = json.loads(resp.read().decode()) - return [m["id"] for m in data.get("data", [])], None - except Exception as e: - return [], str(e) - -# ─── Main ───────────────────────────────────────────────────────────────────── - -def main() -> None: - parser = argparse.ArgumentParser(description="OSScreenObserver interactive agent") - parser.add_argument("--rest", default="http://127.0.0.1:5001", - help="Base URL of the OSScreenObserver REST server") - args = parser.parse_args() - rest = args.rest - - print_banner() - - # ── 1. REST server ──────────────────────────────────────────────────────── - print(_c(f" REST server: {rest}\n", "dim")) - if not wait_for_server(rest): - print(_c( - "\n Could not reach the REST server. " - "Start it with: python main.py --mode inspect\n", "red" - )) - sys.exit(1) - - # ── 2. LLM connection ───────────────────────────────────────────────────── - llm_base, api_key = ask_connection() - - print(_c(f"\n Checking connection to {llm_base} …", "dim"), end="", flush=True) - models, err = list_models(llm_base, api_key) - if models: - print(_c(f" OK ({len(models)} model(s) available)", "green")) - model = pick_model(models) - elif err: - print(_c(f" failed — {err}", "yellow")) - model = prompt(" Model name", "llama3.2:3b") - else: - print(_c(" connected, but no models found", "yellow")) - model = prompt(" Model name", "llama3.2:3b") - - llm = LLMClient(llm_base, api_key, model) - - # ── 3. Main window-selection loop ───────────────────────────────────────── - conversation: List[Dict] = [{"role": "system", "content": SYSTEM_PROMPT}] - selected_window: Optional[int] = None - selected_uid: Optional[str] = None - - print() - print(_c(" Commands: select window r refresh q quit", "dim")) - - while True: - # Fetch and display window list - try: - wdata = api_list_windows(rest) - except Exception as e: - print(_c(f"\n [Failed to list windows: {e}]", "red")) - input(" Press Enter to retry… ") - continue - - windows = wdata.get("windows", []) - if not windows: - print(_c("\n No windows found. Press Enter to refresh.", "yellow")) - input() - continue - - print() - print(_c(" ── Open Windows ─────────────────────────────────────────────", "bold")) - print_window_list(windows) - - raw = input(_c(" Select window [number / r / q]: ", "bold")).strip().lower() - - if raw in ("q", "quit", "exit"): - print(_c("\n Goodbye.\n", "dim")) - break - - if raw in ("r", "refresh", ""): - continue - - try: - chosen = int(raw) - except ValueError: - print(_c(" Invalid input.", "red")) - continue - - valid_indices = [w["index"] for w in windows] - if chosen not in valid_indices: - print(_c(f" Index {chosen} not in list.", "red")) - continue - - selected_window = chosen - chosen_win = next((w for w in windows if w["index"] == chosen), None) - win_title = chosen_win["title"] if chosen_win else str(chosen) - selected_uid = chosen_win.get("window_uid") if chosen_win else None - - # ── 4. Window inspection sub-loop ──────────────────────────────────── - print(_c(f"\n Loading window [{chosen}] {win_title} …", "dim")) - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed to observe window: {e}]", "red")) - continue - - print_window_view(view) - - print(_c(" Commands: send to LLM v view window r refresh b back q quit", - "dim")) - print() - - while True: - raw2 = input(_c(" Task / command: ", "bold", "cyan")).strip() - - if raw2.lower() in ("q", "quit", "exit"): - print(_c("\n Goodbye.\n", "dim")) - sys.exit(0) - - if raw2.lower() in ("b", "back", ""): - break - - if raw2.lower() in ("v", "view"): - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed: {e}]", "red")) - continue - print_window_view(view) - continue - - if raw2.lower() in ("r", "refresh"): - try: - view = api_observe(rest, selected_uid, selected_window) - except Exception as e: - print(_c(f" [Failed: {e}]", "red")) - continue - print_window_view(view) - continue - - # Treat anything else as a task for the LLM agent - print() - win_label = selected_uid or str(selected_window) - print(_c(f" ── Running agent for task on window [{win_label}] ──────────", - "magenta", "bold")) - conversation = run_agent(llm, rest, raw2, selected_uid, conversation, - default_index=selected_window) - print() - print(_c(" ── Agent finished ───────────────────────────────────────────", - "magenta")) - print(_c(" Type 'v' to view the window's current state, or enter another task.", - "dim")) - print() - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print(_c("\n\n Interrupted.\n", "dim")) diff --git a/window_agent/__init__.py b/window_agent/__init__.py new file mode 100644 index 0000000..ccfa283 --- /dev/null +++ b/window_agent/__init__.py @@ -0,0 +1,102 @@ +""" +window_agent — Interactive window inspection and LLM agent for +OSScreenObserver (package form of the former window_agent.py). + +Usage: + python -m window_agent [--rest http://127.0.0.1:5001] + +At startup you are prompted for: + OpenWebUI base URL (e.g. http://localhost:3000) + OpenWebUI API key + Model name (e.g. llama3.2:3b or mistral) + +The program then: + 1. Polls the REST server until it responds. + 2. Lists all open windows; you choose one. + 3. Displays the ASCII sketch + accessibility description. + 4. Opens an interactive prompt where you can type a task for the LLM. + 5. The LLM runs an agentic loop, calling screen tools (observe, click, + type, press keys, get OCR, etc.) until the task is done or it stops. + +No API credentials are saved to disk. + +P3 decomposition: the implementation now lives in submodules (client, +dispatch, tool_schemas, prompts, loop, cli). This __init__ re-exports +the pre-split surface so `import window_agent` keeps working unchanged. +""" + +from __future__ import annotations + +from window_agent.cli import ( + _BANNER, + ask_connection, + list_models, + main, + pick_model, + print_banner, + print_window_list, + print_window_view, + prompt, +) +from window_agent.client import ( + _NO_COLOR, + _NO_REDIRECT_OPENER, + _OWU_PREFIX, + LLMClient, + _c, + _get, + _NoRedirectHandler, + _post, + _win_params, + api_action, + api_bring_to_foreground, + api_description, + api_element_tree, + api_full_screenshot, + api_list_windows, + api_observe, + api_screenshot, + api_sketch, + api_visible_areas, + wait_for_server, +) +from window_agent.dispatch import dispatch_tool +from window_agent.loop import ( + _LLM_MAX_RETRIES, + _LLM_RETRY_DELAY, + MAX_TURNS, + _print_tool_result, + run_agent, +) +from window_agent.prompts import SYSTEM_PROMPT +from window_agent.tool_schemas import ( + _KEYWORD_GROUPS, + _META_TOOLS, + _TOOL_BY_NAME, + _TOOL_TIER, + SCREEN_TOOLS, + _initial_active_tools, + _tool_defs_for, +) + +__all__ = [ + # cli + "_BANNER", "ask_connection", "list_models", "main", "pick_model", + "print_banner", "print_window_list", "print_window_view", "prompt", + # client + "_NO_COLOR", "_NO_REDIRECT_OPENER", "_OWU_PREFIX", "LLMClient", "_c", + "_get", "_NoRedirectHandler", "_post", "_win_params", "api_action", + "api_bring_to_foreground", "api_description", "api_element_tree", + "api_full_screenshot", "api_list_windows", "api_observe", + "api_screenshot", "api_sketch", "api_visible_areas", "wait_for_server", + # dispatch + "dispatch_tool", + # loop + "_LLM_MAX_RETRIES", "_LLM_RETRY_DELAY", "MAX_TURNS", + "_print_tool_result", "run_agent", + # prompts + "SYSTEM_PROMPT", + # tool schemas + "_KEYWORD_GROUPS", "_META_TOOLS", "_TOOL_BY_NAME", "_TOOL_TIER", + "SCREEN_TOOLS", "_initial_active_tools", "_tool_defs_for", +] diff --git a/window_agent/__main__.py b/window_agent/__main__.py new file mode 100644 index 0000000..6d855ed --- /dev/null +++ b/window_agent/__main__.py @@ -0,0 +1,10 @@ +"""python -m window_agent — interactive agent entry point.""" + +from window_agent.cli import main +from window_agent.client import _c + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print(_c("\n\n Interrupted.\n", "dim")) diff --git a/window_agent/cli.py b/window_agent/cli.py new file mode 100644 index 0000000..476e2dd --- /dev/null +++ b/window_agent/cli.py @@ -0,0 +1,282 @@ +""" +Interactive CLI: banner, window picker, prompts, main(). + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request +from typing import Dict, List, Optional, Tuple + +from window_agent.client import ( + _OWU_PREFIX, LLMClient, _c, api_list_windows, api_observe, + wait_for_server, +) +from window_agent.loop import run_agent +from window_agent.prompts import SYSTEM_PROMPT + +# ─── Display helpers ────────────────────────────────────────────────────────── + +_BANNER = r""" + ___ ____ ____ ___ _ + / _ \/ ___/ ___| ___ _ __ ___ ___ _ __ / _ \| |__ ___ ___ _ ____ _____ _ __ +| | | \___ \___ \ / __| '__/ _ \/ _ \ '_ \| | | | '_ \/ __|/ _ \ '__\ \ / / _ \ '__| +| |_| |___) |__) | (__| | | __/ __/ | | | |_| | |_) \__ \ __/ | \ V / __/ | + \___/|____/____/ \___|_| \___|\___|_| |_|\___/|_.__/|___/\___|_| \_/ \___|_| +""" + +def print_banner(): + print(_c(_BANNER.strip("\n"), "cyan", "bold")) + print(_c(" Window Inspection + LLM Agent • OSScreenObserver REST API\n", "dim")) + + +def print_window_list(windows: List[Dict]) -> None: + print(_c(f"\n {'#':>3} {'Title':<50} {'Process':<20} {'Size'}", "bold")) + print(" " + "─" * 90) + for w in windows: + b = w.get("bounds", {}) + size = f"{b.get('width', 0)}×{b.get('height', 0)}" + focused = _c(" ◀", "green") if w.get("focused") else "" + idx_s = _c(f"{w['index']:>3}", "yellow") + title = w.get("title", "")[:50] + proc = w.get("process", "")[:20] + print(f" {idx_s} {title:<50} {proc:<20} {size}{focused}") + print() + + +def print_window_view(data: Dict) -> None: + """Print the sketch and description for a selected window.""" + window = data.get("window", "?") + sketch = data.get("sketch", "") + desc = data.get("description", "") + + width = max((len(ln) for ln in sketch.splitlines()), default=0) + 4 + width = max(width, 60) + header = f" SKETCH — {window} " + pad = max(0, width - len(header) - 2) + print() + print(_c("┌" + header + "─" * pad + "┐", "blue")) + for line in sketch.splitlines(): + print(_c("│ ", "blue") + line) + print(_c("└" + "─" * (width - 2) + "┘", "blue")) + + if desc: + print() + print(_c(" ACCESSIBILITY DESCRIPTION", "bold")) + print(" " + "─" * 50) + for line in desc.splitlines(): + print(" " + line) + print() + +# ─── Interactive prompts ────────────────────────────────────────────────────── + +def prompt(msg: str, default: str = "", secret: bool = False) -> str: + if default: + display = f"{msg} [{default}]: " + else: + display = f"{msg}: " + if secret: + import getpass + val = getpass.getpass(display) + else: + val = input(display).strip() + return val if val else default + + +def ask_connection() -> Tuple[str, str]: + """Interactively collect OpenWebUI connection parameters (model chosen after fetch).""" + print(_c("\n ── OpenWebUI / LLM Connection ──────────────────────────────\n", "bold")) + base_url = prompt(" OpenWebUI base URL", "http://localhost:3000") + api_key = prompt(" API key (leave blank if none)", secret=True) + return base_url, api_key + + +def pick_model(models: List[str]) -> str: + """Display a numbered menu of *models* and return the chosen model id.""" + default = models[0] + print(_c("\n Available models:\n", "bold")) + for i, m in enumerate(models): + print(_c(f" {i + 1:>3}. ", "yellow") + m) + print() + while True: + raw = input(_c(f" Select model (number or name) [{default}]: ", "bold", "cyan")).strip() + if not raw: + return default + if raw.isdigit(): + idx = int(raw) - 1 + if 0 <= idx < len(models): + return models[idx] + print(_c(f" Please enter a number between 1 and {len(models)}.", "red")) + elif raw in models: + return raw + else: + confirm = input(_c(f" '{raw}' not in list — use it anyway? [y/N] ", "yellow")).strip().lower() + if confirm == "y": + return raw + + +def list_models(llm_base: str, api_key: str) -> Tuple[List[str], Optional[str]]: + """Fetch model list from /api/v1/models. Returns (models, error_message).""" + try: + headers: Dict[str, str] = {} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + req = urllib.request.Request( + llm_base.rstrip("/") + f"{_OWU_PREFIX}/models", + headers=headers, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + data = json.loads(resp.read().decode()) + return [m["id"] for m in data.get("data", [])], None + except Exception as e: + return [], str(e) + +# ─── Main ───────────────────────────────────────────────────────────────────── + +def main() -> None: + parser = argparse.ArgumentParser(description="OSScreenObserver interactive agent") + parser.add_argument("--rest", default="http://127.0.0.1:5001", + help="Base URL of the OSScreenObserver REST server") + args = parser.parse_args() + rest = args.rest + + print_banner() + + # ── 1. REST server ──────────────────────────────────────────────────────── + print(_c(f" REST server: {rest}\n", "dim")) + if not wait_for_server(rest): + print(_c( + "\n Could not reach the REST server. " + "Start it with: python main.py --mode inspect\n", "red" + )) + sys.exit(1) + + # ── 2. LLM connection ───────────────────────────────────────────────────── + llm_base, api_key = ask_connection() + + print(_c(f"\n Checking connection to {llm_base} …", "dim"), end="", flush=True) + models, err = list_models(llm_base, api_key) + if models: + print(_c(f" OK ({len(models)} model(s) available)", "green")) + model = pick_model(models) + elif err: + print(_c(f" failed — {err}", "yellow")) + model = prompt(" Model name", "llama3.2:3b") + else: + print(_c(" connected, but no models found", "yellow")) + model = prompt(" Model name", "llama3.2:3b") + + llm = LLMClient(llm_base, api_key, model) + + # ── 3. Main window-selection loop ───────────────────────────────────────── + conversation: List[Dict] = [{"role": "system", "content": SYSTEM_PROMPT}] + selected_window: Optional[int] = None + selected_uid: Optional[str] = None + + print() + print(_c(" Commands: select window r refresh q quit", "dim")) + + while True: + # Fetch and display window list + try: + wdata = api_list_windows(rest) + except Exception as e: + print(_c(f"\n [Failed to list windows: {e}]", "red")) + input(" Press Enter to retry… ") + continue + + windows = wdata.get("windows", []) + if not windows: + print(_c("\n No windows found. Press Enter to refresh.", "yellow")) + input() + continue + + print() + print(_c(" ── Open Windows ─────────────────────────────────────────────", "bold")) + print_window_list(windows) + + raw = input(_c(" Select window [number / r / q]: ", "bold")).strip().lower() + + if raw in ("q", "quit", "exit"): + print(_c("\n Goodbye.\n", "dim")) + break + + if raw in ("r", "refresh", ""): + continue + + try: + chosen = int(raw) + except ValueError: + print(_c(" Invalid input.", "red")) + continue + + valid_indices = [w["index"] for w in windows] + if chosen not in valid_indices: + print(_c(f" Index {chosen} not in list.", "red")) + continue + + selected_window = chosen + chosen_win = next((w for w in windows if w["index"] == chosen), None) + win_title = chosen_win["title"] if chosen_win else str(chosen) + selected_uid = chosen_win.get("window_uid") if chosen_win else None + + # ── 4. Window inspection sub-loop ──────────────────────────────────── + print(_c(f"\n Loading window [{chosen}] {win_title} …", "dim")) + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed to observe window: {e}]", "red")) + continue + + print_window_view(view) + + print(_c(" Commands: send to LLM v view window r refresh b back q quit", + "dim")) + print() + + while True: + raw2 = input(_c(" Task / command: ", "bold", "cyan")).strip() + + if raw2.lower() in ("q", "quit", "exit"): + print(_c("\n Goodbye.\n", "dim")) + sys.exit(0) + + if raw2.lower() in ("b", "back", ""): + break + + if raw2.lower() in ("v", "view"): + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed: {e}]", "red")) + continue + print_window_view(view) + continue + + if raw2.lower() in ("r", "refresh"): + try: + view = api_observe(rest, selected_uid, selected_window) + except Exception as e: + print(_c(f" [Failed: {e}]", "red")) + continue + print_window_view(view) + continue + + # Treat anything else as a task for the LLM agent + print() + win_label = selected_uid or str(selected_window) + print(_c(f" ── Running agent for task on window [{win_label}] ──────────", + "magenta", "bold")) + conversation = run_agent(llm, rest, raw2, selected_uid, conversation, + default_index=selected_window) + print() + print(_c(" ── Agent finished ───────────────────────────────────────────", + "magenta")) + print(_c(" Type 'v' to view the window's current state, or enter another task.", + "dim")) + print() diff --git a/window_agent/client.py b/window_agent/client.py new file mode 100644 index 0000000..c27d096 --- /dev/null +++ b/window_agent/client.py @@ -0,0 +1,186 @@ +""" +ANSI colour, HTTP and REST/LLM client helpers. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Dict, List, Optional + +# ─── ANSI colour helpers ────────────────────────────────────────────────────── + +_NO_COLOR = not sys.stdout.isatty() + +def _c(text: str, *codes: str) -> str: + if _NO_COLOR: + return text + _MAP = { + "bold": "\033[1m", "dim": "\033[2m", "reset": "\033[0m", + "red": "\033[31m", "green": "\033[32m", "yellow": "\033[33m", + "blue": "\033[34m", "magenta": "\033[35m", "cyan": "\033[36m", + "white": "\033[37m", "bright_white": "\033[97m", + } + return "".join(_MAP.get(c, "") for c in codes) + text + "\033[0m" + +# ─── HTTP helpers (stdlib only, no extra deps) ──────────────────────────────── + +def _get(base: str, path: str, params: Optional[Dict] = None, timeout: int = 30) -> Any: + url = base.rstrip("/") + path + if params: + filtered = {k: v for k, v in params.items() if v is not None} + if filtered: + url += "?" + urllib.parse.urlencode(filtered) + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Raise on any redirect so a POST is never silently converted to GET.""" + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise urllib.error.HTTPError(newurl, code, + f"Redirect to {newurl} — update your base URL", + headers, fp) + +_NO_REDIRECT_OPENER = urllib.request.build_opener(_NoRedirectHandler) + + +def _post(base: str, path: str, data: Any, headers: Optional[Dict] = None, + timeout: int = 60) -> Any: + body = json.dumps(data).encode() + req = urllib.request.Request( + base.rstrip("/") + path, + data=body, + headers={"Content-Type": "application/json", **(headers or {})}, + method="POST", + ) + with _NO_REDIRECT_OPENER.open(req, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + +# ─── REST server poller ─────────────────────────────────────────────────────── + +def wait_for_server(rest_base: str, retries: int = 40, delay: float = 0.5) -> bool: + print(_c(" Waiting for REST server …", "dim"), end="", flush=True) + for _ in range(retries): + try: + _get(rest_base, "/api/windows", timeout=2) + print(_c(" ready.", "green")) + return True + except Exception: + print(".", end="", flush=True) + time.sleep(delay) + print(_c(" timed out.", "red")) + return False + +# ─── REST API wrappers ──────────────────────────────────────────────────────── + +def _win_params(uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict[str, Any]: + """Return the minimal window-selector dict: uid > index > title substring.""" + if uid: + return {"window_uid": uid} + if index is not None: + return {"window_index": index} + if title: + return {"window_title": title} + return {} + + +def api_list_windows(rest: str) -> Dict: + return _get(rest, "/api/windows") + + +def api_observe(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + params = _win_params(uid, index, title) + sketch = _get(rest, "/api/sketch", params) + desc = _get(rest, "/api/description", params) + return { + "window": sketch.get("window", "unknown"), + "sketch": sketch.get("sketch", ""), + "description": desc.get("description", ""), + } + + +def api_element_tree(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/structure", _win_params(uid, index, title)) + + +def api_description(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/description", _win_params(uid, index, title)) + + +def api_sketch(rest: str, uid: Optional[str], index: Optional[int], + grid_width: Optional[int] = None, grid_height: Optional[int] = None, + ocr: bool = False, title: Optional[str] = None) -> Dict: + params: Dict[str, Any] = _win_params(uid, index, title) + if grid_width is not None: + params["grid_width"] = grid_width + if grid_height is not None: + params["grid_height"] = grid_height + if ocr: + params["ocr"] = "1" + return _get(rest, "/api/sketch", params) + + +def api_screenshot(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/screenshot", _win_params(uid, index, title)) + + +def api_full_screenshot(rest: str, uid: Optional[str], index: Optional[int], + grid_width: Optional[int] = None, + grid_height: Optional[int] = None, + title: Optional[str] = None) -> Dict: + params: Dict[str, Any] = _win_params(uid, index, title) + if grid_width is not None: + params["grid_width"] = grid_width + if grid_height is not None: + params["grid_height"] = grid_height + return _get(rest, "/api/full_screenshot", params) + + +def api_visible_areas(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/visible_areas", _win_params(uid, index, title)) + + +def api_bring_to_foreground(rest: str, uid: Optional[str], index: Optional[int], + title: Optional[str] = None) -> Dict: + return _get(rest, "/api/bring_to_foreground", _win_params(uid, index, title)) + + +def api_action(rest: str, payload: Dict) -> Dict: + return _post(rest, "/api/action", payload) + +# OpenWebUI OpenAI-compatible API prefix (centralised so both endpoints stay in sync) +_OWU_PREFIX = "/api/v1" + +# ─── LLM client (OpenAI-compatible, OpenWebUI) ──────────────────────────────── + +class LLMClient: + def __init__(self, base_url: str, api_key: str, model: str): + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.model = model + + def chat(self, messages: List[Dict], tools: Optional[List[Dict]] = None) -> Dict: + payload: Dict[str, Any] = { + "model": self.model, + "messages": messages, + "temperature": 0.2, + } + if tools: + payload["tools"] = tools + headers = {} + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + return _post(self.base_url, f"{_OWU_PREFIX}/chat/completions", payload, headers, timeout=240) diff --git a/window_agent/dispatch.py b/window_agent/dispatch.py new file mode 100644 index 0000000..68ce05c --- /dev/null +++ b/window_agent/dispatch.py @@ -0,0 +1,276 @@ +""" +Tool dispatcher: maps LLM tool names to REST calls. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import urllib.request +from typing import Any, Dict, Optional + +from window_agent.client import ( + _NO_REDIRECT_OPENER, _get, _post, _win_params, + api_action, api_bring_to_foreground, api_description, + api_element_tree, api_full_screenshot, api_list_windows, + api_observe, api_screenshot, api_sketch, api_visible_areas, +) + +# ─── Tool dispatcher (maps LLM tool names → REST calls) ────────────────────── + +def dispatch_tool(tool_name: str, args: Dict, rest: str, + default_uid: Optional[str] = None, + default_index: Optional[int] = None) -> Any: + """ + Route a tool call from the LLM to the appropriate REST endpoint. + Returns a Python object (will be JSON-serialised before sending back to LLM). + """ + uid: Optional[str] = args.get("window_uid") + wi: Optional[int] = args.get("window_index") if "window_index" in args else None + title: Optional[str] = args.get("window_title") + # Apply defaults only when the LLM specified none of uid / index / title. + if uid is None and wi is None and title is None: + uid = default_uid + wi = default_index + + if tool_name == "list_windows": + return api_list_windows(rest) + + elif tool_name == "observe_window": + return api_observe(rest, uid, wi, title) + + elif tool_name == "get_element_tree": + return api_element_tree(rest, uid, wi, title) + + elif tool_name == "get_screen_description": + return api_description(rest, uid, wi, title) + + elif tool_name == "get_screen_sketch": + raw_ocr = args.get("ocr", False) + if isinstance(raw_ocr, str): + ocr = raw_ocr.strip().lower() not in ("", "false", "0", "no") + else: + ocr = bool(raw_ocr) + return api_sketch(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), ocr=ocr, title=title) + + elif tool_name == "get_screenshot": + result = api_screenshot(rest, uid, wi, title) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + result["note"] = ( + "Screenshot captured (base64 data omitted from tool result). " + "Use get_screen_description for text content (OCR + VLM when available)." + ) + return result + + elif tool_name == "get_full_screenshot": + result = api_full_screenshot(rest, uid, wi, args.get("grid_width"), args.get("grid_height"), title=title) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + note = "Screenshot captured (base64 data omitted)." + if result.get("sketch"): + note += " Sketch included above." + result["note"] = note + return result + + elif tool_name == "get_visible_areas": + if uid is None and wi is None and title is None: + return {"error": "get_visible_areas requires a selected window (window_uid, window_index, or window_title)"} + return api_visible_areas(rest, uid, wi, title) + + elif tool_name == "bring_to_foreground": + if uid is None and wi is None and title is None: + return {"error": "bring_to_foreground requires a selected window (window_uid, window_index, or window_title)"} + return api_bring_to_foreground(rest, uid, wi, title) + + elif tool_name == "click_at": + payload = { + "action": "click_at", + "x": args["x"], + "y": args["y"], + } + if "button" in args: + payload["button"] = args["button"] + if "double" in args: + payload["double"] = args["double"] + return api_action(rest, payload) + + elif tool_name == "type_text": + return api_action(rest, {"action": "type", "value": args["text"]}) + + elif tool_name == "press_key": + return api_action(rest, {"action": "key", "value": args["keys"]}) + + elif tool_name == "scroll": + payload = {"action": "scroll"} + for k in ("x", "y", "clicks"): + if k in args: + payload[k] = args[k] + return api_action(rest, payload) + + # ── P1: identity / discovery ──────────────────────────────────────────── + elif tool_name == "get_capabilities": + return _get(rest, "/api/capabilities") + + elif tool_name == "get_monitors": + return _get(rest, "/api/monitors") + + elif tool_name == "find_element": + # Honor the resolved uid/index/title (with defaults applied above); + # _win_params drops null/empty selectors so a stray window_uid=None + # key from the LLM doesn't suppress index/title selection. + params: Dict[str, Any] = {"selector": args["selector"], + **_win_params(uid, wi, title)} + return _get(rest, "/api/find_element", params) + + # ── P1 / P6: element-targeted actions ─────────────────────────────────── + elif tool_name in ("click_element", "focus_element", "invoke_element", + "set_value", "select_option", + "right_click_element", "double_click_element", + "hover_element", "clear_text", "key_into_element"): + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + path = { + "click_element": "/api/element/click", + "focus_element": "/api/element/focus", + "invoke_element": "/api/element/invoke", + "set_value": "/api/element/set_value", + "select_option": "/api/element/select", + "right_click_element": "/api/element/right_click", + "double_click_element": "/api/element/double_click", + "hover_element": "/api/hover", + "clear_text": "/api/element/clear_text", + "key_into_element": "/api/element/key", + }[tool_name] + return _post(rest, path, body) + + elif tool_name == "hover_at": + return _post(rest, "/api/hover", {k: args[k] for k in args + if k in ("x", "y", "hover_ms")}) + + elif tool_name == "drag": + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, "/api/drag", body) + + # ── P2: synchronisation and observation ───────────────────────────────── + elif tool_name == "wait_for": + body = dict(args) + return _post(rest, "/api/wait_for", body) + + elif tool_name == "wait_idle": + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, "/api/wait_idle", body) + + elif tool_name == "observe_window_diff": + params = dict(_win_params(uid, wi, title)) + if "since" in args: + params["since"] = args["since"] + if "format" in args: + params["format"] = args["format"] + return _get(rest, "/api/observe", params) + + elif tool_name in ("click_element_and_observe", "type_and_observe", + "press_key_and_observe"): + path = { + "click_element_and_observe": "/api/element/click_and_observe", + "type_and_observe": "/api/type_and_observe", + "press_key_and_observe": "/api/key_and_observe", + }[tool_name] + body = dict(args) + if "window_index" not in body and "window_uid" not in body and wi is not None: + body["window_index"] = wi + return _post(rest, path, body) + + # ── P2: snapshots ─────────────────────────────────────────────────────── + elif tool_name == "snapshot": + return _post(rest, "/api/snapshot", {}) + + elif tool_name == "snapshot_get": + return _get(rest, f"/api/snapshot/{args['snapshot_id']}") + + elif tool_name == "snapshot_diff": + return _post(rest, "/api/snapshot/diff", + {k: args[k] for k in args if k in ("a", "b", "format")}) + + elif tool_name == "snapshot_drop": + sid = args["snapshot_id"] + url = rest.rstrip("/") + f"/api/snapshot/{sid}" + req = urllib.request.Request(url, method="DELETE") + with _NO_REDIRECT_OPENER.open(req, timeout=10) as resp: + return json.loads(resp.read().decode()) + + # ── P4: tracing, replay, scenarios, oracles ───────────────────────────── + elif tool_name == "trace_start": + return _post(rest, "/api/trace/start", + {"label": args.get("label", "")}) + elif tool_name == "trace_stop": + return _post(rest, "/api/trace/stop", {}) + elif tool_name == "trace_status": + return _get(rest, "/api/trace/status") + + elif tool_name == "replay_start": + return _post(rest, "/api/replay/start", + {k: args[k] for k in args + if k in ("path", "mode", "on_divergence")}) + elif tool_name == "replay_step": + return _post(rest, "/api/replay/step", + {"replay_id": args["replay_id"]}) + elif tool_name == "replay_status": + return _post(rest, "/api/replay/status", + {"replay_id": args["replay_id"]}) + elif tool_name == "replay_stop": + return _post(rest, "/api/replay/stop", + {"replay_id": args["replay_id"]}) + + elif tool_name == "load_scenario": + return _post(rest, "/api/scenario/load", {"path": args["path"]}) + + elif tool_name == "assert_state": + return _post(rest, "/api/assert_state", + {"predicate": args.get("predicate", args.get("predicates", []))}) + + # ── P5: safety & status ───────────────────────────────────────────────── + elif tool_name == "get_budget_status": + return _get(rest, "/api/budget_status") + + elif tool_name == "get_redaction_status": + return _get(rest, "/api/redaction_status") + + elif tool_name == "propose_action": + return _post(rest, "/api/propose_action", + {"action": args["action"], "args": args.get("args", {})}) + + # ── P3 extras ─────────────────────────────────────────────────────────── + elif tool_name == "get_screenshot_cropped": + # Pixel data is huge — same omission policy as get_screenshot. + params = dict(_win_params(uid, wi, title)) + for k in ("element_id", "padding_px", "max_width", "bbox"): + if k in args: + params[k] = args[k] + result = _get(rest, "/api/screenshot/cropped", params) + if "data" in result: + result = {k: v for k, v in result.items() if k != "data"} + result["note"] = "Screenshot captured (base64 data omitted)." + return result + + elif tool_name == "get_ocr": + params = dict(_win_params(uid, wi, title)) + for k in ("element_id", "bbox"): + if k in args: + params[k] = args[k] + return _get(rest, "/api/ocr", params) + + # ── Catch-all: drive any registered tool by name ──────────────────────── + elif tool_name == "call_tool": + name = args["name"] + body = args.get("args", {}) or {} + return _post(rest, f"/api/tool/{name}", body) + + else: + return {"error": f"Unknown tool: {tool_name}"} diff --git a/window_agent/loop.py b/window_agent/loop.py new file mode 100644 index 0000000..4e25eda --- /dev/null +++ b/window_agent/loop.py @@ -0,0 +1,245 @@ +""" +Agentic tool-calling loop and tool-result printing. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +import json +import time +import traceback +import urllib.error +from typing import Any, Dict, List, Optional + +from window_agent.client import LLMClient, _c +from window_agent.dispatch import dispatch_tool +from window_agent.tool_schemas import ( + SCREEN_TOOLS, _TOOL_BY_NAME, _TOOL_TIER, _initial_active_tools, + _tool_defs_for, +) + +# ─── Agentic loop ───────────────────────────────────────────────────────────── + +MAX_TURNS = 30 +_LLM_MAX_RETRIES = 3 # retries for transient network errors per turn +_LLM_RETRY_DELAY = 5.0 # seconds before first retry (doubles each attempt) + +def run_agent( + llm: LLMClient, + rest: str, + user_task: str, + default_uid: Optional[str], + history: List[Dict], + default_index: Optional[int] = None, +) -> List[Dict]: + """ + Run the agentic tool-calling loop. + + Appends messages to *history* in place and returns the updated history. + Prints progress to stdout using ANSI colours. + """ + history.append({"role": "user", "content": user_task}) + print() + print(_c(f" User: {user_task}", "cyan")) + + active_tools: set = _initial_active_tools(user_task) + n_keyword = len(active_tools) - sum( + 1 for t, tier in _TOOL_TIER.items() if tier in ("always", "usually") + ) + print(_c( + f" [Tools: {len(active_tools)} active" + + (f" (+{n_keyword} from keywords)" if n_keyword > 0 else "") + + f" / {len(SCREEN_TOOLS)} total]", + "dim", + )) + + for turn in range(MAX_TURNS): + tool_defs = _tool_defs_for(active_tools) + resp = None + for attempt in range(_LLM_MAX_RETRIES + 1): + try: + resp = llm.chat(history, tools=tool_defs) + break + except (TimeoutError, ConnectionError, OSError) as e: + if attempt < _LLM_MAX_RETRIES: + delay = _LLM_RETRY_DELAY * (2 ** attempt) + print(_c(f"\n [LLM timeout/connection error — retrying in {delay:.0f}s: {e}]", "yellow")) + time.sleep(delay) + else: + print(_c(f"\n [LLM request failed after {_LLM_MAX_RETRIES + 1} attempts: {e}]", "red")) + return history + except urllib.error.URLError as e: + # URLError wraps OSError/TimeoutError — retry those too. + cause = e.reason if hasattr(e, "reason") else e + if isinstance(cause, (TimeoutError, ConnectionError, OSError)) and attempt < _LLM_MAX_RETRIES: + delay = _LLM_RETRY_DELAY * (2 ** attempt) + print(_c(f"\n [LLM network error — retrying in {delay:.0f}s: {e}]", "yellow")) + time.sleep(delay) + else: + print(_c(f"\n [LLM request failed: {e}]", "red")) + return history + except Exception as e: + print(_c(f"\n [LLM error: {e}]", "red")) + traceback.print_exc() + return history + if resp is None: + return history + + choices = resp.get("choices", []) + if not choices: + print(_c(" [Empty response from LLM]", "red")) + break + + choice = choices[0] + message = choice.get("message", {}) + reason = choice.get("finish_reason", "stop") + + # Accumulate assistant message into history + history.append({"role": "assistant", **{k: v for k, v in message.items() + if k not in ("role",)}}) + + # Print any text content the LLM produced + content = message.get("content") or "" + if content and content.strip(): + print() + print(_c(" Assistant:", "green", "bold")) + for line in content.strip().splitlines(): + print(_c(f" {line}", "white")) + + if reason != "tool_calls": + # LLM is done + break + + tool_calls = message.get("tool_calls") or [] + if not tool_calls: + print(_c(" [finish_reason=tool_calls but no tool_calls in message]", "yellow")) + break + + tool_results: List[Dict] = [] + for tc in tool_calls: + fn_name = tc.get("function", {}).get("name", "unknown") + raw_args = tc.get("function", {}).get("arguments", "{}") + try: + fn_args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + except json.JSONDecodeError: + fn_args = {} + + # Pretty-print the call + arg_str = ", ".join(f"{k}={v!r}" for k, v in fn_args.items()) if fn_args else "" + print() + print(_c(f" → {fn_name}({arg_str})", "yellow", "bold")) + + # Handle meta-tools locally; dispatch everything else to REST. + if fn_name == "list_available_tools": + inactive = [ + {"name": n, "description": _TOOL_BY_NAME[n]["function"]["description"]} + for n in _TOOL_BY_NAME + if n not in active_tools + ] + result = {"available": inactive, "count": len(inactive)} + elif fn_name == "request_tools": + requested = fn_args.get("names", []) + added, unknown = [], [] + for name in requested: + if name in _TOOL_BY_NAME and name not in active_tools: + active_tools.add(name) + added.append(name) + elif name not in _TOOL_BY_NAME: + unknown.append(name) + result = { + "ok": True, + "added": added, + "unknown": unknown, + "note": ( + f"Activated {len(added)} tool(s). " + "They are available in your tool list from the next turn." + + (f" Unknown: {unknown}" if unknown else "") + ), + } + if added: + print(_c(f" ↳ activated: {', '.join(added)}", "dim")) + else: + try: + result = dispatch_tool(fn_name, fn_args, rest, default_uid, default_index) + except Exception as e: + result = {"error": str(e)} + + # Print a short summary of the result + _print_tool_result(fn_name, result) + + tool_results.append({ + "role": "tool", + "tool_call_id": tc.get("id", ""), + "content": json.dumps(result), + }) + + # Add all tool results as a batch before the next LLM call + history.extend(tool_results) + + else: + print(_c(f"\n [Reached maximum {MAX_TURNS} turns]", "yellow")) + + return history + + +def _print_tool_result(tool_name: str, result: Any) -> None: + """Print a concise, human-readable summary of a tool result.""" + if isinstance(result, dict): + if result.get("error"): + print(_c(f" ✗ error: {result['error']}", "red")) + return + + if tool_name == "list_windows": + windows = result.get("windows", []) + print(_c(f" ← {len(windows)} window(s):", "dim")) + for w in windows[:8]: + flag = " [FOCUSED]" if w.get("focused") else "" + uid = f" uid={w['window_uid']}" if w.get("window_uid") else "" + print(_c(f" [{w['index']}] {w['title']}{flag}{uid}", "dim")) + if len(windows) > 8: + print(_c(f" … and {len(windows) - 8} more", "dim")) + + elif tool_name in ("observe_window",): + sketch = result.get("sketch", "") + if sketch: + first_lines = sketch.splitlines()[:5] + for ln in first_lines: + print(_c(f" │ {ln}", "dim")) + extra = len(sketch.splitlines()) - 5 + if extra > 0: + print(_c(f" │ … ({extra} more lines)", "dim")) + + elif tool_name == "get_screen_sketch": + sketch = result.get("sketch", "") + lines = sketch.splitlines() + for ln in lines[:6]: + print(_c(f" │ {ln}", "dim")) + if len(lines) > 6: + print(_c(f" │ … ({len(lines) - 6} more lines)", "dim")) + + elif tool_name in ("click_at", "type_text", "press_key", "scroll"): + ok = result.get("success", False) + sym = "✓" if ok else "✗" + color = "green" if ok else "red" + note = f" ({result['note']})" if "note" in result else "" + err = f" — {result['error']}" if "error" in result else "" + print(_c(f" ← {sym} {tool_name}{note}{err}", color)) + + elif tool_name == "get_element_tree": + count = result.get("element_count", "?") + window = result.get("window", "") + print(_c(f" ← {count} elements in '{window}'", "dim")) + + elif tool_name == "get_screen_description": + desc = result.get("description", "") + preview = (desc[:120] + "…") if len(desc) > 120 else desc + print(_c(f" ← {preview}", "dim")) + + else: + # Generic: print first 200 chars of JSON + raw = json.dumps(result) + print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) + else: + raw = str(result) + print(_c(f" ← {raw[:200]}{'…' if len(raw) > 200 else ''}", "dim")) diff --git a/window_agent/prompts.py b/window_agent/prompts.py new file mode 100644 index 0000000..99aa801 --- /dev/null +++ b/window_agent/prompts.py @@ -0,0 +1,66 @@ +""" +System prompt for the GUI-automation agent loop. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +SYSTEM_PROMPT = """\ +You are a GUI automation agent operating on a live desktop. +You observe screen state through accessibility tools and execute mouse and keyboard actions. + +COORDINATE RULE +All x, y values must come from get_element_tree bounds — never estimate or recall coordinates. +To click the centre of an element with bounds {x, y, width, height}: + click_x = x + width // 2 + click_y = y + height // 2 + +OBSERVATION RULE +You are blind to the screen unless you call observe_window or get_screen_description. +Call observe_window before deciding where to act, and after every action to confirm the result. +Always read the window title from the observe_window result and confirm it matches the window +you intend to act on before proceeding. + +FINDING ELEMENTS — IMPORTANT +The accessibility tree may be incomplete for web pages and some applications. +If a selector or element search fails with NOT FOUND: + 1. Call get_screen_description to get accessibility + OCR + visual text all at once. + 2. Use get_screenshot and inspect the image to identify element positions visually. + 3. Fall back to click_at with coordinates derived from element bounds in the sketch/screenshot. +Do not give up after one NOT FOUND — always try the screenshot/OCR path before reporting failure. + +WINDOW INDEX INSTABILITY — CRITICAL +window_index values change every time a window is raised, minimised, or closed. +• Every window tool call returns window_uid in its response — capture it immediately and use it + on all subsequent calls for that window instead of window_index. +• When you must call by window_index, the server auto-resolves it to the uid and returns + window_uid in the result — read that value and switch to uid= from that point on. +• Never assume the same index still refers to the same window between tool calls. + +BROWSER TAB SWITCHING +Browser tab bars appear as TabItem elements in the accessibility tree. +To switch to a different tab: observe_window, then click_element on the correct TabItem. +The window title updates to reflect the active tab after the click. + +TASK COMPLETION +Complete every part of the user's task before stopping. +Do not ask for clarification or next steps mid-task when the task is unambiguous. +Only report done when all sub-tasks are finished. + +WORKFLOW +1. list_windows — note window_uid for the target window; use uid on all future calls +2. bring_to_foreground(window_uid=…) — raise the window (result includes window_uid if you used index) +3. observe_window(window_uid=…) — verify window title matches; understand current state +4. get_element_tree(window_uid=…) — get exact coordinates when needed +5. Execute one action +6. observe_window — verify title still matches and the change occurred +7. Repeat until ALL sub-tasks are complete + +TOOL AVAILABILITY +Only a subset of tools is active at session start to keep context short. + • list_available_tools() — see what else exists + • request_tools(names=[…]) — activate specific tools for this session + +If an action does not produce the expected result, re-observe and try an alternative approach. +""" diff --git a/window_agent/tool_schemas.py b/window_agent/tool_schemas.py new file mode 100644 index 0000000..582b873 --- /dev/null +++ b/window_agent/tool_schemas.py @@ -0,0 +1,963 @@ +""" +LLM tool catalogue: schemas, tiers, keyword groups and +meta-tools for on-demand activation. + +Split out of window_agent.py (P3); behavior is unchanged. +""" + +from __future__ import annotations + +from typing import Dict, List + +# ─── Tool definitions (OpenAI / OpenWebUI format) ───────────────────────────── + +# ─── Full tool catalogue (trimmed descriptions) ────────────────────────────── +# Each entry is the schema sent to the LLM. Tiers and keyword groups are in +# _TOOL_TIER / _KEYWORD_GROUPS below — not in the dict itself so the JSON +# payload stays clean. + +SCREEN_TOOLS: List[Dict] = [ + # ── Core observation ───────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "list_windows", + "description": "List visible desktop windows. Returns index, title, process, PID, geometry, and window_uid. Prefer window_uid over index — index changes after every focus change.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "observe_window", + "description": "ASCII sketch + accessibility description of a window. Call before and after every action.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer", "description": "From list_windows. Omit for focused window."}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_element_tree", + "description": "Accessibility tree as JSON. Each node has id, name, role, value, enabled, focused, bounds {x,y,width,height}.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screen_description", + "description": "Describe a window using all available sources: accessibility tree, OCR, and VLM (when enabled). Returns everything available in one call.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screen_sketch", + "description": "ASCII spatial layout of a window. ocr=true overlays Tesseract text.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "grid_width": {"type": "integer"}, + "grid_height": {"type": "integer"}, + "ocr": {"type": "boolean"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_screenshot", + "description": "Capture a window screenshot. Pixel data omitted; use get_screen_description mode=ocr for text.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_full_screenshot", + "description": "Full-display screenshot + ASCII sketch with OCR for a window.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "grid_width": {"type": "integer"}, + "grid_height": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_visible_areas", + "description": "Non-occluded bounding boxes {x,y,width,height} for a window.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": ["window_index"], + }, + }, + }, + + # ── Core actions ───────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "click_at", + "description": "Click at absolute screen coordinates. Derive from get_element_tree: cx=x+w//2, cy=y+h//2.", + "parameters": { + "type": "object", + "properties": { + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "button": {"type": "string", "enum": ["left", "right", "middle"]}, + "double": {"type": "boolean"}, + }, + "required": ["x", "y"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "type_text", + "description": "Type text into the focused element. Click the field first.", + "parameters": { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "press_key", + "description": "Press a key or chord. Examples: enter, tab, escape, ctrl+s, alt+f4.", + "parameters": { + "type": "object", + "properties": {"keys": {"type": "string"}}, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "scroll", + "description": "Scroll at (x,y). clicks>0 scrolls up, clicks<0 scrolls down.", + "parameters": { + "type": "object", + "properties": { + "clicks": {"type": "integer"}, + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "bring_to_foreground", + "description": "Raise a window to the foreground by clicking its title bar. Window indices change after this call — always call list_windows again before using any index.", + "parameters": { + "type": "object", + "properties": {"window_index": {"type": "integer"}}, + "required": ["window_index"], + }, + }, + }, + + # ── Element-targeted actions ────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "find_element", + "description": 'Resolve selector to element_id + bounds. XPath: Window/Pane/Button[name="OK"]. CSS: Window > Button.', + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": ["selector"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "click_element", + "description": "Click by selector or element_id. Returns ActionReceipt (changed, new_dialogs, before/after hashes).", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "button": {"type": "string", "enum": ["left", "right", "middle"]}, + "count": {"type": "integer"}, + "dry_run": {"type": "boolean"}, + "confirm_token": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "click_element_and_observe", + "description": "Click element then return post-click observation diff.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "button": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "type_and_observe", + "description": "type_text + observe_window in one round-trip.", + "parameters": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + "window_uid": {"type": "string"}, + }, + "required": ["text"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "press_key_and_observe", + "description": "press_key + observe_window in one round-trip.", + "parameters": { + "type": "object", + "properties": { + "keys": {"type": "string"}, + "wait_after_ms": {"type": "integer"}, + "since": {"type": "string"}, + "window_uid": {"type": "string"}, + }, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "right_click_element", + "description": "Right-click an element by selector or element_id.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "double_click_element", + "description": "Double-click an element by selector or element_id.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "select_option", + "description": "Open a combo-box / list and select an option by name or zero-based index.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "option_name": {"type": "string"}, + "option_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "set_value", + "description": "Click an editable element, optionally clear it, then type a value.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "value": {"type": "string"}, + "clear_first": {"type": "boolean"}, + }, + "required": ["value"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "focus_element", + "description": "Give keyboard focus to an element.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "invoke_element", + "description": "Invoke an element's primary action (UIA InvokePattern or click fallback).", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "clear_text", + "description": "Click an editable element then select-all + delete.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "key_into_element", + "description": "Click an element then press a key combination atomically.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "keys": {"type": "string"}, + }, + "required": ["keys"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "hover_at", + "description": "Move mouse to (x, y) and hold hover_ms ms.", + "parameters": { + "type": "object", + "properties": { + "x": {"type": "integer"}, "y": {"type": "integer"}, + "hover_ms": {"type": "integer"}, + }, + "required": ["x", "y"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "hover_element", + "description": "Move mouse to an element's centre and hold hover_ms ms.", + "parameters": { + "type": "object", + "properties": { + "selector": {"type": "string"}, + "element_id": {"type": "string"}, + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "hover_ms": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "drag", + "description": "Drag from one point/element to another. Each endpoint: {x,y} or {selector} or {element_id}.", + "parameters": { + "type": "object", + "properties": { + "from": {"type": "object"}, + "to": {"type": "object"}, + "modifiers": {"type": "array", "items": {"type": "string"}}, + "duration_s": {"type": "number"}, + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + }, + "required": ["from", "to"], + }, + }, + }, + + # ── Synchronisation ─────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "wait_for", + "description": "Block until a condition matches or timeout. Conditions: element_appears, element_disappears, text_visible, window_appears, tree_changes.", + "parameters": { + "type": "object", + "properties": { + "any_of": {"type": "array"}, + "window_uid": {"type": "string"}, + "timeout_ms": {"type": "integer"}, + "poll_ms": {"type": "integer"}, + }, + "required": ["any_of"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "wait_idle", + "description": "Block until the accessibility tree is stable for quiet_ms.", + "parameters": { + "type": "object", + "properties": { + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "quiet_ms": {"type": "integer"}, + "timeout_ms": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "observe_window_diff", + "description": "observe_window but returns only the diff since a tree_token.", + "parameters": { + "type": "object", + "properties": { + "window_uid": {"type": "string"}, + "window_index": {"type": "integer"}, + "since": {"type": "string"}, + "format": {"type": "string", "enum": ["custom", "json-patch"]}, + }, + "required": [], + }, + }, + }, + + # ── Snapshots ───────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "snapshot", + "description": "Capture all windows + trees. Returns snapshot_id.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_get", + "description": "Retrieve a saved snapshot.", + "parameters": { + "type": "object", + "properties": {"snapshot_id": {"type": "string"}}, + "required": ["snapshot_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_diff", + "description": "Diff two snapshots (added/removed windows + per-window tree diff).", + "parameters": { + "type": "object", + "properties": { + "a": {"type": "string"}, "b": {"type": "string"}, + "format": {"type": "string", "enum": ["custom", "json-patch"]}, + }, + "required": ["a", "b"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "snapshot_drop", + "description": "Free a snapshot before its TTL expires.", + "parameters": { + "type": "object", + "properties": {"snapshot_id": {"type": "string"}}, + "required": ["snapshot_id"], + }, + }, + }, + + # ── Tracing / replay ────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "trace_start", + "description": "Start recording tool calls to a JSONL trace file.", + "parameters": { + "type": "object", + "properties": {"label": {"type": "string"}}, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "trace_stop", + "description": "Close the active trace.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "trace_status", + "description": "Report whether a trace is active.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "replay_start", + "description": "Load a trace for replay (execute or verify mode).", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string"}, + "mode": {"type": "string", "enum": ["execute", "verify"]}, + "on_divergence": {"type": "string", "enum": ["stop", "warn", "resume"]}, + }, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_step", + "description": "Advance one row of the active replay.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_status", + "description": "Report replay position, total, finished, divergences.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "replay_stop", + "description": "Free a replay handle.", + "parameters": { + "type": "object", + "properties": {"replay_id": {"type": "string"}}, + "required": ["replay_id"], + }, + }, + }, + + # ── Testing / harness ───────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "load_scenario", + "description": "Load a YAML scenario file into the mock adapter.", + "parameters": { + "type": "object", + "properties": {"path": {"type": "string"}}, + "required": ["path"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "assert_state", + "description": "Evaluate predicates against current state. Returns all_passed. Predicates: element_exists, value_equals, text_visible, window_focused, screenshot_similar, …", + "parameters": { + "type": "object", + "properties": { + "predicate": {"type": "array"}, + "predicates": {"type": "array"}, + }, + "required": [], + }, + }, + }, + + # ── Safety / budget ─────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_budget_status", + "description": "Remaining budget: actions, screenshots, vlm_tokens, …", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "get_redaction_status", + "description": "Redaction enabled state and applied count.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "propose_action", + "description": "Issue a confirm_token for a destructive action. Pass the token back to the action to proceed.", + "parameters": { + "type": "object", + "properties": { + "action": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["action"], + }, + }, + }, + + # ── Discovery ───────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_capabilities", + "description": "Features the server supports on this host.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "get_monitors", + "description": "Monitors with bounds and scale factor.", + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + + # ── OCR extras ──────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "get_screenshot_cropped", + "description": "Cropped screenshot around an element or bbox, with optional max_width downscale.", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + "element_id": {"type": "string"}, + "padding_px": {"type": "integer"}, + "max_width": {"type": "integer"}, + }, + "required": [], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_ocr", + "description": "Region-scoped OCR. Returns [{text, confidence, bbox}].", + "parameters": { + "type": "object", + "properties": { + "window_index": {"type": "integer"}, + "window_uid": {"type": "string"}, + "element_id": {"type": "string"}, + }, + "required": [], + }, + }, + }, + + # ── Escape hatch ────────────────────────────────────────────────────────── + { + "type": "function", + "function": { + "name": "call_tool", + "description": "Call any server tool by name with arbitrary JSON args.", + "parameters": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "args": {"type": "object"}, + }, + "required": ["name"], + }, + }, + }, +] + +# ─── Tool index, tiers, and keyword groups ──────────────────────────────────── + +_TOOL_BY_NAME: Dict[str, Dict] = { + t["function"]["name"]: t for t in SCREEN_TOOLS +} + +# "always" — sent every turn regardless of task +# "usually" — sent by default (covers the vast majority of tasks) +# "on_demand"— omitted unless the task keywords or request_tools activates them +_TOOL_TIER: Dict[str, str] = { + # always + "list_windows": "always", + "observe_window": "always", + "get_element_tree": "always", + "find_element": "always", + "click_element": "always", + "type_text": "always", + "press_key": "always", + "scroll": "always", + "click_element_and_observe": "always", + # usually + "bring_to_foreground": "usually", + "right_click_element": "usually", + "double_click_element": "usually", + "select_option": "usually", + "set_value": "usually", + "wait_idle": "usually", + "type_and_observe": "usually", + "press_key_and_observe": "usually", + "get_screen_description": "usually", + # on_demand (everything else) + "get_screen_sketch": "on_demand", + "get_screenshot": "on_demand", + "get_full_screenshot": "on_demand", + "get_visible_areas": "on_demand", + "get_screenshot_cropped": "on_demand", + "get_ocr": "on_demand", + "hover_at": "on_demand", + "hover_element": "on_demand", + "drag": "on_demand", + "key_into_element": "on_demand", + "clear_text": "on_demand", + "focus_element": "on_demand", + "invoke_element": "on_demand", + "click_at": "on_demand", + "wait_for": "on_demand", + "observe_window_diff": "on_demand", + "snapshot": "on_demand", + "snapshot_get": "on_demand", + "snapshot_diff": "on_demand", + "snapshot_drop": "on_demand", + "trace_start": "on_demand", + "trace_stop": "on_demand", + "trace_status": "on_demand", + "replay_start": "on_demand", + "replay_step": "on_demand", + "replay_status": "on_demand", + "replay_stop": "on_demand", + "load_scenario": "on_demand", + "assert_state": "on_demand", + "get_budget_status": "on_demand", + "get_redaction_status": "on_demand", + "propose_action": "on_demand", + "get_capabilities": "on_demand", + "get_monitors": "on_demand", + "call_tool": "on_demand", +} + +# Task keywords that unlock specific on_demand tools without a request_tools call. +_KEYWORD_GROUPS: Dict[str, List[str]] = { + "drag": ["drag"], + "drop": ["drag"], + "move": ["drag"], + "hover": ["hover_at", "hover_element"], + "tooltip": ["hover_at", "hover_element"], + "screenshot": ["get_full_screenshot", "get_screenshot"], + "capture": ["get_full_screenshot"], + "sketch": ["get_screen_sketch"], + "ocr": ["get_ocr", "get_screen_description"], + "read text": ["get_ocr"], + "cropped": ["get_screenshot_cropped"], + "visible area": ["get_visible_areas"], + "occluded": ["get_visible_areas"], + "snapshot": ["snapshot", "snapshot_get", "snapshot_diff", "snapshot_drop"], + "compare": ["snapshot", "snapshot_diff"], + "diff": ["observe_window_diff", "snapshot_diff"], + "wait for": ["wait_for"], + "appear": ["wait_for"], + "loading": ["wait_for"], + "coordinates": ["click_at"], + "pixel": ["click_at"], + "clear": ["clear_text"], + "erase": ["clear_text"], + "focus": ["focus_element"], + "invoke": ["invoke_element"], + "trace": ["trace_start", "trace_stop", "trace_status"], + "record": ["trace_start", "trace_stop", "trace_status"], + "replay": ["replay_start", "replay_step", "replay_status", "replay_stop"], + "playback": ["replay_start", "replay_step", "replay_status", "replay_stop"], + "assert": ["assert_state"], + "verify state": ["assert_state"], + "budget": ["get_budget_status"], + "redact": ["get_redaction_status"], + "capabilities": ["get_capabilities"], + "monitors": ["get_monitors"], +} + +# Meta-tools always included — handled locally in the agent loop, never sent to REST. +_META_TOOLS: List[Dict] = [ + { + "type": "function", + "function": { + "name": "list_available_tools", + "description": ( + "List tools not yet active in this session, with one-line descriptions. " + "Call request_tools with the names you need to activate them." + ), + "parameters": {"type": "object", "properties": {}, "required": []}, + }, + }, + { + "type": "function", + "function": { + "name": "request_tools", + "description": ( + "Activate additional tools by name for the rest of this session. " + "Use list_available_tools first to see what is available." + ), + "parameters": { + "type": "object", + "properties": { + "names": { + "type": "array", + "items": {"type": "string"}, + "description": "Tool names to activate.", + } + }, + "required": ["names"], + }, + }, + }, +] + + +def _initial_active_tools(task: str) -> set: + """Return the set of tool names to send on turn 1 based on tier + keywords.""" + active = {name for name, tier in _TOOL_TIER.items() if tier in ("always", "usually")} + task_lower = task.lower() + for kw, names in _KEYWORD_GROUPS.items(): + if kw in task_lower: + active.update(names) + return active + + +def _tool_defs_for(active: set) -> List[Dict]: + """Build the tools list to send to the LLM from the active name set.""" + defs = [_TOOL_BY_NAME[n] for n in active if n in _TOOL_BY_NAME] + return defs + _META_TOOLS