From 2c18cd240aa23056c6795dcc0212ac01f3c477e8 Mon Sep 17 00:00:00 2001 From: julia Date: Fri, 31 Jul 2026 14:42:17 +0800 Subject: [PATCH] Add OpenHands CLI agent --- Dockerfile | 9 +- README.md | 3 +- README_ZH.md | 3 +- browseruse_bench/agents/__init__.py | 5 + browseruse_bench/agents/openhands.py | 387 ++++++++++++++++++ config.example.yaml | 12 + configs/agent_registry.yaml | 11 + docs/cli-agents-deployment.md | 45 +- .../browseruse_bench/test_openhands_agent.py | 259 ++++++++++++ 9 files changed, 718 insertions(+), 16 deletions(-) create mode 100644 browseruse_bench/agents/openhands.py create mode 100644 tests/browseruse_bench/test_openhands_agent.py diff --git a/Dockerfile b/Dockerfile index 98df299..2416b81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,11 +80,12 @@ RUN chown -R 1000:1000 /app /root/.cache/uv ENV PATH="/app/.venv/bin:$PATH" ENV UV_PROJECT_ENVIRONMENT="/app/.venv" -# Optional: CLI coding agents (codex / cursor / openclaw, plus the claude-code -# CLI). Off by default to keep the CI image small. Enable with: +# Optional: CLI coding agents (claude-code / codex / cursor / openhands / +# openclaw). Off by default to keep the CI image small. Enable with: # docker build --build-arg INSTALL_CLI_AGENTS=true . # Notes (see docs/cli-agents-deployment.md): # - Node 22.x via NodeSource: openclaw declares engines node>=22.19.0. +# - OpenHands is installed as a uv tool with Python 3.12. # - npm cache lives at a world-writable path so the uid-1000 runtime user # reuses the pre-warmed Playwright MCP download (npx caches per-user via # this env, not under /root). @@ -100,12 +101,14 @@ RUN if [ "$INSTALL_CLI_AGENTS" = "true" ]; then \ && rm -rf /var/lib/apt/lists/* \ && if [ "$USE_CN_MIRROR" = "true" ]; then npm config set registry https://registry.npmmirror.com -g; fi \ && npm install -g @anthropic-ai/claude-code @openai/codex openclaw \ + && UV_TOOL_DIR=/opt/uv-tools UV_TOOL_BIN_DIR=/usr/local/bin uv tool install openhands --python 3.12 \ && curl https://cursor.com/install -fsS | bash \ && mv /root/.local/share/cursor-agent /opt/cursor-agent \ && ln -s /opt/cursor-agent/versions/*/cursor-agent /usr/local/bin/cursor-agent \ && npx -y @playwright/mcp@latest --version \ && chmod -R a+rwX /opt/npm-cache \ && chmod -R a+rX /opt/cursor-agent \ + && chmod -R a+rX /opt/uv-tools \ # Real passwd entry + home for the uid-1000 runtime user: codex # refuses a codex_home under temporary dirs, so HOME must not fall # back to /tmp. docker run --user 1000 resolves HOME from passwd. @@ -120,4 +123,4 @@ RUN cp /app/config.example.yaml /app/config.yaml ENTRYPOINT ["/app/scripts/docker-entrypoint.sh"] -CMD ["uv", "run", "scripts/run.py", "--help"] \ No newline at end of file +CMD ["uv", "run", "scripts/run.py", "--help"] diff --git a/README.md b/README.md index 7b5d3b0..208d17a 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ source .venv/bin/activate # macOS / Linux .venv\Scripts\Activate.ps1 # Windows PowerShell ``` -**CLI agents: claude-code / codex / cursor / openclaw** (requires Node.js 18+; **openclaw requires Node.js 22.19+**) +**CLI agents: claude-code / codex / cursor / openhands / openclaw** (requires Node.js 18+; **openclaw requires Node.js 22.19+**; **openhands requires Python 3.12+ via uv**) These agents share the main venv (`uv sync`, no extra) and drive an external CLI installed separately: @@ -114,6 +114,7 @@ npm install -g @anthropic-ai/claude-code # claude-code (auth: ANTHROPIC_API_ npm install -g @openai/codex # codex (auth: codex login or OPENAI_API_KEY) curl https://cursor.com/install -fsS | bash # cursor (auth: CURSOR_API_KEY) export PATH="$HOME/.local/bin:$PATH" # cursor-agent installs to ~/.local/bin +uv tool install openhands --python 3.12 # openhands (auth: LLM_API_KEY via config.yaml) npm install -g openclaw # openclaw (no login; key via config.yaml) ``` diff --git a/README_ZH.md b/README_ZH.md index 7b751bd..02ee6ff 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -102,7 +102,7 @@ source .venv/bin/activate # macOS / Linux .venv\Scripts\Activate.ps1 # Windows PowerShell ``` -**CLI agents: claude-code / codex / cursor / openclaw**（需要 Node.js 18+；**openclaw 需要 Node.js 22.19+**） +**CLI agents: claude-code / codex / cursor / openhands / openclaw**（需要 Node.js 18+；**openclaw 需要 Node.js 22.19+**；**openhands 需要通过 uv 使用 Python 3.12+**） 这些 agent 共用主 venv（`uv sync`,无 extra），并依赖单独安装的外部 CLI： @@ -112,6 +112,7 @@ npm install -g @anthropic-ai/claude-code # claude-code（认证：ANTHROPIC_ npm install -g @openai/codex # codex（认证：codex login 或 OPENAI_API_KEY） curl https://cursor.com/install -fsS | bash # cursor（认证：CURSOR_API_KEY） export PATH="$HOME/.local/bin:$PATH" # cursor-agent 安装在 ~/.local/bin +uv tool install openhands --python 3.12 # openhands（认证：通过 config.yaml 注入 LLM_API_KEY） npm install -g openclaw # openclaw（无需登录；key 通过 config.yaml 注入） ``` diff --git a/browseruse_bench/agents/__init__.py b/browseruse_bench/agents/__init__.py index 248ff55..420e37e 100644 --- a/browseruse_bench/agents/__init__.py +++ b/browseruse_bench/agents/__init__.py @@ -38,6 +38,11 @@ except ImportError as exc: logger.warning("Skipping optional agent module browseruse_bench.agents.cursor: %s", exc) +try: + from browseruse_bench.agents import openhands # noqa: F401 +except ImportError as exc: + logger.warning("Skipping optional agent module browseruse_bench.agents.openhands: %s", exc) + try: from browseruse_bench.agents import openclaw # noqa: F401 except ImportError as exc: diff --git a/browseruse_bench/agents/openhands.py b/browseruse_bench/agents/openhands.py new file mode 100644 index 0000000..313bc77 --- /dev/null +++ b/browseruse_bench/agents/openhands.py @@ -0,0 +1,387 @@ +""" +OpenHandsAgent - Browser automation using OpenHands CLI with Playwright MCP. + +This agent invokes `openhands --headless --json` and supplies model credentials +through OpenHands' environment override path. A task-local home directory holds +the Playwright MCP server config so benchmark runs do not mutate user config. +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from browseruse_bench.agents.cli_agent import CLIAgent +from browseruse_bench.agents.playwright_mcp import ( + DEFAULT_BROWSER_RULES, + SELF_LAUNCH_BROWSER_IDS, + STEP_ITEM_TYPES, + build_playwright_mcp_args, + collect_screenshots, + extract_actions, + write_api_logs, +) +from browseruse_bench.agents.registry import register_agent +from browseruse_bench.browsers import open_browser_session +from browseruse_bench.browsers.providers.local import warn_if_local_proxy_unsupported +from browseruse_bench.schemas import AgentMetrics, AgentResult +from browseruse_bench.utils import IS_WINDOWS + +logger = logging.getLogger(__name__) + +OPENHANDS_BROWSER_RULES = DEFAULT_BROWSER_RULES.replace( + "browser automation agent", "browser task agent" +) + + +def _command_prefix(value: Any, default: list[str]) -> list[str]: + if isinstance(value, list) and all(isinstance(item, str) for item in value): + return list(value) + if isinstance(value, str) and value.strip(): + return [value] + return default + + +def _iter_json(stdout_lines: list[str]) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for raw_line in stdout_lines: + line = raw_line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + events.append(obj) + return events + + +def _extract_text(value: Any) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts = [_extract_text(item) for item in value] + return "\n".join(part for part in parts if part).strip() + if not isinstance(value, dict): + return "" + for key in ("final_answer", "answer", "result", "message", "content", "text"): + text = _extract_text(value.get(key)) + if text: + return text + return "" + + +def _normalize_action(obj: dict[str, Any]) -> dict[str, Any] | None: + event_type = str(obj.get("type") or "") + kind = str(obj.get("kind") or "") + action = obj.get("action") + if event_type != "action" and kind != "ActionEvent" and action is None: + return None + tool_name = obj.get("tool_name") or obj.get("tool") + if isinstance(action, dict): + name = tool_name or action.get("tool") or action.get("tool_name") or action.get("action") or action.get("name") + args = action.get("data") or action.get("args") or action.get("arguments") or {} + command = action.get("command") + else: + name = action or tool_name or obj.get("name") + args = obj.get("args") or obj.get("arguments") or {} + command = obj.get("command") + if action is None and kind == "ActionEvent": + return None + name_text = str(name or "") + if command: + return {"type": "command_execution", "command": str(command), "status": "completed"} + if "mcp" in name_text.lower() or "browser" in name_text.lower() or "playwright" in name_text.lower(): + return { + "type": "mcp_tool_call", + "tool": name_text, + "arguments": args if isinstance(args, dict) else {}, + "status": "completed", + } + return {"type": "command_execution", "command": name_text, "status": "completed"} + + +def _extract_error_message(obj: dict[str, Any]) -> str | None: + if obj.get("kind") == "AgentErrorEvent": + return None + error = obj.get("error") + if isinstance(error, dict): + return _extract_text(error) or str(error) + if isinstance(error, str): + return error + if obj.get("code") or obj.get("detail") or obj.get("kind") == "ConversationErrorEvent": + detail = obj.get("detail") + if isinstance(detail, str) and detail: + return detail + code = obj.get("code") + if isinstance(code, str) and code: + return code + if obj.get("type") == "error": + return _extract_text(obj) or "OpenHands reported an error" + return None + + +def _parse_jsonl(stdout_lines: list[str]) -> tuple[str, list[dict[str, Any]], str | None]: + """Parse OpenHands `--json` JSONL events into shared bench fields.""" + answer = "" + items: list[dict[str, Any]] = [] + error_message: str | None = None + + for obj in _iter_json(stdout_lines): + error_message = _extract_error_message(obj) or error_message + item = _normalize_action(obj) + if item: + items.append(item) + event_type = str(obj.get("type") or "").lower() + kind = str(obj.get("kind") or "") + if event_type in {"finish", "finished", "final", "result", "message"}: + text = _extract_text(obj) + if text: + answer = text + elif kind == "MessageEvent" and obj.get("source") == "agent": + text = _extract_text(obj.get("llm_message")) + if text: + answer = text + elif obj.get("final_answer") or obj.get("answer"): + text = _extract_text(obj.get("final_answer") or obj.get("answer")) + if text: + answer = text + + return answer, items, error_message + + +@register_agent +class OpenHandsAgent(CLIAgent): + """Browser automation agent using OpenHands CLI with Playwright MCP.""" + + name = "openhands" + + def run_task( + self, + task_info: dict[str, Any], + agent_config: dict[str, Any], + task_workspace: Path, + ) -> AgentResult | dict[str, Any]: + browser_id = str(agent_config.get("browser_id") or "") + if browser_id in SELF_LAUNCH_BROWSER_IDS: + warn_if_local_proxy_unsupported(agent_config, self.name) + return self._execute(task_info, agent_config, task_workspace, cdp_url=None) + with open_browser_session( + browser_id=browser_id, + agent_name=self.name, + agent_config=agent_config, + ) as session_context: + cdp_url = session_context.cdp_url if session_context.transport == "cdp" else None + if not cdp_url: + return self._unsupported_backend_result( + task_info["task_id"], browser_id, session_context.transport + ) + return self._execute(task_info, agent_config, task_workspace, cdp_url=cdp_url) + + def _unsupported_backend_result( + self, task_id: str, browser_id: str, transport: str + ) -> AgentResult: + return AgentResult( + task_id=task_id, + timestamp=datetime.now(UTC), + env_status="failed", # type: ignore[arg-type] + agent_done="error", # type: ignore[arg-type] + error=( + f"Browser backend '{browser_id}' (transport={transport}) provides no CDP " + "endpoint, so the openhands agent cannot attach Playwright MCP to it. " + "Use a CDP-capable backend (e.g. lexmount, cdp) or browser_id=local." + ), + metrics=AgentMetrics(end_to_end_ms=0, steps=0), + ) + + def _execute( + self, + task_info: dict[str, Any], + agent_config: dict[str, Any], + task_workspace: Path, + cdp_url: str | None, + ) -> AgentResult: + task_id = task_info["task_id"] + prompt = task_info.get("prompt") or self.build_task_prompt(task_info) + rules = agent_config.get("system_prompt") or OPENHANDS_BROWSER_RULES + model = agent_config.get("model_id") or agent_config.get("model", "gpt-5.4") + timeout = self._resolve_timeout(task_id, agent_config) + trajectory_dir = task_workspace / "trajectory" + trajectory_dir.mkdir(parents=True, exist_ok=True) + home_dir = self._write_workspace_config(agent_config, task_workspace, cdp_url) + cmd = self._build_command(f"{rules}\n\n{prompt}", agent_config) + + env = {**os.environ, "HOME": str(home_dir), "OH_PERSISTENCE_DIR": str(home_dir / ".openhands")} + env.setdefault("UV_CACHE_DIR", str(Path.home() / ".cache" / "uv")) + env["OPENHANDS_SUPPRESS_BANNER"] = "1" + env["LLM_MODEL"] = str(model) + if agent_config.get("api_key"): + env["LLM_API_KEY"] = str(agent_config["api_key"]) + if agent_config.get("base_url"): + env["LLM_BASE_URL"] = str(agent_config["base_url"]) + + logger.info("Executing OpenHands for task %s (model=%s, timeout=%ds)", task_id, model, timeout) + t_start = time.monotonic() + try: + returncode, stdout_lines, execution_error = self._run_subprocess( + cmd, + timeout=timeout, + task_workspace=task_workspace, + cwd=task_workspace, + env=env, + collect_stdout=True, + stdout_line_hook=_stdout_hook, + stderr_line_hook=_stderr_hook, + terminate_process_group=True, + ) + except FileNotFoundError: + return AgentResult( + task_id=task_id, + timestamp=datetime.now(UTC), + env_status="failed", # type: ignore[arg-type] + agent_done="error", # type: ignore[arg-type] + error="Executable 'openhands' not found. Please install OpenHands CLI.", + metrics=AgentMetrics(end_to_end_ms=0, steps=0), + ) + duration_ms = int((time.monotonic() - t_start) * 1000) + return self._finalize_result( + task_id=task_id, + model=str(model), + rules=rules, + stdout_lines=stdout_lines, + returncode=returncode, + execution_error=execution_error, + duration_ms=duration_ms, + task_workspace=task_workspace, + trajectory_dir=trajectory_dir, + ) + + @staticmethod + def _resolve_timeout(task_id: str, agent_config: dict[str, Any]) -> int: + timeout_val = agent_config.get("timeout_seconds") or agent_config.get("timeout", 600) + try: + return int(timeout_val) + except (TypeError, ValueError) as exc: + logger.warning("Invalid timeout for task %s (%r): %s", task_id, timeout_val, exc) + return 600 + + @staticmethod + def _write_workspace_config( + agent_config: dict[str, Any], + task_workspace: Path, + cdp_url: str | None, + ) -> Path: + home_dir = task_workspace / ".openhands-home" + config_dir = home_dir / ".openhands" + config_dir.mkdir(parents=True, exist_ok=True) + mcp_config = { + "mcpServers": { + "playwright": { + "command": agent_config.get("playwright_mcp_command", "npx"), + "args": build_playwright_mcp_args(agent_config, cdp_url), + "env": {}, + } + } + } + (config_dir / "mcp.json").write_text( + json.dumps(mcp_config, ensure_ascii=False, indent=2), encoding="utf-8" + ) + return home_dir + + @staticmethod + def _build_command( + full_prompt: str, + agent_config: dict[str, Any] | None = None, + ) -> list[str]: + default_cmd = ["openhands.cmd"] if IS_WINDOWS else ["openhands"] + prefix = _command_prefix((agent_config or {}).get("openhands_command"), default_cmd) + return [ + *prefix, + "--headless", + "--json", + "--override-with-envs", + "--task", full_prompt, + ] + + def _finalize_result( + self, + task_id: str, + model: str, + rules: str, + stdout_lines: list[str], + returncode: int, + execution_error: str | None, + duration_ms: int, + task_workspace: Path, + trajectory_dir: Path, + ) -> AgentResult: + answer, items, error_message = _parse_jsonl(stdout_lines) + if execution_error and "Timeout" in execution_error: + logger.error("OpenHands task %s timed out", task_id) + env_status, agent_done = self._map_exit_status( + returncode, execution_error, has_result=bool(answer) + ) + if agent_done != "timeout" and error_message: + env_status, agent_done = "failed", "error" + if agent_done != "timeout" and env_status == "success" and not answer: + env_status, agent_done = "failed", "error" + error_message = self._stderr_error(task_workspace) or "OpenHands exited without an answer" + if env_status == "failed" and not answer: + answer = f"[Task Failed: {execution_error or error_message or 'No output from OpenHands'}]" + + saved_screenshots = collect_screenshots(task_workspace, trajectory_dir) + steps = sum(1 for item in items if item.get("type") in STEP_ITEM_TYPES) + if items: + try: + write_api_logs(task_id, model, rules, items, task_workspace / "api_logs") + except (OSError, TypeError, ValueError) as exc: + logger.warning("Failed to generate api_logs for task %s: %s", task_id, exc) + + return AgentResult( + task_id=task_id, + timestamp=datetime.now(UTC), + env_status=env_status, # type: ignore[arg-type] + agent_done=agent_done, # type: ignore[arg-type] + answer=answer, + error=(execution_error or error_message) if env_status == "failed" else None, + action_history=extract_actions(items), + screenshots=saved_screenshots, + model_id=model, + metrics=AgentMetrics(end_to_end_ms=duration_ms, steps=steps), + ) + + @staticmethod + def _stderr_error(task_workspace: Path) -> str | None: + stderr_file = task_workspace / "stderr.txt" + if not stderr_file.is_file(): + return None + lines = [line.strip() for line in stderr_file.read_text(encoding="utf-8").splitlines() if line.strip()] + for line in lines: + if "error" in line.lower() or "failed" in line.lower(): + return line[:500] + return None + + +def _stdout_hook(line: str) -> None: + clean = line.strip() + if not clean.startswith("{"): + return + try: + obj = json.loads(clean) + except json.JSONDecodeError: + return + item = _normalize_action(obj) + if item: + logger.info("[OpenHands] Action: %s", item.get("tool") or item.get("command")) + + +def _stderr_hook(line: str) -> None: + clean = line.strip() + if clean and "error" in clean.lower(): + logger.warning("[OpenHands] %s", clean) diff --git a/config.example.yaml b/config.example.yaml index efab84b..b8f09c1 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -191,6 +191,13 @@ models: model_type: OPENAI model_id: gpt-5.2 api_key: $CURSOR_API_KEY + # OpenHands CLI model. Only supported by the openhands agent; credentials are + # passed via LLM_API_KEY / LLM_MODEL / LLM_BASE_URL with --override-with-envs. + openhands: + model_type: OPENAI + model_id: gpt-5.5 + api_key: $OPENAI_API_KEY + base_url: $OPENAI_BASE_URL # OpenClaw CLI model. Only supported by the openclaw agent; the agent writes # a per-task OpenClaw provider config from these values (OpenAI-compatible # endpoints work, e.g. a LiteLLM proxy). @@ -313,6 +320,11 @@ agents: cursor: active_model: cursor timeout: 600 + openhands: + active_model: openhands + timeout: 600 + # Defaults to openhands; use ["uv", "tool", "run", "openhands"] without a PATH install. + openhands_command: openhands openclaw: active_model: openclaw timeout: 600 diff --git a/configs/agent_registry.yaml b/configs/agent_registry.yaml index 6b21254..bf64c65 100644 --- a/configs/agent_registry.yaml +++ b/configs/agent_registry.yaml @@ -65,6 +65,17 @@ cursor: - WebVoyager - Odysseys +openhands: + path: browseruse_bench/agents + entrypoint: browseruse_bench/runner/agent_runner.py + venv: .venv + supported_benchmarks: + - Online-Mind2Web + - BrowseComp + - LexBench-Browser + - WebVoyager + - Odysseys + openclaw: path: browseruse_bench/agents entrypoint: browseruse_bench/runner/agent_runner.py diff --git a/docs/cli-agents-deployment.md b/docs/cli-agents-deployment.md index ea78ba2..a521120 100644 --- a/docs/cli-agents-deployment.md +++ b/docs/cli-agents-deployment.md @@ -1,4 +1,4 @@ -# CLI Agents Deployment (claude-code / codex / cursor / openclaw) +# CLI Agents Deployment (claude-code / codex / cursor / openhands / openclaw) The CLI-based agents drive an external coding-agent CLI as a subprocess. They share the repo's main venv (`uv sync`, no extra) but each requires its CLI @@ -15,7 +15,8 @@ upgrading. | Requirement | Why | |---|---| | Python >= 3.11 + `uv sync` | bench runtime (all CLI agents use the root `.venv`) | -| Node.js >= 18 + npm (claude-code, codex, **cursor**) / **>= 22.19 (openclaw)** | installs the npm CLIs, and `npx` launches Playwright MCP at runtime for codex **and cursor** (cursor's own binary installs via curl but still needs `npx`). openclaw declares `engines: >=22.19.0` — install Node 22+ when openclaw is in the fleet | +| Node.js >= 18 + npm (claude-code, codex, **cursor**) / **>= 22.19 (openclaw)** | installs the npm CLIs, and `npx` launches Playwright MCP at runtime for codex, cursor, and openhands (cursor's own binary installs via curl but still needs `npx`). openclaw declares `engines: >=22.19.0` — install Node 22+ when openclaw is in the fleet | +| Python 3.12+ for OpenHands (`uv tool install openhands --python 3.12`) | OpenHands CLI runtime | | Outbound HTTPS | model APIs, Cursor backend, lexmount CDP (wss) | | `LEXMOUNT_API_KEY` in `.env` | recommended browser path on servers (see below) | @@ -23,15 +24,15 @@ upgrading. - **Recommended on servers: `lexmount`** (default via `agents..active_browser`). The browser runs in the cloud; the server only needs outbound network. No - local Chrome required for the CDP-capable agents (claude-code/codex/cursor/openclaw) + local Chrome required for the CDP-capable agents (claude-code/codex/cursor/openhands/openclaw) in this mode — claude-code now opens the managed backend session and attaches Playwright MCP to its CDP endpoint (`--cdp-endpoint`), like the others. - **`browser_id=local`**: requires a local Chrome/Chromium plus headless-Linux - dependencies. For claude-code/codex/cursor, Playwright MCP downloads its own + dependencies. For claude-code/codex/cursor/openhands, Playwright MCP downloads its own browser on first run — pre-warm with `npx -y @playwright/mcp@latest --version` during image build to avoid first-task latency. - Cloud-native backends (`browser-use-cloud`, `skyvern-cloud`) are **not - supported** by CLI agents (no CDP endpoint). All four (claude-code/codex/cursor/openclaw) + supported** by CLI agents (no CDP endpoint). All CLI agents fail fast with a clear error rather than silently self-launching a local browser. ## Per-agent install and auth @@ -110,6 +111,25 @@ export PATH="$HOME/.local/bin:$PATH" # ensure on PATH for the bench pro `agent_metadata.reported_model` in each result; avoid `model_id: auto` (non-deterministic, breaks the experiments-dir / eval model match). +### openhands (verified: OpenHands CLI 1.16.0 / SDK 1.21.0) + +```bash +uv tool install openhands --python 3.12 +``` + +- Auth: `models.openhands.api_key/base_url/model_id` are passed via + `LLM_API_KEY`, `LLM_BASE_URL`, and `LLM_MODEL` with `--override-with-envs`. + OpenAI-compatible endpoints work; in local smoke testing `gpt-5.5` through + the LiteLLM gateway succeeded, while `gpt-5.4` returned 404 from that route. +- Browser: Playwright MCP via `npx`; managed CDP backends attach automatically. +- The agent writes a task-local `~/.openhands/mcp.json` by setting `HOME` and + `OH_PERSISTENCE_DIR` to a workspace-local directory. `UV_CACHE_DIR` remains + pointed at the operator cache so `uv tool run openhands` does not reinstall + OpenHands for every task. +- OpenHands may emit transient `AgentErrorEvent`s when the model retries a tool + call with the right schema; the adapter records final success when a later + agent message answers the task. + ### openclaw (verified: openclaw 2026.5.22) ```bash @@ -128,7 +148,7 @@ npm install -g openclaw | Variable | Used by | Notes | |---|---|---| -| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | codex (direct), openclaw (via config) | proxy endpoints OK | +| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | codex (direct), openclaw/openhands (via config) | proxy endpoints OK | | `ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL` | claude-code | | | `CURSOR_API_KEY` | cursor | Cursor key, not OpenAI | | `LEXMOUNT_API_KEY` / `LEXMOUNT_PROJECT_ID` | lexmount browser backend | recommended on servers | @@ -143,7 +163,8 @@ docker build --build-arg INSTALL_CLI_AGENTS=true -t bubench-cli . ``` This installs Node.js 22.x (openclaw requires >= 22.19), the -claude-code/codex/openclaw CLIs, cursor-agent (relocated to +claude-code/codex/openclaw CLIs, OpenHands as a uv tool, +cursor-agent (relocated to `/opt/cursor-agent` and symlinked into `/usr/local/bin` so the uid-1000 runtime user can execute it), and pre-warms the Playwright MCP download into a world-readable npm cache (`NPM_CONFIG_CACHE=/opt/npm-cache`). @@ -154,7 +175,7 @@ plus a `config.yaml`. Verified container invocations (image runs as uid 1000, whose home is `/home/bench`): ```bash -# openclaw / cursor — env-only auth: +# openclaw / cursor / openhands — env-only auth: docker run --rm --user 1000 -v "$PWD/.env:/app/.env:ro" bubench-cli \ uv run scripts/run.py --agent openclaw --data LexBench-Browser --mode single @@ -172,18 +193,20 @@ With API-key codex auth instead, set `models.codex.base_url` when using a proxy endpoint (the key alone routes to api.openai.com). With the lexmount browser path no Chrome is needed in the image for any of the -CLI agents (claude-code/codex/cursor/openclaw); **include Chrome/Chromium (and +CLI agents (claude-code/codex/cursor/openhands/openclaw); **include Chrome/Chromium (and headless deps) separately only if `browser_id=local` runs are planned**. ## Smoke verification (per agent, after deploy) ```bash # CLI self-checks -claude --version; codex --version; cursor-agent --version; openclaw --version +claude --version; codex --version; cursor-agent --version +openhands --version; openclaw --version # Real end-to-end check (one task, see AGENTS.md smoke policy) bubench run --agent codex --data LexBench-Browser --mode single bubench run --agent cursor --data LexBench-Browser --mode single +bubench run --agent openhands --data LexBench-Browser --mode single bubench run --agent openclaw --data LexBench-Browser --mode single # claude-code on lexmount needs no local Chrome; config.example.yaml pins # agents.claude-code.active_model: sonnet (an Anthropic model): @@ -192,7 +215,7 @@ bubench run --agent claude-code --data LexBench-Browser --mode single Verify by log, not exit code: -- All four CLI agents on a managed backend: `run.log` shows +- CLI agents on a managed backend: `run.log` shows `Lexmount session created: wss://...` plus the agent's model-call lines. - Any `browser_id=local` run: no Lexmount line is expected — look for the Playwright MCP browser starting and the agent's tool calls. diff --git a/tests/browseruse_bench/test_openhands_agent.py b/tests/browseruse_bench/test_openhands_agent.py new file mode 100644 index 0000000..6a4db0d --- /dev/null +++ b/tests/browseruse_bench/test_openhands_agent.py @@ -0,0 +1,259 @@ +"""Tests for OpenHandsAgent: JSONL parsing, MCP config, and run_task.""" + +from __future__ import annotations + +import contextlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from browseruse_bench.agents.openhands import OPENHANDS_BROWSER_RULES, OpenHandsAgent, _parse_jsonl +from browseruse_bench.browsers.types import BrowserSessionContext +from browseruse_bench.schemas import AgentResult + + +def _line(obj: dict[str, Any]) -> str: + return json.dumps(obj) + "\n" + + +TASK_INFO: dict[str, Any] = { + "task_id": "t1", + "task_text": "Go to example.com", + "url": "https://example.com", +} + +AGENT_CONFIG: dict[str, Any] = { + "model_id": "gpt-test", + "timeout": 10, + "api_key": "test-key", + "base_url": "https://proxy.example/v1", +} + + +class TestParseJsonl: + def test_empty_input(self) -> None: + answer, items, error = _parse_jsonl([]) + assert answer == "" + assert items == [] + assert error is None + + def test_final_answer_and_browser_action_parsed(self) -> None: + lines = [ + _line({ + "type": "action", + "action": { + "tool": "browser_navigate", + "arguments": {"url": "https://example.com"}, + }, + }), + _line({"type": "finish", "final_answer": "The price is $42"}), + ] + answer, items, error = _parse_jsonl(lines) + assert answer == "The price is $42" + assert items[0]["type"] == "mcp_tool_call" + assert items[0]["tool"] == "browser_navigate" + assert items[0]["arguments"] == {"url": "https://example.com"} + assert error is None + + def test_openhands_action_event_and_agent_message_parsed(self) -> None: + lines = [ + _line({ + "kind": "AgentErrorEvent", + "source": "agent", + "tool_name": "browser_navigate", + "error": "first attempt used wrong schema", + }), + _line({ + "kind": "ActionEvent", + "source": "agent", + "tool_name": "browser_navigate", + "action": {"kind": "MCPToolAction", "data": {"url": "https://example.com"}}, + }), + _line({ + "kind": "MessageEvent", + "source": "agent", + "llm_message": { + "role": "assistant", + "content": [{"type": "text", "text": "The price is $42"}], + }, + }), + ] + answer, items, error = _parse_jsonl(lines) + assert answer == "The price is $42" + assert items == [{ + "type": "mcp_tool_call", + "tool": "browser_navigate", + "arguments": {"url": "https://example.com"}, + "status": "completed", + }] + assert error is None + + def test_command_action_normalized(self) -> None: + _, items, _ = _parse_jsonl([_line({"type": "action", "action": {"command": "ls"}})]) + assert items[0]["type"] == "command_execution" + assert items[0]["command"] == "ls" + + def test_error_event_captured(self) -> None: + _, _, error = _parse_jsonl([_line({"type": "error", "error": {"message": "bad key"}})]) + assert error == "bad key" + + def test_conversation_error_event_captured(self) -> None: + _, _, error = _parse_jsonl([ + _line({ + "source": "environment", + "code": "NotFoundError", + "detail": "model group not found", + "kind": "ConversationErrorEvent", + }) + ]) + assert error == "model group not found" + + def test_invalid_lines_skipped(self) -> None: + answer, _, _ = _parse_jsonl(["not json\n", "{broken\n", _line({"type": "result", "result": "ok"})]) + assert answer == "ok" + + +class TestOpenHandsAgentRunTask: + def _stream(self) -> list[str]: + return [ + _line({ + "type": "action", + "action": { + "tool": "browser_navigate", + "arguments": {"url": "https://example.com"}, + }, + }), + _line({"type": "finish", "final_answer": "The price is $42"}), + ] + + def test_successful_run_returns_answer( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + agent = OpenHandsAgent() + monkeypatch.setattr(agent, "_run_subprocess", lambda *a, **kw: (0, self._stream(), None)) + result = agent.run_task(TASK_INFO, AGENT_CONFIG, tmp_path) + assert isinstance(result, AgentResult) + assert result.answer == "The price is $42" + assert result.env_status == "success" + assert result.agent_done == "done" + assert result.metrics.steps == 1 + assert result.action_history == ["Navigate to https://example.com"] + assert "browser automation agent" not in OPENHANDS_BROWSER_RULES + + def test_workspace_config_and_env_written( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + captured_env: dict[str, str] = {} + + def fake_run(cmd: list[str], **kw: Any) -> tuple[int, list[str], None]: + captured_env.update(kw.get("env") or {}) + return 0, self._stream(), None + + agent = OpenHandsAgent() + monkeypatch.setattr(agent, "_run_subprocess", fake_run) + agent.run_task(TASK_INFO, AGENT_CONFIG, tmp_path) + + mcp_config = json.loads( + (tmp_path / ".openhands-home" / ".openhands" / "mcp.json").read_text() + ) + server = mcp_config["mcpServers"]["playwright"] + assert server["command"] == "npx" + assert "@playwright/mcp@latest" in server["args"] + assert captured_env["HOME"] == str(tmp_path / ".openhands-home") + assert captured_env["OH_PERSISTENCE_DIR"] == str( + tmp_path / ".openhands-home" / ".openhands" + ) + assert captured_env["OPENHANDS_SUPPRESS_BANNER"] == "1" + assert captured_env["UV_CACHE_DIR"] + assert captured_env["LLM_MODEL"] == "gpt-test" + assert captured_env["LLM_API_KEY"] == "test-key" + assert captured_env["LLM_BASE_URL"] == "https://proxy.example/v1" + + def test_command_flags(self) -> None: + cmd = OpenHandsAgent._build_command("do it") + assert cmd[:3] == ["openhands", "--headless", "--json"] + assert "--override-with-envs" in cmd + assert cmd[cmd.index("--task") + 1] == "do it" + + def test_command_can_use_uv_tool_run_prefix(self) -> None: + cmd = OpenHandsAgent._build_command( + "do it", {"openhands_command": ["uv", "tool", "run", "openhands"]} + ) + assert cmd[:6] == ["uv", "tool", "run", "openhands", "--headless", "--json"] + + def test_timeout_keeps_partial_answer( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + agent = OpenHandsAgent() + monkeypatch.setattr( + agent, + "_run_subprocess", + lambda *a, **kw: (-1, [_line({"type": "finish", "final_answer": "4.5 stars"})], "Timeout after 10 seconds"), + ) + result = agent.run_task(TASK_INFO, AGENT_CONFIG, tmp_path) + assert result.env_status == "success" + assert result.agent_done == "timeout" + assert "4.5 stars" in result.answer + + def test_executable_not_found_returns_error_result( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + agent = OpenHandsAgent() + + def _raise(*a: Any, **kw: Any) -> None: + raise FileNotFoundError("openhands not found") + + monkeypatch.setattr(agent, "_run_subprocess", _raise) + result = agent.run_task(TASK_INFO, AGENT_CONFIG, tmp_path) + assert result.env_status == "failed" + assert result.agent_done == "error" + assert "not found" in (result.error or "").lower() + + def test_managed_browser_opens_backend_session( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + from browseruse_bench.agents import openhands as openhands_module + + opened: dict[str, str] = {} + + @contextlib.contextmanager + def fake_session(browser_id: str, agent_name: str, agent_config: dict[str, Any]): + opened["browser_id"] = browser_id + yield BrowserSessionContext( + backend_id=browser_id, transport="cdp", cdp_url="ws://cdp.example/1" + ) + + monkeypatch.setattr(openhands_module, "open_browser_session", fake_session) + agent = OpenHandsAgent() + monkeypatch.setattr(agent, "_run_subprocess", lambda *a, **kw: (0, self._stream(), None)) + result = agent.run_task(TASK_INFO, {**AGENT_CONFIG, "browser_id": "lexmount"}, tmp_path) + mcp_config = json.loads( + (tmp_path / ".openhands-home" / ".openhands" / "mcp.json").read_text() + ) + args = mcp_config["mcpServers"]["playwright"]["args"] + assert opened["browser_id"] == "lexmount" + assert "--cdp-endpoint" in args + assert "ws://cdp.example/1" in args + assert result.env_status == "success" + + def test_non_cdp_backend_fails_fast( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + from browseruse_bench.agents import openhands as openhands_module + + @contextlib.contextmanager + def fake_session(browser_id: str, agent_name: str, agent_config: dict[str, Any]): + yield BrowserSessionContext(backend_id=browser_id, transport="cloud_native") + + monkeypatch.setattr(openhands_module, "open_browser_session", fake_session) + agent = OpenHandsAgent() + monkeypatch.setattr( + agent, "_run_subprocess", lambda *a, **kw: pytest.fail("subprocess must not run") + ) + result = agent.run_task( + TASK_INFO, {**AGENT_CONFIG, "browser_id": "browser-use-cloud"}, tmp_path + ) + assert result.env_status == "failed" + assert "browser-use-cloud" in (result.error or "")