From 5f50ba23db52e930b3fc8fb6172e3a3653f185fd Mon Sep 17 00:00:00 2001 From: miguel Date: Sun, 16 Aug 2026 10:15:20 -0700 Subject: [PATCH] feat(evals): add stagehand_facade tool surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New via:"mcp" surface whose mount spawns the shipped facade stdio server (@browserbasehq/stagehand-integrations/facade/stdio-server) with an allowlisted STAGEHAND_*/BROWSERBASE_* env and the canonical FACADE_AGENT_INSTRUCTIONS. Evals now benchmarks the byte-identical surface the claude-code/codex/pi integrations ship, alongside stagehand_code rather than replacing it. The facade server owns its browser (tool_launch_local / tool_create_browserbase profiles); no runner-side captureEvidence — verification relies on the tool_result stream, matching the other external-MCP surfaces. --- packages/evals/ARCHITECTURE.mmd | 2 +- packages/evals/core/contracts/tool.ts | 1 + packages/evals/core/tools/registry.ts | 4 + packages/evals/core/tools/stagehand_facade.ts | 135 ++++++++++++++++++ .../evals/framework/claudeCodeToolAdapter.ts | 25 ++-- packages/evals/framework/codexToolAdapter.ts | 22 ++- packages/evals/framework/context.ts | 1 + .../evals/tests/core/stagehand-facade.test.ts | 97 +++++++++++++ packages/evals/tui/commands/help.ts | 4 +- 9 files changed, 271 insertions(+), 20 deletions(-) create mode 100644 packages/evals/core/tools/stagehand_facade.ts create mode 100644 packages/evals/tests/core/stagehand-facade.test.ts diff --git a/packages/evals/ARCHITECTURE.mmd b/packages/evals/ARCHITECTURE.mmd index 0920aa7f20..b18c86df9e 100644 --- a/packages/evals/ARCHITECTURE.mmd +++ b/packages/evals/ARCHITECTURE.mmd @@ -47,7 +47,7 @@ flowchart TB CoreContext["framework/context.ts
buildCoreContext"] FixtureServer["core/fixtures
local deterministic pages"] CoreTargets["core/targets
local Chrome
Browserbase CDP"] - CoreTools["core/tools registry
understudy_code
playwright_code
cdp_code
playwright_mcp
chrome_devtools_mcp
browse_cli"] + CoreTools["core/tools registry
understudy_code
playwright_code
cdp_code
playwright_mcp
chrome_devtools_mcp
stagehand_facade
browse_cli"] CoreAssertions["assertions + metrics
adapter-backed results"] CoreDeps["core/runtime/coreDeps.ts
browserbase + ws
lazy require"] end diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index 565c499f31..2971bb7662 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -11,6 +11,7 @@ export type ToolSurface = | "cdp_code" | "playwright_mcp" | "chrome_devtools_mcp" + | "stagehand_facade" | "browse_cli"; export type StartupProfile = diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index 7a14bce997..68a7225c21 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -5,6 +5,7 @@ import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js"; import { PlaywrightCodeTool } from "./playwright_code.js"; import { PlaywrightMcpTool } from "./playwright_mcp.js"; import { StagehandCodeTool } from "./stagehand_code.js"; +import { StagehandFacadeTool } from "./stagehand_facade.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; export function listCoreTools(): ToolSurface[] { @@ -15,6 +16,7 @@ export function listCoreTools(): ToolSurface[] { "cdp_code", "playwright_mcp", "chrome_devtools_mcp", + "stagehand_facade", "browse_cli", ]; } @@ -33,6 +35,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { return new PlaywrightMcpTool(); case "chrome_devtools_mcp": return new ChromeDevtoolsMcpTool(); + case "stagehand_facade": + return new StagehandFacadeTool(); case "browse_cli": return new BrowseCliTool(); default: diff --git a/packages/evals/core/tools/stagehand_facade.ts b/packages/evals/core/tools/stagehand_facade.ts new file mode 100644 index 0000000000..ae63231ece --- /dev/null +++ b/packages/evals/core/tools/stagehand_facade.ts @@ -0,0 +1,135 @@ +import { FACADE_AGENT_INSTRUCTIONS } from "@browserbasehq/stagehand-integrations/facade"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; +import { fileURLToPath } from "node:url"; +import type { Artifact, ConnectionMode } from "../contracts/results.js"; +import type { + CoreCapability, + CorePageHandle, + CoreSession, + CoreTool, + StartupProfile, + ToolStartInput, + ToolStartResult, +} from "../contracts/tool.js"; +import type { TargetKind } from "../contracts/targets.js"; + +const SUPPORTED_CAPABILITIES: CoreCapability[] = [ + "session", + "navigation", + "evaluation", + "screenshot", + "viewport", + "wait", + "click", + "hover", + "scroll", + "type", + "press", + "tabs", + "representation", +]; + +const serverPath = fileURLToPath( + import.meta.resolve("@browserbasehq/stagehand-integrations/facade/stdio-server"), +); + +function unsupportedSessionOperation(): never { + throw new Error("stagehand_facade is available only through its agent MCP mount"); +} + +/** + * The facade process and its browser are spawned by the agent harness. This + * placeholder satisfies the CoreTool lifecycle contract without launching a + * second, unobservable browser solely for the runner-side session. + */ +class StagehandFacadeMountSession implements CoreSession { + async listPages(): Promise { + return unsupportedSessionOperation(); + } + + async activePage(): Promise { + return unsupportedSessionOperation(); + } + + async newPage(): Promise { + return unsupportedSessionOperation(); + } + + async selectPage(): Promise { + unsupportedSessionOperation(); + } + + async closePage(): Promise { + unsupportedSessionOperation(); + } + + async close(): Promise {} + + async getArtifacts(): Promise { + return []; + } + + async getRawMetrics(): Promise> { + return {}; + } +} + +export function buildStagehandFacadeEnv( + environment: ToolStartInput["environment"], +): Record { + return { + ...buildAllowlistedEnv(), + STAGEHAND_BROWSER: environment === "BROWSERBASE" ? "browserbase" : "local", + }; +} + +function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMode { + return startupProfile === "tool_create_browserbase" ? "browserbase_native" : "launch"; +} + +export class StagehandFacadeTool implements CoreTool { + readonly id = "stagehand_facade"; + readonly surface = "mcp"; + readonly family = "stagehand"; + readonly supportedStartupProfiles: StartupProfile[] = [ + "tool_launch_local", + "tool_create_browserbase", + ]; + readonly supportedCapabilities: CoreCapability[] = [...SUPPORTED_CAPABILITIES]; + readonly supportedTargetKinds: TargetKind[] = ["selector", "coords", "focused", "snapshot_ref"]; + + async start(input: ToolStartInput): Promise { + const expectedProfile = + input.environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; + if (input.startupProfile !== expectedProfile) { + throw new Error( + `stagehand_facade startup profile "${input.startupProfile}" is not valid for environment "${input.environment}"`, + ); + } + + const session = new StagehandFacadeMountSession(); + return { + session, + agentMount: { + via: "mcp", + promptInstructions: FACADE_AGENT_INSTRUCTIONS, + mcpServers: { + stagehand: { + command: process.execPath, + args: [serverPath], + env: buildStagehandFacadeEnv(input.environment), + }, + }, + }, + cleanup: async () => { + await session.close(); + }, + metadata: { + environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", + browserOwnership: "tool", + connectionMode: connectionModeFromProfile(input.startupProfile), + startupProfile: input.startupProfile, + }, + }; + } +} diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 0c3e06bcfb..54b6a33674 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -178,7 +178,8 @@ export async function prepareClaudeCodeToolAdapter( case "cdp_code": case "stagehand_code": case "playwright_mcp": - case "chrome_devtools_mcp": { + case "chrome_devtools_mcp": + case "stagehand_facade": { return prepareMountedCoreToolAdapter({ ...input, toolSurface, @@ -187,7 +188,7 @@ export async function prepareClaudeCodeToolAdapter( } default: throw new EvalsError( - `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp for execution right now; received "${toolSurface}".`, + `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp, with stagehand_facade also available; received "${toolSurface}".`, ); } } @@ -200,12 +201,13 @@ export function resolveClaudeCodeToolSurface(requested?: ToolSurface): ToolSurfa requested === "cdp_code" || requested === "stagehand_code" || requested === "playwright_mcp" || - requested === "chrome_devtools_mcp" + requested === "chrome_devtools_mcp" || + requested === "stagehand_facade" ) { return requested; } throw new EvalsError( - `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp for execution right now; received "${requested}".`, + `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp, with stagehand_facade also available; received "${requested}".`, ); } @@ -216,14 +218,17 @@ export function resolveClaudeCodeStartupProfile( ): StartupProfile { if (requested) return requested; - // browse_cli and stagehand_code own their browser (the Stagehand SDK launches or - // creates it via the extension stack), so no runner-provided CDP endpoint. - if (toolSurface === "browse_cli" || toolSurface === "stagehand_code") { + // browse_cli, stagehand_code, and stagehand_facade own their browser (the + // Stagehand SDK launches or creates it), so no runner-provided CDP endpoint. + if ( + toolSurface === "browse_cli" || + toolSurface === "stagehand_code" || + toolSurface === "stagehand_facade" + ) { return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; } - // The MCP surfaces need a runner-provided endpoint so the harness-side - // session (evidence capture) and the agent's own server instance attach to - // the same browser. + // The attachable surfaces need a runner-provided endpoint so the harness-side + // session (evidence capture) and the agent's server instance share a browser. if ( toolSurface === "playwright_code" || toolSurface === "cdp_code" || diff --git a/packages/evals/framework/codexToolAdapter.ts b/packages/evals/framework/codexToolAdapter.ts index ab790236c4..5628489c9b 100644 --- a/packages/evals/framework/codexToolAdapter.ts +++ b/packages/evals/framework/codexToolAdapter.ts @@ -52,7 +52,11 @@ export interface PreparedCodexCodeAdapter { export type PreparedCodexToolAdapter = PreparedBrowseCliHarnessAdapter | PreparedCodexCodeAdapter; const CODE_SURFACES = new Set(["stagehand_code", "playwright_code", "cdp_code"]); -const MCP_SURFACES = new Set(["playwright_mcp", "chrome_devtools_mcp"]); +const MCP_SURFACES = new Set([ + "playwright_mcp", + "chrome_devtools_mcp", + "stagehand_facade", +]); /** Mirrors the claude adapter's bounded, best-effort terminal capture. */ function boundedCaptureEvidence( @@ -281,7 +285,7 @@ export function resolveCodexToolSurface(requested?: ToolSurface): ToolSurface { return requested; } throw new EvalsError( - `Codex harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp for execution right now; received "${requested}".`, + `Codex harness supports --tool browse_cli, playwright_code, cdp_code, stagehand_code, playwright_mcp, or chrome_devtools_mcp, with stagehand_facade also available; received "${requested}".`, ); } @@ -292,13 +296,17 @@ export function resolveCodexStartupProfile( ): StartupProfile { if (requested) return requested; - // browse_cli and stagehand_code own their browser; playwright/cdp attach to a - // runner-provided CDP endpoint (same defaults as the claude_code harness). - if (toolSurface === "browse_cli" || toolSurface === "stagehand_code") { + // browse_cli, stagehand_code, and stagehand_facade own their browser; + // playwright/cdp attach to a runner-provided CDP endpoint. + if ( + toolSurface === "browse_cli" || + toolSurface === "stagehand_code" || + toolSurface === "stagehand_facade" + ) { return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; } - // MCP surfaces need a runner-provided endpoint so the harness-side session - // (evidence capture) and the agent's own server instance share one browser. + // Attachable surfaces need a runner-provided endpoint so the harness-side + // session (evidence capture) and the agent's server instance share a browser. if ( toolSurface === "playwright_code" || toolSurface === "cdp_code" || diff --git a/packages/evals/framework/context.ts b/packages/evals/framework/context.ts index 9dbc43b47f..b2699a74e1 100644 --- a/packages/evals/framework/context.ts +++ b/packages/evals/framework/context.ts @@ -34,6 +34,7 @@ export function resolveDefaultCoreStartupProfile( ): StartupProfile { switch (toolSurface) { case "browse_cli": + case "stagehand_facade": return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; case "understudy_code": case "playwright_code": diff --git a/packages/evals/tests/core/stagehand-facade.test.ts b/packages/evals/tests/core/stagehand-facade.test.ts new file mode 100644 index 0000000000..039603f6d7 --- /dev/null +++ b/packages/evals/tests/core/stagehand-facade.test.ts @@ -0,0 +1,97 @@ +import { FACADE_AGENT_INSTRUCTIONS } from "@browserbasehq/stagehand-integrations/facade"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { getCoreTool, listCoreTools } from "../../core/tools/registry.js"; +import { buildStagehandFacadeEnv, StagehandFacadeTool } from "../../core/tools/stagehand_facade.js"; +import { + resolveClaudeCodeStartupProfile, + resolveClaudeCodeToolSurface, +} from "../../framework/claudeCodeToolAdapter.js"; +import { + resolveCodexStartupProfile, + resolveCodexToolSurface, +} from "../../framework/codexToolAdapter.js"; +import type { EvalLogger } from "../../logger.js"; + +const ORIGINAL_ENV = { ...process.env }; + +beforeEach(() => { + for (const key of Object.keys(process.env)) { + if (/^(STAGEHAND_|BROWSERBASE_)/u.test(key)) delete process.env[key]; + } +}); + +afterEach(() => { + for (const key of Object.keys(process.env)) { + if (!(key in ORIGINAL_ENV)) delete process.env[key]; + } + Object.assign(process.env, ORIGINAL_ENV); +}); + +describe("stagehand facade tool surface", () => { + it("is registered", () => { + expect(listCoreTools()).toContain("stagehand_facade"); + expect(getCoreTool("stagehand_facade")).toBeInstanceOf(StagehandFacadeTool); + }); + + it("builds the shipped facade MCP mount", async () => { + process.env.STAGEHAND_MODEL_NAME = "openai/gpt-5-mini"; + process.env.BROWSERBASE_API_KEY = "browserbase-secret"; + process.env.OPENAI_API_KEY = "must-not-cross-the-mount"; + + const running = await new StagehandFacadeTool().start({ + logger: {} as EvalLogger, + environment: "LOCAL", + startupProfile: "tool_launch_local", + }); + + expect(running.agentMount?.via).toBe("mcp"); + if (running.agentMount?.via !== "mcp") throw new Error("expected MCP mount"); + expect(running.agentMount.promptInstructions).toBe(FACADE_AGENT_INSTRUCTIONS); + expect(Object.keys(running.agentMount.mcpServers)).toEqual(["stagehand"]); + expect(running.agentMount.mcpServers.stagehand).toMatchObject({ + command: process.execPath, + args: [expect.stringMatching(/facade[/\\]stdio-server\.mjs$/u)], + env: { + STAGEHAND_BROWSER: "local", + STAGEHAND_MODEL_NAME: "openai/gpt-5-mini", + BROWSERBASE_API_KEY: "browserbase-secret", + }, + }); + expect( + (running.agentMount.mcpServers.stagehand as { env: Record }).env, + ).not.toHaveProperty("OPENAI_API_KEY"); + expect(running.captureEvidence).toBeUndefined(); + await running.cleanup(); + }); + + it("filters host env and overrides browser selection for each eval environment", () => { + process.env.STAGEHAND_BROWSER = "browserbase"; + process.env.STAGEHAND_MODEL_API_KEY = "model-secret"; + process.env.BROWSERBASE_PROJECT_ID = "project-id"; + process.env.ANTHROPIC_API_KEY = "must-not-cross-the-mount"; + + expect(buildStagehandFacadeEnv("LOCAL")).toEqual({ + STAGEHAND_BROWSER: "local", + STAGEHAND_MODEL_API_KEY: "model-secret", + BROWSERBASE_PROJECT_ID: "project-id", + }); + expect(buildStagehandFacadeEnv("BROWSERBASE")).toEqual({ + STAGEHAND_BROWSER: "browserbase", + STAGEHAND_MODEL_API_KEY: "model-secret", + BROWSERBASE_PROJECT_ID: "project-id", + }); + }); + + it("is supported by both agent harnesses with tool-owned startup profiles", () => { + expect(resolveClaudeCodeToolSurface("stagehand_facade")).toBe("stagehand_facade"); + expect(resolveClaudeCodeStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); + expect(resolveClaudeCodeStartupProfile("stagehand_facade", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); + expect(resolveCodexToolSurface("stagehand_facade")).toBe("stagehand_facade"); + expect(resolveCodexStartupProfile("stagehand_facade", "LOCAL")).toBe("tool_launch_local"); + expect(resolveCodexStartupProfile("stagehand_facade", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); + }); +}); diff --git a/packages/evals/tui/commands/help.ts b/packages/evals/tui/commands/help.ts index eb233a185e..f0b884df94 100644 --- a/packages/evals/tui/commands/help.ts +++ b/packages/evals/tui/commands/help.ts @@ -172,7 +172,7 @@ export function printConfigHelp(): void { row(`${cyan("reset")} ${dim("[key]")}`, "Reset one key or the whole core section"), row(cyan("setup"), `Interactive wizard ${gray("(coming soon)")}`), "", - ` ${bold("Valid core tools:")} ${gray("understudy_code, playwright_code, cdp_code, playwright_mcp, chrome_devtools_mcp, browse_cli")}`, + ` ${bold("Valid core tools:")} ${gray("understudy_code, stagehand_code, playwright_code, cdp_code, playwright_mcp, chrome_devtools_mcp, stagehand_facade, browse_cli")}`, "", ` ${bold("Examples:")}`, "", @@ -202,7 +202,7 @@ export function printConfigCoreHelp(): void { row(`${cyan("reset")} ${dim("[key]")}`, "Reset one key or the whole core section"), row(cyan("setup"), `Interactive wizard ${gray("(coming soon)")}`), "", - ` ${bold("Valid core tools:")} ${gray("understudy_code, playwright_code, cdp_code, playwright_mcp, chrome_devtools_mcp, browse_cli")}`, + ` ${bold("Valid core tools:")} ${gray("understudy_code, stagehand_code, playwright_code, cdp_code, playwright_mcp, chrome_devtools_mcp, stagehand_facade, browse_cli")}`, "", ` ${bold("Examples:")}`, "",