From 32653f1c11dab4b526ef294e63c63b4c95b0ae6b Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Wed, 5 Aug 2026 16:55:43 -0700 Subject: [PATCH] feat(evals): run v4_code through shared MCP --- .../evals/framework/claudeCodeToolAdapter.ts | 163 +++++++++++++++++- .../framework/claudeCodeToolAdapter.test.ts | 135 +++++++++++---- 2 files changed, 260 insertions(+), 38 deletions(-) diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index a806b2fb80..8ed7472645 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -2,9 +2,11 @@ import fs from "node:fs"; import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import type * as StagehandSdk from "@browserbasehq/stagehand"; import { executeStagehandSnippet, + STAGEHAND_CODEMODE_REFERENCE, STAGEHAND_CODEMODE_SKILL, } from "@browserbasehq/stagehand-integrations/codemode"; import matter from "gray-matter"; @@ -104,6 +106,9 @@ const ALLOW_UNSANDBOXED_LOCAL_ENV = "EVAL_CLAUDE_CODE_ALLOW_UNSANDBOXED_LOCAL"; const V4_CODE_STAGEHAND_MODEL_ENV = "EVAL_V4_CODE_STAGEHAND_MODEL"; const RUN_TOOL_SERVER = "stagehand_browser"; const RUN_TOOL_NAME = `mcp__${RUN_TOOL_SERVER}__run`; +const V4_CODE_TOOL_NAME = `mcp__${RUN_TOOL_SERVER}__code_execute`; +const V4_CODE_STDIO_EXPORT = "@browserbasehq/stagehand-integrations/codemode/stdio-server"; +const V4_CODE_SKILL_NAME = "stagehand-v4-code"; type ClaudeToolResult = { content: Array<{ type: "text"; text: string }>; @@ -206,12 +211,17 @@ export async function prepareClaudeCodeToolAdapter( startupProfile, }); case "v4_code_deterministic": - case "v4_code": return prepareV4CodeAdapter({ ...input, toolSurface, startupProfile, }); + case "v4_code": + return prepareV4CodeMcpAdapter({ + ...input, + toolSurface, + startupProfile, + }); default: throw new EvalsError( `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, v4_code_deterministic, or v4_code for execution right now; received "${toolSurface}".`, @@ -240,7 +250,18 @@ export function resolveClaudeCodeStartupProfile( environment: "LOCAL" | "BROWSERBASE", requested?: StartupProfile, ): StartupProfile { - if (toolSurface === "v4_code_deterministic" || toolSurface === "v4_code") { + if (toolSurface === "v4_code") { + const expected = + environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; + if (requested && requested !== expected) { + throw new EvalsError( + `${toolSurface} requires startup profile "${expected}" in ${environment}; received "${requested}".`, + ); + } + return expected; + } + + if (toolSurface === "v4_code_deterministic") { if (environment !== "LOCAL") { throw new EvalsError(`${toolSurface} currently supports only the LOCAL environment.`); } @@ -268,6 +289,38 @@ export function resolveClaudeCodeStartupProfile( ); } +export function resolveV4CodeStdioEntrypoint(): string { + return fileURLToPath(import.meta.resolve(V4_CODE_STDIO_EXPORT)); +} + +export function buildV4CodeMcpServerConfig( + environment: "LOCAL" | "BROWSERBASE", + agentModel: AvailableModel | undefined, + sourceEnv: NodeJS.ProcessEnv = process.env, +): Record { + const childEnv = copyStringEnvironment(sourceEnv); + childEnv.STAGEHAND_BROWSER = environment === "BROWSERBASE" ? "browserbase" : "local"; + + const stagehandModel = resolveV4CodeStagehandModel(agentModel, sourceEnv); + if (stagehandModel) childEnv.STAGEHAND_MODEL_NAME = stagehandModel; + + return { + type: "stdio", + command: process.execPath, + args: [resolveV4CodeStdioEntrypoint()], + env: childEnv, + alwaysLoad: true, + }; +} + +function copyStringEnvironment(sourceEnv: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(sourceEnv).filter((entry): entry is [string, string] => { + return typeof entry[1] === "string"; + }), + ); +} + async function prepareBrowseCliAdapter( input: ClaudeCodeToolAdapterInput & { toolSurface: "browse_cli"; @@ -613,6 +666,73 @@ async function prepareCdpCodeAdapter( } } +async function prepareV4CodeMcpAdapter( + input: ClaudeCodeToolAdapterInput & { + toolSurface: "v4_code"; + startupProfile: StartupProfile; + }, +): Promise { + if (!input.model) { + throw new EvalsError("v4_code requires the selected harness model."); + } + + const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-v4-code-")); + const env = copyStringEnvironment(process.env); + const stagehandModel = resolveV4CodeStagehandModel(input.model); + const mcpServers = { + [RUN_TOOL_SERVER]: buildV4CodeMcpServerConfig(input.environment, input.model), + }; + + await installV4CodeSkill(cwd); + + input.logger.log({ + category: "claude_code", + message: `Configured ${input.environment.toLowerCase()} v4_code through the shared Stagehand code-mode stdio MCP.`, + level: 1, + auxiliary: { + startupProfile: { + value: input.startupProfile, + type: "string", + }, + environment: { + value: input.environment, + type: "string", + }, + ...(stagehandModel + ? { + stagehandModel: { + value: stagehandModel, + type: "string" as const, + }, + } + : {}), + }, + }); + + return { + toolSurface: input.toolSurface, + startupProfile: input.startupProfile, + cwd, + env, + allowedTools: ["Skill", "Bash", V4_CODE_TOOL_NAME], + settingSources: ["project"], + mcpServers, + canUseTool: async (toolName, commandInput) => { + if (toolName === "Skill" || toolName === V4_CODE_TOOL_NAME || toolName === "Bash") { + return { behavior: "allow", updatedInput: commandInput }; + } + return { + behavior: "deny", + message: `Use Bash for inspection and ${V4_CODE_TOOL_NAME} for V4 browser automation.`, + }; + }, + promptInstructions: buildV4CodeMcpPromptInstructions(), + cleanup: async () => { + await fsp.rm(cwd, { recursive: true, force: true }); + }, + }; +} + async function prepareV4CodeAdapter( input: ClaudeCodeToolAdapterInput & { toolSurface: "v4_code_deterministic" | "v4_code"; @@ -1321,6 +1441,45 @@ function buildCdpCodePromptInstructions(plan: ExternalHarnessTaskPlan): string { ].join("\n"); } +function buildV4CodeMcpPromptInstructions(): string { + return [ + "Browser tool surface: v4_code.", + `A project skill named ${V4_CODE_SKILL_NAME} is available. Use the Skill tool to load it before writing Stagehand code; consult its REFERENCE.md when you need the exact API surface.`, + `Use the ${V4_CODE_TOOL_NAME} tool for browser automation. It is the shared Stagehand code-mode MCP tool.`, + "The tool exposes one long-lived Stagehand V4 page, context, Stagehand instance, Zod as z, and console across code calls.", + "Use page and context for deterministic browser operations. Use stagehand.act(), stagehand.observe(), and stagehand.extract() for AI-assisted browser operations.", + "The benchmark Start URL is written in this task prompt; it is not injected as a startUrl variable. Navigate to that literal URL when the task requires it.", + "Do not initialize or close Stagehand or its browser. The MCP process owns browser provisioning and cleanup.", + "Use Bash for inspection and lightweight scripting. Do not create a separate browser process.", + "Do not edit repository files.", + "Return useful JSON-serializable values from code snippets so you can inspect progress.", + ].join("\n"); +} + +async function installV4CodeSkill(cwd: string): Promise { + const targetDir = path.join(cwd, ".claude", "skills", V4_CODE_SKILL_NAME); + await fsp.mkdir(targetDir, { recursive: true }); + await Promise.all([ + fsp.writeFile( + path.join(targetDir, "SKILL.md"), + [ + "---", + `name: ${V4_CODE_SKILL_NAME}`, + "description: Write correct Stagehand V4 code for the shared code_execute MCP tool.", + "---", + "", + STAGEHAND_CODEMODE_SKILL, + "", + "## Detailed API reference", + "", + "Read `REFERENCE.md` in this skill directory when the short guide does not answer an API question.", + "", + ].join("\n"), + ), + fsp.writeFile(path.join(targetDir, "REFERENCE.md"), `${STAGEHAND_CODEMODE_REFERENCE}\n`), + ]); +} + function buildV4CodePromptInstructions( toolSurface: "v4_code_deterministic" | "v4_code", plan: ExternalHarnessTaskPlan, diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 6562f53654..af6e4937df 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -4,13 +4,14 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { AvailableModel } from "stagehand-v3"; import { - executeV4AiSnippet, + buildV4CodeMcpServerConfig, executeV4DeterministicSnippet, getBrowseCliAllowedTools, getBrowseCliToolMetadata, insertAfterFrontmatter, isAllowedBrowseCommand, installBrowseSkill, + prepareClaudeCodeToolAdapter, resolveClaudeCodeStartupProfile, resolveClaudeCodeToolSurface, resolveV4CodeStagehandModel, @@ -72,15 +73,108 @@ describe("claude code tool adapter resolution", () => { ); }); - it("supports AI-enabled V4 only as a local tool-launched surface", () => { + it("maps v4_code startup to the shared MCP browser environment", () => { expect(resolveClaudeCodeToolSurface("v4_code")).toBe("v4_code"); expect(resolveClaudeCodeStartupProfile("v4_code", "LOCAL")).toBe("tool_launch_local"); + expect(resolveClaudeCodeStartupProfile("v4_code", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); expect(() => resolveClaudeCodeStartupProfile("v4_code", "LOCAL", "runner_provided_local_cdp"), - ).toThrow(/requires startup profile "tool_launch_local"/); - expect(() => resolveClaudeCodeStartupProfile("v4_code", "BROWSERBASE")).toThrow( - /supports only the LOCAL environment/, + ).toThrow(/requires startup profile "tool_launch_local" in LOCAL/); + expect(() => + resolveClaudeCodeStartupProfile("v4_code", "BROWSERBASE", "tool_launch_local"), + ).toThrow(/requires startup profile "tool_create_browserbase" in BROWSERBASE/); + }); + + it("configures the shared stdio MCP explicitly for local and Browserbase runs", () => { + const local = buildV4CodeMcpServerConfig( + "LOCAL", + "anthropic/claude-sonnet-5" as AvailableModel, + { + PATH: "/test/bin", + BROWSERBASE_API_KEY: "test-browserbase-key", + EVAL_V4_CODE_STAGEHAND_MODEL: "groq/openai/gpt-oss-120b", + }, + ); + const remote = buildV4CodeMcpServerConfig( + "BROWSERBASE", + "anthropic/claude-sonnet-5" as AvailableModel, + { + PATH: "/test/bin", + BROWSERBASE_API_KEY: "test-browserbase-key", + }, ); + + expect(local).toMatchObject({ + type: "stdio", + command: process.execPath, + args: [expect.stringMatching(/integrations[/\\]dist[/\\]codemode[/\\]stdio-server\.mjs$/u)], + alwaysLoad: true, + env: { + PATH: "/test/bin", + BROWSERBASE_API_KEY: "test-browserbase-key", + STAGEHAND_BROWSER: "local", + STAGEHAND_MODEL_NAME: "groq/openai/gpt-oss-120b", + }, + }); + expect(remote).toMatchObject({ + env: { + STAGEHAND_BROWSER: "browserbase", + STAGEHAND_MODEL_NAME: "anthropic/claude-sonnet-5", + }, + }); + }); + + it("prepares v4_code as one shared MCP tool without initializing Stagehand in evals", async () => { + const adapter = await prepareClaudeCodeToolAdapter({ + toolSurface: "v4_code", + startupProfile: "tool_create_browserbase", + environment: "BROWSERBASE", + plan: { + dataset: "webvoyager", + taskId: "task-1", + startUrl: "https://example.com", + instruction: "Inspect the example page", + }, + logger: new EvalLogger(false), + model: "anthropic/claude-sonnet-5" as AvailableModel, + }); + + try { + expect(adapter.allowedTools).toEqual([ + "Skill", + "Bash", + "mcp__stagehand_browser__code_execute", + ]); + expect(adapter.settingSources).toEqual(["project"]); + expect(adapter.mcpServers).toMatchObject({ + stagehand_browser: { + type: "stdio", + command: process.execPath, + alwaysLoad: true, + env: { STAGEHAND_BROWSER: "browserbase" }, + }, + }); + expect(adapter.promptInstructions).toContain("shared Stagehand code-mode MCP tool"); + expect(adapter.promptInstructions).toContain("project skill named stagehand-v4-code"); + expect(adapter.promptInstructions).toContain("not injected as a startUrl variable"); + await expect(adapter.canUseTool?.("Skill", { skill: "stagehand-v4-code" })).resolves.toEqual({ + behavior: "allow", + updatedInput: { skill: "stagehand-v4-code" }, + }); + await expect( + fsp.readFile(path.join(adapter.cwd, ".claude/skills/stagehand-v4-code/SKILL.md"), "utf8"), + ).resolves.toContain("# Stagehand V4 code-mode syntax"); + await expect( + fsp.readFile( + path.join(adapter.cwd, ".claude/skills/stagehand-v4-code/REFERENCE.md"), + "utf8", + ), + ).resolves.toContain("# Stagehand V4 code-mode reference"); + } finally { + await adapter.cleanup(); + } }); it("allows the Stagehand operation model to differ from the Claude Code model", () => { @@ -149,37 +243,6 @@ describe("claude code tool adapter resolution", () => { ]); }); - it("adds native Stagehand and Zod bindings only for the AI-enabled V4 surface", async () => { - const logger = new EvalLogger(false); - const stagehand = { - act: async (instruction: string) => ({ instruction, success: true }), - }; - - const result = await executeV4AiSnippet({ - code: ` - const schema = z.object({ heading: z.string() }); - return { - action: await stagehand.act("click the link"), - parsed: schema.parse({ heading: "Example Domain" }), - }; - `, - stagehand: stagehand as never, - page: {} as never, - context: {} as never, - plan: { - dataset: "webvoyager", - startUrl: "https://example.com", - instruction: "Inspect the example page", - }, - logger, - }); - - expect(result).toEqual({ - action: { instruction: "click the link", success: true }, - parsed: { heading: "Example Domain" }, - }); - }); - it("supports browse_cli as the first Codex tool surface", () => { expect(resolveCodexToolSurface()).toBe("browse_cli"); expect(resolveCodexToolSurface("browse_cli")).toBe("browse_cli");