diff --git a/packages/integrations/README.md b/packages/integrations/README.md index 86cbf8043b..6aeb8b8618 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -1,5 +1,67 @@ # Stagehand integrations -Private workspace package for Stagehand integration adapters. +This package contains shared integration surfaces for Stagehand V4. It is private while the public API and packaging contract are validated. -The code-mode stdio entrypoint currently provides the MCP host and process lifecycle used by later code-mode capabilities. It intentionally advertises no tools yet. +## Code mode + +The `./codemode` export gives an agent one `code_execute` tool backed by a persistent Stagehand browser. Frameworks can either launch the thin local MCP server or wrap `StagehandCodeExecutor` as a native tool. + +```ts +import { + StagehandCodeExecutor, + stagehandCodeConfigFromEnv, +} from "@browserbasehq/stagehand-integrations/codemode"; + +const executor = new StagehandCodeExecutor(stagehandCodeConfigFromEnv()); + +try { + const result = await executor.execute({ + code: ` + await page.goto("https://example.com", { waitUntil: "domcontentloaded" }); + return { title: await page.title(), url: await page.url() }; + `, + }); + console.log(result); +} finally { + await executor.close(); +} +``` + +The executor initializes the browser on the first valid call, serializes calls, and preserves pages, cookies, and navigation state until its owner closes it. + +### Low-level eval integration + +Eval harnesses that already own Stagehand and browser initialization should call `executeStagehandSnippet` directly. This reuses the exact generated-code semantics without replacing the eval harness's startup, cleanup, task bindings, or metrics collection. + +### Local MCP integration + +The `./codemode/stdio-server` export is an internal process entrypoint. It is not a command-line interface and accepts no arguments. The owning framework launches one process per agent run and selects local or Browserbase startup through its environment: + +```text +STAGEHAND_BROWSER=local +STAGEHAND_BROWSER=browserbase +``` + +The process stays alive across calls and closes when its input stream ends. `SIGINT` and `SIGTERM` perform bounded graceful cleanup and preserve signal-style exit codes. If generated JavaScript blocks the JavaScript event loop, the server cannot run its cleanup handlers. The owner must terminate the entire process tree, escalate to `SIGKILL` after its own deadline, and start a new process before accepting more work. Killing only the Node process can leave its local browser child alive. + +### Configuration + +`stagehandCodeConfigFromEnv()` recognizes: + +| Variable | Purpose | +| ------------------------------------------------------------------ | ---------------------------------------------------------------- | +| `STAGEHAND_BROWSER` | Optional `local` or `browserbase` override | +| `BROWSERBASE_API_KEY` | Selects and authenticates Browserbase when present | +| `BROWSERBASE_PROJECT_ID` | Optional Browserbase project forwarded when creating a session | +| `STAGEHAND_MODEL_NAME` | Optional Stagehand model name | +| `STAGEHAND_MODEL_API_KEY` | Optional explicit model-provider key | +| Provider API keys | Supplies the key for a matching explicit model provider | +| `GEMINI_API_KEY`, `GOOGLE_GENERATIVE_AI_API_KEY`, `GOOGLE_API_KEY` | Selects `google/gemini-2.5-flash-lite` when no model is explicit | + +Without a browser override, the helper selects Browserbase when `BROWSERBASE_API_KEY` exists and a headless local browser otherwise. + +Native callers run generated JavaScript in their own process. An `AbortSignal` can cancel queued work before the snippet begins, but it cannot safely preempt arbitrary JavaScript already running in the same process. Native integrations that require hard time limits should put the executor behind a child-process boundary, as the stdio MCP integration does. + +### Security boundary + +The code-mode executor does not provide a sandbox. Generated JavaScript runs in the host process and inherits that process's filesystem, network, and environment access. A framework may place the tool inside its own sandbox, container, or other isolation boundary. diff --git a/packages/integrations/package.json b/packages/integrations/package.json index 264cfa56b9..a29ede83e4 100644 --- a/packages/integrations/package.json +++ b/packages/integrations/package.json @@ -8,6 +8,10 @@ ], "type": "module", "exports": { + "./codemode": { + "types": "./dist/codemode/index.d.mts", + "import": "./dist/codemode/index.mjs" + }, "./codemode/stdio-server": { "import": "./dist/codemode/stdio-server.mjs" } @@ -19,7 +23,9 @@ "typecheck": "tsc --noEmit -p tsconfig.json" }, "dependencies": { - "@modelcontextprotocol/sdk": "catalog:" + "@browserbasehq/stagehand": "workspace:*", + "@modelcontextprotocol/sdk": "catalog:", + "zod": "catalog:" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/integrations/src/codemode/config.ts b/packages/integrations/src/codemode/config.ts new file mode 100644 index 0000000000..6301845a71 --- /dev/null +++ b/packages/integrations/src/codemode/config.ts @@ -0,0 +1,109 @@ +import { StagehandClientCreateConfigSchema } from "@browserbasehq/stagehand"; +import type { StagehandCodeConfig } from "./types.js"; + +const ANTHROPIC_DIRECT_BROWSER_ACCESS_HEADER = "anthropic-dangerous-direct-browser-access"; + +class StagehandCodeConfigError extends Error { + override readonly name = "StagehandCodeConfigError"; +} + +export function stagehandCodeConfigFromEnv( + env: NodeJS.ProcessEnv = process.env, +): StagehandCodeConfig { + const requestedBrowser = nonEmpty(env.STAGEHAND_BROWSER)?.toLowerCase(); + if ( + requestedBrowser !== undefined && + requestedBrowser !== "local" && + requestedBrowser !== "browserbase" + ) { + throw new StagehandCodeConfigError( + 'STAGEHAND_BROWSER must be either "local" or "browserbase".', + ); + } + + const browserbaseApiKey = nonEmpty(env.BROWSERBASE_API_KEY); + const browserbaseProjectId = nonEmpty(env.BROWSERBASE_PROJECT_ID); + const browserType = requestedBrowser ?? (browserbaseApiKey ? "browserbase" : "local"); + if (browserType === "browserbase" && !browserbaseApiKey) { + throw new StagehandCodeConfigError( + 'BROWSERBASE_API_KEY is required when STAGEHAND_BROWSER="browserbase".', + ); + } + + const explicitModelName = nonEmpty(env.STAGEHAND_MODEL_NAME); + const explicitModelApiKey = nonEmpty(env.STAGEHAND_MODEL_API_KEY); + if (!explicitModelName && explicitModelApiKey) { + throw new StagehandCodeConfigError( + "STAGEHAND_MODEL_NAME is required when STAGEHAND_MODEL_API_KEY is set.", + ); + } + + const inferredGoogleKey = providerApiKey("google", env); + const modelName = + explicitModelName ?? (inferredGoogleKey ? "google/gemini-2.5-flash-lite" : undefined); + const modelProvider = modelName ? providerName(modelName) : undefined; + const modelApiKey = explicitModelApiKey ?? providerApiKey(modelProvider, env); + + const stagehand = StagehandClientCreateConfigSchema.parse({ + logging: { level: "off" }, + ...(modelName + ? { + model: { + modelName, + ...(modelApiKey ? { apiKey: modelApiKey } : {}), + ...(modelProvider === "anthropic" + ? { headers: { [ANTHROPIC_DIRECT_BROWSER_ACCESS_HEADER]: "true" } } + : {}), + }, + } + : {}), + }); + + return { + browser: + browserType === "browserbase" + ? { + type: "browserbase", + launchOptions: { + apiKey: browserbaseApiKey, + ...(browserbaseProjectId ? { projectId: browserbaseProjectId } : {}), + }, + } + : { + type: "local", + launchOptions: { headless: true }, + }, + stagehand, + }; +} + +function providerName(modelName: string): string | undefined { + const separator = modelName.indexOf("/"); + return separator === -1 ? undefined : modelName.slice(0, separator).toLowerCase(); +} + +function providerApiKey(provider: string | undefined, env: NodeJS.ProcessEnv): string | undefined { + switch (provider) { + case "openai": + return nonEmpty(env.OPENAI_API_KEY); + case "anthropic": + return nonEmpty(env.ANTHROPIC_API_KEY); + case "google": + return ( + nonEmpty(env.GOOGLE_GENERATIVE_AI_API_KEY) ?? + nonEmpty(env.GEMINI_API_KEY) ?? + nonEmpty(env.GOOGLE_API_KEY) + ); + case "groq": + return nonEmpty(env.GROQ_API_KEY); + case "cerebras": + return nonEmpty(env.CEREBRAS_API_KEY); + default: + return undefined; + } +} + +function nonEmpty(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} diff --git a/packages/integrations/src/codemode/executor.ts b/packages/integrations/src/codemode/executor.ts new file mode 100644 index 0000000000..413b1230c7 --- /dev/null +++ b/packages/integrations/src/codemode/executor.ts @@ -0,0 +1,342 @@ +import { + browserbase, + localBrowser, + Stagehand, + type Page, + type StagehandBrowser, + type StagehandMetrics, +} from "@browserbasehq/stagehand"; +import { MAX_CODE_BYTES } from "./limits.js"; +import { executeStagehandSnippet } from "./snippet.js"; +import type { + CodeExecuteFailure, + CodeExecuteInput, + CodeExecuteResult, + CodeLogEntry, + CodePageState, + StagehandCodeConfig, +} from "./types.js"; + +export type StagehandCodeExecutorOptions = StagehandCodeConfig; + +const MAX_LOG_BYTES = 64 * 1024; +const MAX_RESULT_BYTES = 256 * 1024; +const MAX_ERROR_MESSAGE_LENGTH = 4_000; +const MIN_SENSITIVE_VALUE_LENGTH = 8; +const SECRET_FIELD = /(?:api.?key|authorization|cookie|password|secret|token)/i; +const URL = /\b(?:https?|wss?):\/\/[^\s"'<>]+/gi; +const CREDENTIAL = + /\b(authorization|api[_-]?key|password|secret|token)\s*[:=]\s*(?:bearer\s+)?[^\s,;]+/gi; +const BEARER_TOKEN = /\bbearer\s+[^\s,;]+/gi; + +class StagehandCodeCloseError extends Error { + override readonly name = "StagehandCodeCloseError"; + + constructor() { + super("Failed to close Stagehand code mode."); + } +} + +class StagehandCodeInitializationError extends Error { + override readonly name = "StagehandCodeInitializationError"; + + constructor() { + super("Stagehand code mode initialization and browser cleanup both failed."); + } +} + +export class StagehandCodeExecutor { + private stagehand?: Stagehand; + private browser?: StagehandBrowser; + private queue = Promise.resolve(); + private closed = false; + private closePromise?: Promise; + private readonly sensitiveValues: string[]; + + constructor(private readonly options: StagehandCodeExecutorOptions) { + this.sensitiveValues = collectSensitiveValues(options); + } + + execute(input: CodeExecuteInput, signal?: AbortSignal): Promise { + const validation = validate(input); + if (validation) return Promise.resolve(validation); + + const operation = this.queue.then(() => this.executeQueued(input, signal)); + this.queue = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + metrics(): Promise { + const operation = this.queue.then(() => this.stagehand?.metrics()); + this.queue = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + close(): Promise { + this.closed = true; + this.closePromise ??= this.queue.then(async () => { + const stagehand = this.stagehand; + const browser = this.browser; + this.stagehand = undefined; + this.browser = undefined; + + let failed = false; + if (stagehand) { + await stagehand.close().catch(() => { + failed = true; + }); + } + if (browser) { + await browser.close().catch(() => { + failed = true; + }); + } + if (failed) throw new StagehandCodeCloseError(); + }); + return this.closePromise; + } + + private async executeQueued( + input: CodeExecuteInput, + signal?: AbortSignal, + ): Promise { + if (this.closed) { + return failure("closed", "Code executor is closed."); + } + if (signal?.aborted) { + return failure("aborted", "Code execution was aborted before it began."); + } + + const logs: CodeLogEntry[] = []; + let page: Page | undefined; + try { + const stagehand = await this.ensureStagehand(); + const context = stagehand.browser.context; + page = + (await context.activePage()) ?? (await context.pages())[0] ?? (await context.newPage()); + + if (signal?.aborted) { + return failure("aborted", "Code execution was aborted before it began.", "CodeModeError", { + page: await readPageState(page), + }); + } + + const value = await executeStagehandSnippet({ + code: input.code, + page, + context, + stagehand, + console: createCodeConsole(logs), + }); + const currentPage = (await context.activePage()) ?? page; + + return { + ok: true, + page: await readPageState(currentPage), + ...(value === undefined ? {} : { value: jsonSafe(value) }), + ...(logs.length === 0 ? {} : { logs }), + }; + } catch (error) { + const normalized = normalizeError(error, this.sensitiveValues); + const currentPage = (await this.activePage().catch(() => undefined)) ?? page; + return failure("runtime", normalized.message, normalized.name, { + ...(currentPage ? { page: await readPageState(currentPage).catch(() => undefined) } : {}), + ...(logs.length === 0 ? {} : { logs }), + }); + } + } + + private async ensureStagehand(): Promise { + if (this.stagehand) return this.stagehand; + + const browserConfig = this.options.browser; + const browser = + browserConfig.type === "browserbase" + ? await browserbase.launch(browserConfig.launchOptions) + : await localBrowser.launch(browserConfig.launchOptions); + + try { + const stagehand = await Stagehand.create({ + browser, + logging: { level: "off" }, + ...this.options.stagehand, + }); + this.browser = browser; + this.stagehand = stagehand; + return stagehand; + } catch (error) { + try { + await browser.close(); + } catch { + throw new StagehandCodeInitializationError(); + } + throw error; + } + } + + private async activePage(): Promise { + if (!this.stagehand) return undefined; + return ( + (await this.stagehand.browser.context.activePage()) ?? + (await this.stagehand.browser.context.pages())[0] + ); + } +} + +function validate(input: CodeExecuteInput): CodeExecuteFailure | undefined { + if (!input || typeof input.code !== "string" || input.code.trim().length === 0) { + return failure("validation", "code must be a non-empty JavaScript function body."); + } + if (Buffer.byteLength(input.code) > MAX_CODE_BYTES) { + return failure("validation", `code must be at most ${MAX_CODE_BYTES} UTF-8 bytes.`); + } + return undefined; +} + +function createCodeConsole(logs: CodeLogEntry[]) { + let logBytes = 0; + const append = (level: CodeLogEntry["level"], values: unknown[]) => { + if (logBytes >= MAX_LOG_BYTES) return; + const text = formatLog(values); + const remaining = MAX_LOG_BYTES - logBytes; + const bounded = truncateUtf8(text, remaining); + if (bounded.length === 0) { + if (text.length > 0) logBytes = MAX_LOG_BYTES; + return; + } + logBytes += Buffer.byteLength(bounded); + logs.push({ level, text: bounded }); + }; + return Object.freeze({ + log: (...values: unknown[]) => append("log", values), + warn: (...values: unknown[]) => append("warn", values), + error: (...values: unknown[]) => append("error", values), + }); +} + +async function readPageState(page: Page): Promise { + const [url, title] = await Promise.all([page.url(), page.title()]); + return { url, title }; +} + +function jsonSafe(value: unknown): unknown { + if (value === undefined) return undefined; + const serialized = JSON.stringify(value, (_key, nested) => { + if (typeof nested === "bigint") return nested.toString(); + if (nested instanceof Uint8Array) { + return { + type: "bytes", + encoding: "base64", + data: Buffer.from(nested).toString("base64"), + }; + } + return nested; + }); + if (serialized === undefined) return undefined; + const bytes = Buffer.byteLength(serialized); + if (bytes <= MAX_RESULT_BYTES) return JSON.parse(serialized); + return { + truncated: true, + original_bytes: bytes, + preview: truncateUtf8(serialized, MAX_RESULT_BYTES), + }; +} + +function truncateUtf8(value: string, maxBytes: number): string { + if (maxBytes <= 0) return ""; + if (Buffer.byteLength(value) <= maxBytes) return value; + + const characters: string[] = []; + let bytes = 0; + for (const character of value) { + const characterBytes = Buffer.byteLength(character); + if (bytes + characterBytes > maxBytes) break; + characters.push(character); + bytes += characterBytes; + } + return characters.join(""); +} + +function formatLog(values: unknown[]): string { + return values + .map((value) => { + if (typeof value === "string") return value; + try { + const safe = jsonSafe(value); + return safe === undefined ? String(value) : JSON.stringify(safe); + } catch { + return "[Unserializable value]"; + } + }) + .join(" "); +} + +function normalizeError( + error: unknown, + sensitiveValues: string[], +): { name: string; message: string } { + if (!(error instanceof Error)) { + return { name: "Error", message: "Code execution failed with a non-Error value." }; + } + + const safeName = /^[A-Za-z_$][A-Za-z0-9_$.-]{0,99}$/.test(error.name) ? error.name : "Error"; + return { + name: safeName, + message: sanitizeErrorMessage(error.message, sensitiveValues), + }; +} + +function sanitizeErrorMessage(message: string, sensitiveValues: string[]): string { + let sanitized = message; + // Remove complete configured secrets before pattern redaction and final truncation. + for (const sensitiveValue of sensitiveValues) { + sanitized = sanitized.replaceAll(sensitiveValue, "[REDACTED]"); + } + sanitized = sanitized + .replace(URL, "[REDACTED_URL]") + .replace(CREDENTIAL, "$1=[REDACTED]") + .replace(BEARER_TOKEN, "Bearer [REDACTED]"); + return sanitized.slice(0, MAX_ERROR_MESSAGE_LENGTH) || "Code execution failed."; +} + +function collectSensitiveValues(value: unknown): string[] { + const values = new Set(); + const seen = new WeakMap(); + + const visit = (current: unknown, key = "", parentIsSensitive = false) => { + const isSensitive = parentIsSensitive || SECRET_FIELD.test(key); + if (typeof current === "string") { + if (isSensitive && current.length >= MIN_SENSITIVE_VALUE_LENGTH) values.add(current); + return; + } + if (!current || typeof current !== "object") return; + const previousSensitivity = seen.get(current); + if (previousSensitivity === true || (previousSensitivity === false && !isSensitive)) return; + seen.set(current, isSensitive); + for (const [nestedKey, nestedValue] of Object.entries(current)) { + visit(nestedValue, nestedKey, isSensitive); + } + }; + + visit(value); + return [...values].sort((left, right) => right.length - left.length); +} + +function failure( + kind: CodeExecuteFailure["error"]["kind"], + message: string, + name = "CodeModeError", + evidence: Pick = {}, +): CodeExecuteFailure { + return { + ok: false, + ...evidence, + error: { kind, name, message }, + }; +} diff --git a/packages/integrations/src/codemode/index.ts b/packages/integrations/src/codemode/index.ts new file mode 100644 index 0000000000..9f0611f390 --- /dev/null +++ b/packages/integrations/src/codemode/index.ts @@ -0,0 +1,23 @@ +export { stagehandCodeConfigFromEnv } from "./config.js"; +export { StagehandCodeExecutor, type StagehandCodeExecutorOptions } from "./executor.js"; +export { connectCodeModeStdio, createCodeModeMcp, createCodeModeMcpServer } from "./mcp-server.js"; +export { executeStagehandSnippet } from "./snippet.js"; +export { + CODE_EXECUTE_DESCRIPTION, + codeExecuteResultText, + codeExecuteSchema, +} from "./tool-contract.js"; +export type { + CodeExecuteErrorKind, + CodeExecuteFailure, + CodeExecuteInput, + CodeExecuteResult, + CodeExecuteSuccess, + CodeLogEntry, + CodePageState, + ExecuteStagehandSnippetInput, + StagehandCodeBrowserConfig, + StagehandCodeConfig, + StagehandSnippetBindings, + StagehandSnippetConsole, +} from "./types.js"; diff --git a/packages/integrations/src/codemode/limits.ts b/packages/integrations/src/codemode/limits.ts new file mode 100644 index 0000000000..624fd5e789 --- /dev/null +++ b/packages/integrations/src/codemode/limits.ts @@ -0,0 +1 @@ +export const MAX_CODE_BYTES = 100_000; diff --git a/packages/integrations/src/codemode/mcp-server.ts b/packages/integrations/src/codemode/mcp-server.ts new file mode 100644 index 0000000000..6862e8cca1 --- /dev/null +++ b/packages/integrations/src/codemode/mcp-server.ts @@ -0,0 +1,50 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StagehandCodeExecutor, type StagehandCodeExecutorOptions } from "./executor.js"; +import * as codeModeMcpRuntime from "./mcp-runtime.js"; +import { + CODE_EXECUTE_DESCRIPTION, + codeExecuteOutputSchema, + codeExecuteResultText, + codeExecuteSchema, +} from "./tool-contract.js"; +import type { CodeExecuteInput, CodeExecuteResult } from "./types.js"; + +export function createCodeModeMcpServer(executor: StagehandCodeExecutor): McpServer { + const server = codeModeMcpRuntime.createCodeModeMcpHost(); + server.registerTool( + "code_execute", + { + title: "Execute Stagehand V4 code", + description: CODE_EXECUTE_DESCRIPTION, + inputSchema: codeExecuteSchema.shape, + outputSchema: codeExecuteOutputSchema, + }, + async (input, extra) => { + const result = await executor.execute(input as CodeExecuteInput, extra.signal); + return mcpResult(result); + }, + ); + return server; +} + +export async function connectCodeModeStdio(executor: StagehandCodeExecutor): Promise { + const server = createCodeModeMcpServer(executor); + await codeModeMcpRuntime.connectCodeModeStdio(server); + return server; +} + +export function createCodeModeMcp(options: StagehandCodeExecutorOptions): { + executor: StagehandCodeExecutor; + server: McpServer; +} { + const executor = new StagehandCodeExecutor(options); + return { executor, server: createCodeModeMcpServer(executor) }; +} + +function mcpResult(result: CodeExecuteResult) { + return { + content: [{ type: "text" as const, text: codeExecuteResultText(result) }], + structuredContent: result as unknown as Record, + isError: !result.ok, + }; +} diff --git a/packages/integrations/src/codemode/snippet.ts b/packages/integrations/src/codemode/snippet.ts new file mode 100644 index 0000000000..279c0e4a22 --- /dev/null +++ b/packages/integrations/src/codemode/snippet.ts @@ -0,0 +1,48 @@ +import { z } from "zod/v4"; +import type { ExecuteStagehandSnippetInput } from "./types.js"; + +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new ( + ...args: string[] +) => (...values: unknown[]) => Promise; + +const RESERVED_BINDINGS = new Set(["page", "context", "stagehand", "z", "console"]); +const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/; + +export async function executeStagehandSnippet( + input: ExecuteStagehandSnippetInput, +): Promise { + const bindings = Object.entries(input.bindings ?? {}); + for (const [name] of bindings) { + if (!isAsyncFunctionParameter(name)) { + throw new TypeError(`Code-mode binding "${name}" is not a valid JavaScript identifier.`); + } + if (RESERVED_BINDINGS.has(name)) { + throw new TypeError(`Code-mode binding "${name}" is reserved.`); + } + } + + const parameters: Array<[string, unknown]> = [ + ["page", input.page], + ["context", input.context], + ...(input.stagehand + ? ([ + ["stagehand", input.stagehand], + ["z", z], + ] as Array<[string, unknown]>) + : []), + ...bindings, + ["console", input.console ?? console], + ]; + const fn = new AsyncFunction(...parameters.map(([name]) => name), input.code); + return await fn(...parameters.map(([, value]) => value)); +} + +function isAsyncFunctionParameter(name: string): boolean { + if (!IDENTIFIER.test(name)) return false; + try { + new AsyncFunction(name, ""); + return true; + } catch { + return false; + } +} diff --git a/packages/integrations/src/codemode/stdio-server.ts b/packages/integrations/src/codemode/stdio-server.ts index 4d93cb4050..d673f96929 100644 --- a/packages/integrations/src/codemode/stdio-server.ts +++ b/packages/integrations/src/codemode/stdio-server.ts @@ -1,13 +1,17 @@ -import { connectCodeModeStdio, createCodeModeMcpHost } from "./mcp-runtime.js"; +import { stagehandCodeConfigFromEnv } from "./config.js"; +import { StagehandCodeExecutor } from "./executor.js"; +import { createCodeModeMcpServer } from "./mcp-server.js"; +import { connectCodeModeStdio } from "./mcp-runtime.js"; import { closeCodeModeStdio } from "./stdio-lifecycle.js"; -const server = createCodeModeMcpHost(); +const executor = new StagehandCodeExecutor(stagehandCodeConfigFromEnv()); +const server = createCodeModeMcpServer(executor); let closing = false; async function shutdown(code: number): Promise { if (closing) return; closing = true; - const clean = await closeCodeModeStdio([server]); + const clean = await closeCodeModeStdio([server, executor]); if (!clean) { process.stderr.write("Failed to close Stagehand code mode cleanly.\n"); } @@ -20,4 +24,4 @@ process.stdin.once("end", () => void shutdown(0)); process.stdin.once("close", () => void shutdown(0)); await connectCodeModeStdio(server); -process.stderr.write("Stagehand code-mode MCP host listening on stdio\n"); +process.stderr.write("Stagehand code-mode MCP listening on stdio\n"); diff --git a/packages/integrations/src/codemode/tool-contract.ts b/packages/integrations/src/codemode/tool-contract.ts new file mode 100644 index 0000000000..d6024e22bd --- /dev/null +++ b/packages/integrations/src/codemode/tool-contract.ts @@ -0,0 +1,72 @@ +import { z } from "zod/v4"; +import { MAX_CODE_BYTES } from "./limits.js"; +import type { CodeExecuteResult } from "./types.js"; + +export const CODE_EXECUTE_DESCRIPTION = [ + "Execute an async JavaScript function body against one long-lived Stagehand V4 browser.", + "The executor lazily creates a local or Browserbase browser on the first call and reuses it for later calls.", + "The executor itself is not a security sandbox. The owning framework may run it inside a sandbox or another isolation boundary.", + "If execution stops responding, the owning framework should terminate and restart the local tool process.", +].join("\n"); + +export const codeExecuteSchema = z.object({ + code: z + .string() + .refine((code) => code.trim().length > 0, "code must contain JavaScript source") + .refine( + (code) => new TextEncoder().encode(code).byteLength <= MAX_CODE_BYTES, + `code must be at most ${MAX_CODE_BYTES} UTF-8 bytes`, + ) + .describe( + "Async JavaScript function body. page, context, stagehand, z, and console are in scope.", + ), +}); + +export const codeExecuteOutputSchema = z + .object({ + ok: z.boolean(), + page: z + .object({ + url: z.string(), + title: z.string(), + }) + .optional(), + value: z.unknown().optional(), + logs: z + .array( + z.object({ + level: z.enum(["log", "warn", "error"]), + text: z.string(), + }), + ) + .optional(), + error: z + .object({ + kind: z.enum(["validation", "runtime", "aborted", "closed"]), + name: z.string(), + message: z.string(), + }) + .optional(), + }) + .superRefine((result, context) => { + if (result.ok) { + if (!result.page) { + context.addIssue({ code: "custom", message: "successful results require page state" }); + } + if (result.error) { + context.addIssue({ code: "custom", message: "successful results cannot include an error" }); + } + return; + } + + if (!result.error) { + context.addIssue({ code: "custom", message: "failed results require an error" }); + } + if (result.value !== undefined) { + context.addIssue({ code: "custom", message: "failed results cannot include a value" }); + } + }); + +export function codeExecuteResultText(result: CodeExecuteResult): string { + return JSON.stringify(result, null, 2); +} diff --git a/packages/integrations/src/codemode/types.ts b/packages/integrations/src/codemode/types.ts new file mode 100644 index 0000000000..41f7e31dc8 --- /dev/null +++ b/packages/integrations/src/codemode/types.ts @@ -0,0 +1,72 @@ +import type { + BrowserbaseLaunchOptions, + BrowserContext, + LocalBrowserLaunchOptions, + Page, + Stagehand, + StagehandClientCreateConfig, +} from "@browserbasehq/stagehand"; + +export type CodeExecuteInput = { + code: string; +}; + +export type CodePageState = { + url: string; + title: string; +}; + +export type CodeLogEntry = { + level: "log" | "warn" | "error"; + text: string; +}; + +export type CodeExecuteErrorKind = "validation" | "runtime" | "aborted" | "closed"; + +export type CodeExecuteSuccess = { + ok: true; + page: CodePageState; + value?: unknown; + logs?: CodeLogEntry[]; +}; + +export type CodeExecuteFailure = { + ok: false; + page?: CodePageState; + logs?: CodeLogEntry[]; + error: { + kind: CodeExecuteErrorKind; + name: string; + message: string; + }; +}; + +export type CodeExecuteResult = CodeExecuteSuccess | CodeExecuteFailure; + +export type StagehandSnippetConsole = Pick; + +export type StagehandSnippetBindings = Record; + +export type ExecuteStagehandSnippetInput = { + code: string; + page: Page; + context: BrowserContext; + stagehand?: Stagehand; + bindings?: StagehandSnippetBindings; + console?: StagehandSnippetConsole; +}; + +export type StagehandCodeBrowserConfig = + | { + type: "local"; + launchOptions?: LocalBrowserLaunchOptions; + } + | { + type: "browserbase"; + launchOptions: BrowserbaseLaunchOptions; + }; + +export type StagehandCodeConfig = { + browser: StagehandCodeBrowserConfig; + stagehand?: StagehandClientCreateConfig; +}; diff --git a/packages/integrations/tests/config.test.ts b/packages/integrations/tests/config.test.ts new file mode 100644 index 0000000000..dcc982bfc6 --- /dev/null +++ b/packages/integrations/tests/config.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from "vitest"; +import { stagehandCodeConfigFromEnv } from "../src/codemode/config.js"; + +describe("stagehandCodeConfigFromEnv", () => { + it("defaults to a headless local browser without Browserbase credentials", () => { + expect(stagehandCodeConfigFromEnv({})).toMatchObject({ + browser: { type: "local", launchOptions: { headless: true } }, + stagehand: { logging: { level: "off" } }, + }); + }); + + it("lets an explicit local selection win when Browserbase credentials are present", () => { + expect( + stagehandCodeConfigFromEnv({ + STAGEHAND_BROWSER: " local ", + BROWSERBASE_API_KEY: "bb_secret", + BROWSERBASE_PROJECT_ID: "project-id", + }).browser, + ).toStrictEqual({ type: "local", launchOptions: { headless: true } }); + }); + + it("forwards Browserbase API key and project ID", () => { + expect( + stagehandCodeConfigFromEnv({ + STAGEHAND_BROWSER: "browserbase", + BROWSERBASE_API_KEY: " bb_secret ", + BROWSERBASE_PROJECT_ID: " project-id ", + }).browser, + ).toStrictEqual({ + type: "browserbase", + launchOptions: { apiKey: "bb_secret", projectId: "project-id" }, + }); + }); + + it("omits a blank Browserbase project ID", () => { + expect( + stagehandCodeConfigFromEnv({ + BROWSERBASE_API_KEY: "bb_secret", + BROWSERBASE_PROJECT_ID: " ", + }).browser, + ).toStrictEqual({ + type: "browserbase", + launchOptions: { apiKey: "bb_secret" }, + }); + }); + + it("rejects invalid or unauthenticated Browserbase selections", () => { + for (const [env, message] of [ + [ + { STAGEHAND_BROWSER: "remote" }, + 'STAGEHAND_BROWSER must be either "local" or "browserbase".', + ], + [ + { STAGEHAND_BROWSER: "browserbase" }, + 'BROWSERBASE_API_KEY is required when STAGEHAND_BROWSER="browserbase".', + ], + ] as const) { + try { + stagehandCodeConfigFromEnv(env); + throw new Error("expected configuration to be rejected"); + } catch (error) { + expect(error).toMatchObject({ name: "StagehandCodeConfigError", message }); + } + } + }); + + it.each([ + ["openai/gpt-5.4-mini", "OPENAI_API_KEY", "openai-key"], + ["anthropic/claude-sonnet-4-6", "ANTHROPIC_API_KEY", "anthropic-key"], + ["google/gemini-3-flash-preview", "GEMINI_API_KEY", "google-key"], + ["groq/llama-3.3-70b-versatile", "GROQ_API_KEY", "groq-key"], + ["cerebras/llama3.1-8b", "CEREBRAS_API_KEY", "cerebras-key"], + ])("pairs an explicit %s model with its provider key", (modelName, envName, apiKey) => { + const config = stagehandCodeConfigFromEnv({ + STAGEHAND_MODEL_NAME: modelName, + [envName]: apiKey, + }); + + expect(config.stagehand?.model).toStrictEqual({ + modelName, + apiKey, + ...(modelName.startsWith("anthropic/") + ? { + headers: { + "anthropic-dangerous-direct-browser-access": "true", + }, + } + : {}), + }); + }); + + it("prefers the explicit model key over provider environment keys", () => { + const config = stagehandCodeConfigFromEnv({ + STAGEHAND_MODEL_NAME: "openai/gpt-5.4-mini", + STAGEHAND_MODEL_API_KEY: "explicit-key", + OPENAI_API_KEY: "provider-key", + }); + + expect(config.stagehand?.model).toStrictEqual({ + modelName: "openai/gpt-5.4-mini", + apiKey: "explicit-key", + }); + }); + + it("uses the eval-native Google key precedence", () => { + const config = stagehandCodeConfigFromEnv({ + STAGEHAND_MODEL_NAME: "google/gemini-3-flash-preview", + GEMINI_API_KEY: "gemini-key", + GOOGLE_GENERATIVE_AI_API_KEY: "generative-key", + GOOGLE_API_KEY: "google-key", + }); + + expect(config.stagehand?.model).toStrictEqual({ + modelName: "google/gemini-3-flash-preview", + apiKey: "generative-key", + }); + }); + + it("infers the default Google model only when no model is explicit", () => { + const config = stagehandCodeConfigFromEnv({ + GOOGLE_GENERATIVE_AI_API_KEY: "generative-key", + }); + + expect(config.stagehand?.model).toStrictEqual({ + modelName: "google/gemini-2.5-flash-lite", + apiKey: "generative-key", + }); + }); + + it("does not apply Google credentials to an explicit non-Google model", () => { + const config = stagehandCodeConfigFromEnv({ + STAGEHAND_MODEL_NAME: "anthropic/claude-sonnet-4-6", + GEMINI_API_KEY: "google-key", + }); + + expect(config.stagehand?.model).toStrictEqual({ + modelName: "anthropic/claude-sonnet-4-6", + headers: { + "anthropic-dangerous-direct-browser-access": "true", + }, + }); + }); + + it("rejects an explicit model key without a model name", () => { + try { + stagehandCodeConfigFromEnv({ STAGEHAND_MODEL_API_KEY: "orphan-key" }); + throw new Error("expected configuration to be rejected"); + } catch (error) { + expect(error).toMatchObject({ + name: "StagehandCodeConfigError", + message: "STAGEHAND_MODEL_NAME is required when STAGEHAND_MODEL_API_KEY is set.", + }); + } + }); +}); diff --git a/packages/integrations/tests/executor.test.ts b/packages/integrations/tests/executor.test.ts new file mode 100644 index 0000000000..eff7956826 --- /dev/null +++ b/packages/integrations/tests/executor.test.ts @@ -0,0 +1,434 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { StagehandCodeConfig } from "../src/codemode/types.js"; + +const sdkMocks = vi.hoisted(() => ({ + browserbaseLaunch: vi.fn(), + localLaunch: vi.fn(), + stagehandCreate: vi.fn(), +})); + +vi.mock("@browserbasehq/stagehand", () => ({ + browserbase: { launch: sdkMocks.browserbaseLaunch }, + localBrowser: { launch: sdkMocks.localLaunch }, + Stagehand: { create: sdkMocks.stagehandCreate }, +})); + +const { StagehandCodeExecutor } = await import("../src/codemode/executor.js"); + +type Deferred = { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +}; + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function localConfig(stagehand: Record = {}): StagehandCodeConfig { + return { + browser: { type: "local", launchOptions: { headless: true } }, + stagehand: stagehand as never, + }; +} + +function fakeRuntime() { + const page = { + url: vi.fn(async () => "https://example.com"), + title: vi.fn(async () => "Example"), + hold: vi.fn(async () => undefined), + sideEffect: vi.fn(() => "side-effect"), + }; + const context = { + activePage: vi.fn(async () => page), + pages: vi.fn(async () => [page]), + newPage: vi.fn(async () => page), + }; + const stagehand = { + browser: { context }, + close: vi.fn(async () => undefined), + metrics: vi.fn(async () => ({ act: { prompt_tokens: 1 } })), + }; + const browser = { close: vi.fn(async () => undefined) }; + return { page, context, stagehand, browser }; +} + +describe("StagehandCodeExecutor", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("validates code before launching a browser", async () => { + const executor = new StagehandCodeExecutor(localConfig()); + + await expect(executor.execute({ code: " " })).resolves.toMatchObject({ + ok: false, + error: { kind: "validation" }, + }); + await expect(executor.execute({ code: "é".repeat(50_001) })).resolves.toMatchObject({ + ok: false, + error: { kind: "validation", message: expect.stringContaining("100000 UTF-8 bytes") }, + }); + expect(sdkMocks.localLaunch).not.toHaveBeenCalled(); + }); + + it("lazily launches once, reuses state, and closes both owners once", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect(executor.execute({ code: "return 1;" })).resolves.toMatchObject({ + ok: true, + value: 1, + page: { url: "https://example.com", title: "Example" }, + }); + await expect(executor.execute({ code: "return 2;" })).resolves.toMatchObject({ + ok: true, + value: 2, + }); + + expect(sdkMocks.localLaunch).toHaveBeenCalledOnce(); + expect(sdkMocks.localLaunch).toHaveBeenCalledWith({ headless: true }); + expect(sdkMocks.stagehandCreate).toHaveBeenCalledOnce(); + await executor.close(); + await executor.close(); + expect(runtime.stagehand.close).toHaveBeenCalledOnce(); + expect(runtime.browser.close).toHaveBeenCalledOnce(); + }); + + it("reports lifecycle cleanup failures without retaining underlying errors", async () => { + const runtime = fakeRuntime(); + runtime.stagehand.close.mockRejectedValue(new Error("stagehand close secret")); + runtime.browser.close.mockRejectedValue(new Error("browser close secret")); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + await executor.execute({ code: "return 1;" }); + const error = await executor.close().then( + () => undefined, + (reason: unknown) => reason, + ); + + expect(error).toMatchObject({ + name: "StagehandCodeCloseError", + message: "Failed to close Stagehand code mode.", + }); + expect(error).not.toBeInstanceOf(AggregateError); + expect(error).not.toHaveProperty("errors"); + expect(String(error)).not.toContain("secret"); + }); + + it("forwards Browserbase launch options", async () => { + const runtime = fakeRuntime(); + sdkMocks.browserbaseLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor({ + browser: { + type: "browserbase", + launchOptions: { apiKey: "bb_secret", projectId: "project-id" }, + }, + }); + + await executor.execute({ code: "return 1;" }); + + expect(sdkMocks.browserbaseLaunch).toHaveBeenCalledWith({ + apiKey: "bb_secret", + projectId: "project-id", + }); + expect(sdkMocks.localLaunch).not.toHaveBeenCalled(); + }); + + it("serializes concurrent calls in FIFO order", async () => { + const runtime = fakeRuntime(); + const gate = deferred(); + runtime.page.hold.mockImplementationOnce(() => gate.promise); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + const first = executor.execute({ code: 'await page.hold(); return "first";' }); + const second = executor.execute({ code: 'return "second";' }); + await vi.waitFor(() => expect(runtime.page.hold).toHaveBeenCalledOnce()); + expect(await Promise.race([second.then(() => "settled"), Promise.resolve("queued")])).toBe( + "queued", + ); + + gate.resolve(); + + await expect(first).resolves.toMatchObject({ ok: true, value: "first" }); + await expect(second).resolves.toMatchObject({ ok: true, value: "second" }); + }); + + it("cancels queued work before its snippet begins", async () => { + const runtime = fakeRuntime(); + const gate = deferred(); + runtime.page.hold.mockImplementationOnce(() => gate.promise); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + const controller = new AbortController(); + + const first = executor.execute({ code: "await page.hold();" }); + const second = executor.execute({ code: "return page.sideEffect();" }, controller.signal); + await vi.waitFor(() => expect(runtime.page.hold).toHaveBeenCalledOnce()); + controller.abort(); + gate.resolve(); + + await first; + await expect(second).resolves.toMatchObject({ ok: false, error: { kind: "aborted" } }); + expect(runtime.page.sideEffect).not.toHaveBeenCalled(); + }); + + it("rechecks cancellation after lazy browser initialization", async () => { + const runtime = fakeRuntime(); + const launch = deferred(); + sdkMocks.localLaunch.mockReturnValue(launch.promise); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + const controller = new AbortController(); + + const result = executor.execute({ code: "return page.sideEffect();" }, controller.signal); + await vi.waitFor(() => expect(sdkMocks.localLaunch).toHaveBeenCalledOnce()); + controller.abort(); + launch.resolve(runtime.browser); + + await expect(result).resolves.toMatchObject({ + ok: false, + page: { url: "https://example.com", title: "Example" }, + error: { kind: "aborted" }, + }); + expect(runtime.page.sideEffect).not.toHaveBeenCalled(); + }); + + it("marks queued work closed and drains before cleanup", async () => { + const runtime = fakeRuntime(); + const gate = deferred(); + runtime.page.hold.mockImplementationOnce(() => gate.promise); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + const active = executor.execute({ code: "await page.hold();" }); + const queued = executor.execute({ code: "return page.sideEffect();" }); + await vi.waitFor(() => expect(runtime.page.hold).toHaveBeenCalledOnce()); + const close = executor.close(); + gate.resolve(); + + await active; + await expect(queued).resolves.toMatchObject({ ok: false, error: { kind: "closed" } }); + await close; + expect(runtime.page.sideEffect).not.toHaveBeenCalled(); + expect(runtime.stagehand.close).toHaveBeenCalledOnce(); + expect(runtime.browser.close).toHaveBeenCalledOnce(); + await expect(executor.execute({ code: "return 1;" })).resolves.toMatchObject({ + ok: false, + error: { kind: "closed" }, + }); + }); + + it("queues metrics behind execution", async () => { + const runtime = fakeRuntime(); + const gate = deferred(); + runtime.page.hold.mockImplementationOnce(() => gate.promise); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + const active = executor.execute({ code: "await page.hold();" }); + const metrics = executor.metrics(); + await vi.waitFor(() => expect(runtime.page.hold).toHaveBeenCalledOnce()); + expect(runtime.stagehand.metrics).not.toHaveBeenCalled(); + gate.resolve(); + + await active; + await expect(metrics).resolves.toStrictEqual({ act: { prompt_tokens: 1 } }); + }); + + it("closes a launched browser when Stagehand initialization fails", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockRejectedValue(new Error("initialization failed")); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect(executor.execute({ code: "return 1;" })).resolves.toMatchObject({ + ok: false, + error: { kind: "runtime", message: "initialization failed" }, + }); + expect(runtime.browser.close).toHaveBeenCalledOnce(); + }); + + it("reports a generic aggregate when initialization and cleanup both fail", async () => { + const runtime = fakeRuntime(); + runtime.browser.close.mockRejectedValue(new Error("browser close secret")); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockRejectedValue(new Error("init secret")); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect(executor.execute({ code: "return 1;" })).resolves.toMatchObject({ + ok: false, + error: { + kind: "runtime", + name: "StagehandCodeInitializationError", + message: "Stagehand code mode initialization and browser cleanup both failed.", + }, + }); + }); + + it("normalizes JSON values and bounds returned output", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect( + executor.execute({ code: "return { big: 12n, bytes: new Uint8Array([1, 2, 3]) };" }), + ).resolves.toMatchObject({ + ok: true, + value: { + big: "12", + bytes: { type: "bytes", encoding: "base64", data: "AQID" }, + }, + }); + const large = await executor.execute({ code: 'return "x".repeat(300000);' }); + expect(large).toMatchObject({ + ok: true, + value: { truncated: true, original_bytes: 300_002 }, + }); + if (large.ok && typeof large.value === "object" && large.value) { + expect( + Buffer.byteLength(String((large.value as { preview: string }).preview)), + ).toBeLessThanOrEqual(256 * 1024); + } + + const multibyte = await executor.execute({ code: 'return "é".repeat(200000);' }); + expect(multibyte).toMatchObject({ ok: true, value: { truncated: true } }); + if (multibyte.ok && typeof multibyte.value === "object" && multibyte.value) { + const preview = String((multibyte.value as { preview: string }).preview); + expect(Buffer.byteLength(preview)).toBeLessThanOrEqual(256 * 1024); + expect(preview).not.toContain("�"); + } + }); + + it("truncates captured logs on UTF-8 character boundaries", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + const result = await executor.execute({ + code: 'console.log("a" + "é".repeat(40000)); return "ok";', + }); + + expect(result).toMatchObject({ ok: true, value: "ok" }); + if (result.ok && result.logs) { + expect(Buffer.byteLength(result.logs[0].text)).toBeLessThanOrEqual(64 * 1024); + expect(result.logs[0].text).not.toContain("�"); + } + }); + + it("stops capturing logs when the remaining byte cannot hold a UTF-8 character", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + const result = await executor.execute({ + code: ` + console.log("a".repeat(65535)); + for (let index = 0; index < 100; index += 1) console.log("é"); + return "ok"; + `, + }); + + expect(result).toMatchObject({ ok: true, value: "ok" }); + if (result.ok && result.logs) { + expect(result.logs).toHaveLength(1); + expect(Buffer.byteLength(result.logs[0].text)).toBe(65_535); + expect(result.logs.every((entry) => entry.text.length > 0)).toBe(true); + } + }); + + it("captures circular console values without changing snippet success", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect( + executor.execute({ + code: 'const circular = {}; circular.self = circular; console.log(circular); return "ok";', + }), + ).resolves.toMatchObject({ + ok: true, + value: "ok", + logs: [{ level: "log", text: "[Unserializable value]" }], + }); + }); + + it("redacts nested and shared secrets while preserving short ordinary text", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const shared = { value: "shared-secret-value" }; + const circular: Record = { value: "circular-secret-value" }; + circular.self = circular; + const executor = new StagehandCodeExecutor( + localConfig({ + publicCopy: shared, + tokens: { shared, circular }, + cookies: { nested: { value: "nested-secret-value" } }, + apiKey: "short", + }), + ); + + const result = await executor.execute({ + code: ` + throw new Error( + "shared-secret-value nested-secret-value circular-secret-value short ordinary " + + "token=short Bearer bearer-value https://example.com/private" + ); + `, + }); + + expect(result).toMatchObject({ ok: false, error: { kind: "runtime" } }); + if (result.ok === false) { + expect(result.error.message).toContain("[REDACTED]"); + expect(result.error.message).toContain("short ordinary"); + expect(result.error.message).toContain("token=[REDACTED]"); + expect(result.error.message).toContain("Bearer [REDACTED]"); + expect(result.error.message).toContain("[REDACTED_URL]"); + expect(result.error.message).not.toContain("shared-secret-value"); + expect(result.error.message).not.toContain("nested-secret-value"); + expect(result.error.message).not.toContain("circular-secret-value"); + expect(result.error.message).not.toContain("bearer-value"); + } + }); + + it("normalizes non-Error throws and invalid error names", async () => { + const runtime = fakeRuntime(); + sdkMocks.localLaunch.mockResolvedValue(runtime.browser); + sdkMocks.stagehandCreate.mockResolvedValue(runtime.stagehand); + const executor = new StagehandCodeExecutor(localConfig()); + + await expect(executor.execute({ code: 'throw "raw secret";' })).resolves.toMatchObject({ + ok: false, + error: { name: "Error", message: "Code execution failed with a non-Error value." }, + }); + await expect( + executor.execute({ + code: 'const error = new Error("failed"); error.name = "bad name"; throw error;', + }), + ).resolves.toMatchObject({ + ok: false, + error: { name: "Error", message: "failed" }, + }); + }); +}); diff --git a/packages/integrations/tests/mcp-server.test.ts b/packages/integrations/tests/mcp-server.test.ts new file mode 100644 index 0000000000..3ec4fcb6ad --- /dev/null +++ b/packages/integrations/tests/mcp-server.test.ts @@ -0,0 +1,101 @@ +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { StagehandCodeExecutor } from "../src/codemode/executor.js"; +import { createCodeModeMcpServer } from "../src/codemode/mcp-server.js"; +import type { CodeExecuteResult } from "../src/codemode/types.js"; + +describe("code-mode MCP server", () => { + let client: Client; + let server: ReturnType; + let execute: ReturnType; + + beforeEach(async () => { + execute = vi.fn(); + server = createCodeModeMcpServer({ execute } as unknown as StagehandCodeExecutor); + client = new Client({ name: "stagehand-codemode-test", version: "1.0.0" }); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + }); + + afterEach(async () => { + await client.close(); + await server.close(); + }); + + it("advertises exactly one tool with complete input and output schemas", async () => { + const tools = await client.listTools(); + + expect(tools.tools).toHaveLength(1); + expect(tools.tools[0]).toMatchObject({ + name: "code_execute", + inputSchema: { + type: "object", + required: ["code"], + properties: { code: { type: "string" } }, + }, + outputSchema: { + type: "object", + required: ["ok"], + properties: { + ok: {}, + page: {}, + value: {}, + logs: {}, + error: {}, + }, + }, + }); + }); + + it("rejects invalid input before invoking the executor", async () => { + const response = await client.callTool({ + name: "code_execute", + arguments: { code: " " }, + }); + + expect(response).toMatchObject({ + isError: true, + content: [ + { + type: "text", + text: expect.stringContaining("code must contain JavaScript source"), + }, + ], + }); + expect(execute).not.toHaveBeenCalled(); + }); + + it.each([ + { + result: { + ok: true, + page: { url: "https://example.com", title: "Example" }, + value: { answer: 42 }, + } satisfies CodeExecuteResult, + isError: false, + }, + { + result: { + ok: false, + error: { kind: "runtime", name: "Error", message: "failed" }, + } satisfies CodeExecuteResult, + isError: true, + }, + ])("returns structured and text results for ok=$result.ok", async ({ result, isError }) => { + execute.mockResolvedValueOnce(result); + + const response = await client.callTool({ + name: "code_execute", + arguments: { code: "return 42;" }, + }); + + expect(response.structuredContent).toStrictEqual(result); + expect(response.isError).toBe(isError); + expect(response.content).toStrictEqual([ + { type: "text", text: JSON.stringify(result, null, 2) }, + ]); + expect(execute).toHaveBeenCalledWith({ code: "return 42;" }, expect.any(AbortSignal)); + }); +}); diff --git a/packages/integrations/tests/snippet.test.ts b/packages/integrations/tests/snippet.test.ts new file mode 100644 index 0000000000..462f81ade9 --- /dev/null +++ b/packages/integrations/tests/snippet.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it, vi } from "vitest"; +import { executeStagehandSnippet } from "../src/codemode/snippet.js"; + +const page = { marker: "page" }; +const context = { marker: "context" }; +const stagehand = { marker: "stagehand" }; + +describe("executeStagehandSnippet", () => { + it("injects browser, Stagehand, Zod, custom bindings, and console", async () => { + const codeConsole = { log: vi.fn(), warn: vi.fn(), error: vi.fn() }; + + const result = await executeStagehandSnippet({ + code: ` + console.log(label); + return { + page: page.marker, + context: context.marker, + stagehand: stagehand.marker, + parsed: z.object({ value: z.number() }).parse({ value: count }).value, + }; + `, + page: page as never, + context: context as never, + stagehand: stagehand as never, + bindings: { label: "ready", count: 3 }, + console: codeConsole, + }); + + expect(result).toStrictEqual({ + page: "page", + context: "context", + stagehand: "stagehand", + parsed: 3, + }); + expect(codeConsole.log).toHaveBeenCalledWith("ready"); + }); + + it("omits Stagehand and Zod in deterministic mode", async () => { + await expect( + executeStagehandSnippet({ + code: "return { stagehand: typeof stagehand, z: typeof z };", + page: page as never, + context: context as never, + }), + ).resolves.toStrictEqual({ stagehand: "undefined", z: "undefined" }); + }); + + it("awaits asynchronous code and propagates runtime errors", async () => { + await expect( + executeStagehandSnippet({ + code: "return await Promise.resolve(42);", + page: page as never, + context: context as never, + }), + ).resolves.toBe(42); + + await expect( + executeStagehandSnippet({ + code: 'throw new Error("snippet failed");', + page: page as never, + context: context as never, + }), + ).rejects.toThrow("snippet failed"); + }); + + it.each(["bad-name", "await", "class"])("rejects invalid binding name %s", async (name) => { + await expect( + executeStagehandSnippet({ + code: "return 1;", + page: page as never, + context: context as never, + bindings: { [name]: true }, + }), + ).rejects.toThrow(`Code-mode binding "${name}" is not a valid JavaScript identifier.`); + }); + + it.each(["page", "context", "stagehand", "z", "console"])( + "rejects reserved binding name %s", + async (name) => { + await expect( + executeStagehandSnippet({ + code: "return 1;", + page: page as never, + context: context as never, + bindings: { [name]: true }, + }), + ).rejects.toThrow(`Code-mode binding "${name}" is reserved.`); + }, + ); + + it("does not persist local variables between calls", async () => { + await executeStagehandSnippet({ + code: "const localOnly = 1; return localOnly;", + page: page as never, + context: context as never, + }); + + await expect( + executeStagehandSnippet({ + code: "return localOnly;", + page: page as never, + context: context as never, + }), + ).rejects.toThrow("localOnly is not defined"); + }); +}); diff --git a/packages/integrations/tests/stdio-server.test.ts b/packages/integrations/tests/stdio-server.test.ts index 25d8bb3018..ce9bac7245 100644 --- a/packages/integrations/tests/stdio-server.test.ts +++ b/packages/integrations/tests/stdio-server.test.ts @@ -6,12 +6,15 @@ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js" import { describe, expect, it } from "vitest"; const entrypoint = fileURLToPath(new URL("../dist/codemode/stdio-server.mjs", import.meta.url)); -const baseEnv = { PATH: process.env.PATH ?? "" }; -const readyMessage = "Stagehand code-mode MCP host listening on stdio"; +const baseEnv = { + PATH: process.env.PATH ?? "", + STAGEHAND_BROWSER: "local", +}; +const readyMessage = "Stagehand code-mode MCP listening on stdio"; -function startServer(): ChildProcessWithoutNullStreams { +function startServer(env: NodeJS.ProcessEnv = baseEnv): ChildProcessWithoutNullStreams { return spawn(process.execPath, [entrypoint], { - env: baseEnv, + env, stdio: ["pipe", "pipe", "pipe"], }); } @@ -20,7 +23,7 @@ async function waitForReady(child: ChildProcessWithoutNullStreams): Promise((resolve, reject) => { const timeout = setTimeout( - () => reject(new Error(`stdio host did not start: ${stderr}`)), + () => reject(new Error(`stdio server did not start: ${stderr}`)), 10_000, ); const onData = (chunk: Buffer) => { @@ -31,10 +34,10 @@ async function waitForReady(child: ChildProcessWithoutNullStreams): Promise { + child.once("close", (code, signal) => { clearTimeout(timeout); reject( - new Error(`stdio host exited before ready (code=${code}, signal=${signal}): ${stderr}`), + new Error(`stdio server exited before ready (code=${code}, signal=${signal}): ${stderr}`), ); }); }); @@ -82,17 +85,17 @@ function waitForExit( return new Promise((resolve, reject) => { const timeout = setTimeout(() => { child.kill("SIGKILL"); - reject(new Error("stdio host did not exit within 10 seconds")); + reject(new Error("stdio server did not exit within 10 seconds")); }, 10_000); child.once("error", reject); - child.once("exit", (code, signal) => { + child.once("close", (code, signal) => { clearTimeout(timeout); resolve({ code, signal }); }); }); } -describe("built code-mode stdio host", () => { +describe("built code-mode stdio server", () => { it("cleans up output waiters when the stream closes before the expected output", async () => { const stream = new PassThrough(); const output = waitForOutput(stream, readyMessage); @@ -106,8 +109,12 @@ describe("built code-mode stdio host", () => { expect(stream.listenerCount("close")).toBe(0); }); - it("starts and exits successfully on stdin EOF", async () => { - const child = startServer(); + it("starts in explicit local mode and exits successfully on stdin EOF", async () => { + const child = startServer({ + ...baseEnv, + BROWSERBASE_API_KEY: "unused-browserbase-key", + BROWSERBASE_PROJECT_ID: "unused-project-id", + }); try { await waitForReady(child); const exit = waitForExit(child); @@ -139,7 +146,20 @@ describe("built code-mode stdio host", () => { 30_000, ); - it("initializes without advertising tools through the compiled child", async () => { + it("fails startup for an invalid browser mode", async () => { + const child = startServer({ ...baseEnv, STAGEHAND_BROWSER: "remote" }); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + + const exit = await waitForExit(child); + + expect(exit.code).not.toBe(0); + expect(stderr).toContain('STAGEHAND_BROWSER must be either "local" or "browserbase".'); + }); + + it("supports MCP initialization, discovery, and validation through the compiled child", async () => { const transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint], @@ -152,7 +172,19 @@ describe("built code-mode stdio host", () => { try { await Promise.all([client.connect(transport), ready]); - expect(client.getServerCapabilities()).not.toHaveProperty("tools"); + const tools = await client.listTools(); + expect(tools.tools.map((tool) => tool.name)).toStrictEqual(["code_execute"]); + await expect( + client.callTool({ name: "code_execute", arguments: { code: " " } }), + ).resolves.toMatchObject({ + isError: true, + content: [ + { + type: "text", + text: expect.stringContaining("code must contain JavaScript source"), + }, + ], + }); } finally { await client.close(); } diff --git a/packages/integrations/tests/tool-contract.test.ts b/packages/integrations/tests/tool-contract.test.ts new file mode 100644 index 0000000000..b512d6ee9c --- /dev/null +++ b/packages/integrations/tests/tool-contract.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { + codeExecuteOutputSchema, + codeExecuteResultText, + codeExecuteSchema, +} from "../src/codemode/tool-contract.js"; + +describe("code-mode tool contract", () => { + it("accepts nonblank code up to 100,000 UTF-8 bytes", () => { + expect(codeExecuteSchema.parse({ code: "return 1;" })).toStrictEqual({ code: "return 1;" }); + expect(codeExecuteSchema.safeParse({ code: " \n\t " }).success).toBe(false); + expect(codeExecuteSchema.safeParse({ code: "é".repeat(50_000) }).success).toBe(true); + expect(codeExecuteSchema.safeParse({ code: `${"é".repeat(50_000)}a` }).success).toBe(false); + }); + + it("validates complete success and failure results", () => { + expect( + codeExecuteOutputSchema.parse({ + ok: true, + page: { url: "https://example.com", title: "Example" }, + value: { answer: 42 }, + logs: [{ level: "log", text: "ready" }], + }), + ).toMatchObject({ ok: true, value: { answer: 42 } }); + + expect( + codeExecuteOutputSchema.parse({ + ok: false, + error: { kind: "runtime", name: "Error", message: "failed" }, + }), + ).toMatchObject({ ok: false, error: { kind: "runtime" } }); + }); + + it("rejects invalid success/failure combinations", () => { + expect(codeExecuteOutputSchema.safeParse({ ok: true }).success).toBe(false); + expect( + codeExecuteOutputSchema.safeParse({ + ok: true, + page: { url: "https://example.com", title: "Example" }, + error: { kind: "runtime", name: "Error", message: "failed" }, + }).success, + ).toBe(false); + expect(codeExecuteOutputSchema.safeParse({ ok: false }).success).toBe(false); + expect( + codeExecuteOutputSchema.safeParse({ + ok: false, + value: 42, + error: { kind: "runtime", name: "Error", message: "failed" }, + }).success, + ).toBe(false); + }); + + it("rejects unknown error kinds and log levels", () => { + expect( + codeExecuteOutputSchema.safeParse({ + ok: false, + error: { kind: "timeout", name: "Error", message: "failed" }, + }).success, + ).toBe(false); + expect( + codeExecuteOutputSchema.safeParse({ + ok: true, + page: { url: "https://example.com", title: "Example" }, + logs: [{ level: "debug", text: "nope" }], + }).success, + ).toBe(false); + }); + + it("renders the result as stable pretty JSON", () => { + expect( + codeExecuteResultText({ + ok: true, + page: { url: "https://example.com", title: "Example" }, + value: 42, + }), + ).toBe( + '{\n "ok": true,\n "page": {\n "url": "https://example.com",\n "title": "Example"\n },\n "value": 42\n}', + ); + }); +}); diff --git a/packages/integrations/tsdown.config.ts b/packages/integrations/tsdown.config.ts index 587cfd7b80..c203574ed1 100644 --- a/packages/integrations/tsdown.config.ts +++ b/packages/integrations/tsdown.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "tsdown"; export default defineConfig({ entry: { + "codemode/index": "src/codemode/index.ts", "codemode/stdio-server": "src/codemode/stdio-server.ts", }, format: ["esm"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7882eaeae3..4e014c0158 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -566,9 +566,15 @@ importers: packages/integrations: dependencies: + '@browserbasehq/stagehand': + specifier: workspace:* + version: link:../sdk-ts '@modelcontextprotocol/sdk': specifier: 'catalog:' version: 1.29.0(zod@4.4.3) + zod: + specifier: 'catalog:' + version: 4.4.3 devDependencies: '@types/node': specifier: 'catalog:'