diff --git a/packages/evals/.gitignore b/packages/evals/.gitignore index d67d472e2a..c4ff719702 100644 --- a/packages/evals/.gitignore +++ b/packages/evals/.gitignore @@ -1 +1,4 @@ .browserbase/ +.trajectories/ +.rubric-cache/ +packages/ diff --git a/packages/evals/framework/codexCodeBridge.ts b/packages/evals/framework/codexCodeBridge.ts index c50fb671b8..45a329cdf8 100644 --- a/packages/evals/framework/codexCodeBridge.ts +++ b/packages/evals/framework/codexCodeBridge.ts @@ -15,6 +15,7 @@ */ import http from "node:http"; import type { AddressInfo } from "node:net"; +import { sanitizeErrorMessage } from "@browserbasehq/stagehand-integrations/harness"; import type { AgentMount } from "../core/contracts/tool.js"; import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; import type { EvalLogger } from "../logger.js"; @@ -55,18 +56,6 @@ function stringifyResult(value: unknown): string { } } -/** - * Snippet errors can embed connection URLs whose query strings carry - * credentials (Browserbase connect URLs include signing keys). Redact - * credential-bearing fragments before the message crosses the HTTP boundary - * or reaches the logs; the rest stays intact so the agent can self-correct. - */ -export function sanitizeErrorMessage(message: string): string { - return message - .replace(/([?&](?:signingKey|apiKey|api_key|token|key)=)[^&\s"']+/gi, "$1[redacted]") - .replace(/\b(sk-[A-Za-z0-9_-]{6})[A-Za-z0-9_-]+/g, "$1[redacted]"); -} - export interface CodeBridge { port: number; close: () => Promise; diff --git a/packages/evals/package.json b/packages/evals/package.json index c36f3a69d6..6de67da3ef 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -24,6 +24,7 @@ "@braintrust/otel": "^0.2.1", "@browserbasehq/sdk": "catalog:", "@browserbasehq/stagehand": "workspace:*", + "@browserbasehq/stagehand-integrations": "workspace:*", "@openai/codex-sdk": "catalog:", "@opentelemetry/api": "catalog:", "@opentelemetry/exporter-trace-otlp-proto": "^0.220.0", diff --git a/packages/integrations/claude-code/src/agent.ts b/packages/integrations/claude-code/src/agent.ts index 6b7e07bca1..14f3d95f1e 100644 --- a/packages/integrations/claude-code/src/agent.ts +++ b/packages/integrations/claude-code/src/agent.ts @@ -3,10 +3,9 @@ import { FACADE_AGENT_INSTRUCTIONS, FACADE_TOOLS, } from "@browserbasehq/stagehand-integrations/facade"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; -import { buildAllowlistedEnv } from "./env.ts"; - export const STAGEHAND_TOOL_NAMES = FACADE_TOOLS.map((tool) => `mcp__stagehand__${tool.name}`); async function main(): Promise { diff --git a/packages/integrations/claude-code/tests/agent.test.ts b/packages/integrations/claude-code/tests/agent.test.ts index 83b5d07eec..4abd549b9f 100644 --- a/packages/integrations/claude-code/tests/agent.test.ts +++ b/packages/integrations/claude-code/tests/agent.test.ts @@ -1,8 +1,8 @@ import { FACADE_TOOLS } from "@browserbasehq/stagehand-integrations/facade"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { describe, expect, it } from "vitest"; import { STAGEHAND_TOOL_NAMES } from "../src/agent.ts"; -import { buildAllowlistedEnv } from "../src/env.ts"; describe("claude-code stagehand example", () => { it("allows exactly the namespaced facade tools", () => { diff --git a/packages/integrations/codex/src/agent.ts b/packages/integrations/codex/src/agent.ts index c27eef56d7..fdb29a9f16 100644 --- a/packages/integrations/codex/src/agent.ts +++ b/packages/integrations/codex/src/agent.ts @@ -1,8 +1,7 @@ import { Codex, type CodexOptions } from "@openai/codex-sdk"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; -import { buildAllowlistedEnv } from "./env.ts"; - const serverPath = fileURLToPath( import.meta.resolve("@browserbasehq/stagehand-integrations/facade/stdio-server"), ); diff --git a/packages/integrations/codex/src/env.ts b/packages/integrations/codex/src/env.ts deleted file mode 100644 index 849c34954b..0000000000 --- a/packages/integrations/codex/src/env.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into the server. */ -export function buildAllowlistedEnv(): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (/^(STAGEHAND_|BROWSERBASE_)/.test(key) && value) { - env[key] = value; - } - } - return env; -} diff --git a/packages/integrations/codex/tests/agent.test.ts b/packages/integrations/codex/tests/agent.test.ts index c506965f65..2358b534ef 100644 --- a/packages/integrations/codex/tests/agent.test.ts +++ b/packages/integrations/codex/tests/agent.test.ts @@ -1,7 +1,7 @@ +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { describe, expect, it } from "vitest"; import { buildCodexConfig } from "../src/agent.ts"; -import { buildAllowlistedEnv } from "../src/env.ts"; describe("codex stagehand example", () => { it("mounts the facade server in the config override", () => { diff --git a/packages/integrations/core/package.json b/packages/integrations/core/package.json index 895932a846..88d7715590 100644 --- a/packages/integrations/core/package.json +++ b/packages/integrations/core/package.json @@ -19,6 +19,10 @@ }, "./facade/stdio-server": { "import": "./dist/facade/stdio-server.mjs" + }, + "./harness": { + "types": "./dist/harness/index.d.mts", + "import": "./dist/harness/index.mjs" } }, "scripts": { diff --git a/packages/integrations/core/src/facade/stdio-server.ts b/packages/integrations/core/src/facade/stdio-server.ts index 3d0c782203..53b1bc5bac 100644 --- a/packages/integrations/core/src/facade/stdio-server.ts +++ b/packages/integrations/core/src/facade/stdio-server.ts @@ -10,6 +10,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; import { closeCodeModeStdio } from "../codemode/stdio-lifecycle.js"; +import { sanitizeErrorMessage } from "../harness/redact.js"; import { stagehandFacadeConfigFromEnv } from "./config.js"; import { CodeModeRunInputSchema, @@ -149,15 +150,6 @@ function stringifyResult(value: unknown): string { } } -export function sanitizeErrorMessage(message: string): string { - return message - .replace(/([?&](?:signingKey|apiKey|api_key|token|key)=)[^&\s"']+/gi, "$1[redacted]") - .replace(/\b(sk-[A-Za-z0-9_-]{6})[A-Za-z0-9_-]+/g, "$1[redacted]") - .replace(/\b(bb_(?:live|test)_[A-Za-z0-9]{4})[A-Za-z0-9_-]+/g, "$1[redacted]") - .replace(/\bAIza[0-9A-Za-z_-]{30,}/g, "AIza[redacted]") - .replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi, "$1[redacted]"); -} - async function shutdown(code: number): Promise { if (closing) return; closing = true; diff --git a/packages/integrations/core/src/harness/contract.ts b/packages/integrations/core/src/harness/contract.ts new file mode 100644 index 0000000000..0a65299fe3 --- /dev/null +++ b/packages/integrations/core/src/harness/contract.ts @@ -0,0 +1,29 @@ +type LogLine = { + id?: string; + category?: string; + message: string; + level?: 0 | 1 | 2; + timestamp?: string; + auxiliary?: Record< + string, + { + value: string; + type: "object" | "string" | "html" | "integer" | "float" | "boolean"; + } + >; +}; + +/** Logger surface required by agent harness adapters. */ +export type HarnessLogger = { + log(line: LogLine): void; + warn(line: LogLine): void; + error(line: LogLine): void; +}; + +/** Error raised when a harness adapter cannot complete its work. */ +export class HarnessAdapterError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "HarnessAdapterError"; + } +} diff --git a/packages/integrations/claude-code/src/env.ts b/packages/integrations/core/src/harness/env.ts similarity index 93% rename from packages/integrations/claude-code/src/env.ts rename to packages/integrations/core/src/harness/env.ts index 849c34954b..43069f433a 100644 --- a/packages/integrations/claude-code/src/env.ts +++ b/packages/integrations/core/src/harness/env.ts @@ -1,4 +1,4 @@ -/** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into the server. */ +/** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into harness processes. */ export function buildAllowlistedEnv(): Record { const env: Record = {}; for (const [key, value] of Object.entries(process.env)) { diff --git a/packages/integrations/core/src/harness/index.ts b/packages/integrations/core/src/harness/index.ts new file mode 100644 index 0000000000..75962aa097 --- /dev/null +++ b/packages/integrations/core/src/harness/index.ts @@ -0,0 +1,3 @@ +export * from "./contract.js"; +export * from "./env.js"; +export * from "./redact.js"; diff --git a/packages/integrations/core/src/harness/redact.ts b/packages/integrations/core/src/harness/redact.ts new file mode 100644 index 0000000000..454f3a4d6b --- /dev/null +++ b/packages/integrations/core/src/harness/redact.ts @@ -0,0 +1,19 @@ +export function sanitizeErrorMessage(message: string): string { + let sanitized = message; + // stdio-server and codexCodeBridge: credential-bearing URL query parameters. + sanitized = sanitized.replace( + /([?&](?:signingKey|apiKey|api_key|token|key)=)[^&\s"']+/gi, + "$1[redacted]", + ); + // stdio-server and codexCodeBridge: OpenAI-style secret keys. + sanitized = sanitized.replace(/\b(sk-[A-Za-z0-9_-]{6})[A-Za-z0-9_-]+/g, "$1[redacted]"); + // stdio-server: Browserbase live and test keys. + sanitized = sanitized.replace( + /\b(bb_(?:live|test)_[A-Za-z0-9]{4})[A-Za-z0-9_-]+/g, + "$1[redacted]", + ); + // stdio-server: Google API keys. + sanitized = sanitized.replace(/\bAIza[0-9A-Za-z_-]{30,}/g, "AIza[redacted]"); + // stdio-server: bearer authorization values. + return sanitized.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi, "$1[redacted]"); +} diff --git a/packages/integrations/core/tests/harness.test.ts b/packages/integrations/core/tests/harness.test.ts new file mode 100644 index 0000000000..8ec2288c96 --- /dev/null +++ b/packages/integrations/core/tests/harness.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildAllowlistedEnv, sanitizeErrorMessage } from "../src/harness/index.js"; + +describe("harness contract", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("redacts credential-bearing URL query parameters", () => { + expect(sanitizeErrorMessage("wss://example.test?signingKey=top-secret&foo=bar")).toBe( + "wss://example.test?signingKey=[redacted]&foo=bar", + ); + }); + + it("redacts OpenAI-style secret keys", () => { + expect(sanitizeErrorMessage("key sk-abcdef1234567890")).toBe("key sk-abcdef[redacted]"); + }); + + it("redacts Browserbase keys", () => { + expect(sanitizeErrorMessage("key bb_live_abcd1234567890")).toBe("key bb_live_abcd[redacted]"); + }); + + it("redacts Google API keys", () => { + expect(sanitizeErrorMessage(`key AIza${"a".repeat(32)}`)).toBe("key AIza[redacted]"); + }); + + it("redacts bearer authorization values", () => { + expect(sanitizeErrorMessage("Authorization: Bearer abcdefgh123456")).toBe( + "Authorization: Bearer [redacted]", + ); + }); + + it("allows Stagehand and Browserbase env vars while excluding other and empty values", () => { + vi.stubEnv("STAGEHAND_MODEL", "model"); + vi.stubEnv("BROWSERBASE_API_KEY", "browserbase-key"); + vi.stubEnv("STAGEHAND_EMPTY", ""); + vi.stubEnv("NOT_ALLOWLISTED_SECRET", "secret"); + + const env = buildAllowlistedEnv(); + + expect(env.STAGEHAND_MODEL).toBe("model"); + expect(env.BROWSERBASE_API_KEY).toBe("browserbase-key"); + expect(env).not.toHaveProperty("STAGEHAND_EMPTY"); + expect(env).not.toHaveProperty("NOT_ALLOWLISTED_SECRET"); + }); +}); diff --git a/packages/integrations/core/tsdown.config.ts b/packages/integrations/core/tsdown.config.ts index d18c6ef4b6..2766e2babd 100644 --- a/packages/integrations/core/tsdown.config.ts +++ b/packages/integrations/core/tsdown.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ "codemode/stdio-server": "src/codemode/stdio-server.ts", "facade/index": "src/facade/index.ts", "facade/stdio-server": "src/facade/stdio-server.ts", + "harness/index": "src/harness/index.ts", }, format: ["esm"], platform: "node", diff --git a/packages/integrations/mastra/src/client.ts b/packages/integrations/mastra/src/client.ts index aeff3baa56..8326585b7f 100644 --- a/packages/integrations/mastra/src/client.ts +++ b/packages/integrations/mastra/src/client.ts @@ -1,4 +1,5 @@ import { MCPClient } from "@mastra/mcp"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; export type FacadeMCPClientOptions = { @@ -9,17 +10,6 @@ export type FacadeMCPClientOptions = { * Starts the facade MCP server and returns its Mastra client. * Call `client.disconnect()` when finished to stop the child process and browser. */ -/** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into the server. */ -export function buildAllowlistedEnv(): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (/^(STAGEHAND_|BROWSERBASE_)/.test(key) && value) { - env[key] = value; - } - } - return env; -} - export function createFacadeMCPClient(options: FacadeMCPClientOptions = {}) { // Build @browserbasehq/stagehand-integrations first so this dist entrypoint exists. const serverPath = fileURLToPath( diff --git a/packages/integrations/mastra/tests/client.test.ts b/packages/integrations/mastra/tests/client.test.ts index 06eafadbd3..55f12d4637 100644 --- a/packages/integrations/mastra/tests/client.test.ts +++ b/packages/integrations/mastra/tests/client.test.ts @@ -1,7 +1,8 @@ import { FACADE_TOOLS } from "@browserbasehq/stagehand-integrations/facade"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { afterEach, describe, expect, it } from "vitest"; -import { buildAllowlistedEnv, createFacadeMCPClient } from "../src/client.js"; +import { createFacadeMCPClient } from "../src/client.js"; type FacadeClient = ReturnType; diff --git a/packages/integrations/vercel-ai/src/client.ts b/packages/integrations/vercel-ai/src/client.ts index 35f18d6731..2e00bf23d4 100644 --- a/packages/integrations/vercel-ai/src/client.ts +++ b/packages/integrations/vercel-ai/src/client.ts @@ -1,22 +1,12 @@ import { createMCPClient } from "@ai-sdk/mcp"; import { Experimental_StdioMCPTransport } from "@ai-sdk/mcp/mcp-stdio"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; export type FacadeMCPClientOptions = { env?: Record; }; -/** Only STAGEHAND_* and BROWSERBASE_* host env vars cross into the server. */ -export function buildAllowlistedEnv(): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (/^(STAGEHAND_|BROWSERBASE_)/.test(key) && value) { - env[key] = value; - } - } - return env; -} - /** * Starts the stdio facade server as a child process and returns the AI SDK * MCP client connected to it. Call `client.close()` when finished to stop the diff --git a/packages/integrations/vercel-ai/tests/client.test.ts b/packages/integrations/vercel-ai/tests/client.test.ts index 8bfcf4d5bd..714cb51819 100644 --- a/packages/integrations/vercel-ai/tests/client.test.ts +++ b/packages/integrations/vercel-ai/tests/client.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; +import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { createFacadeMCPClient } from "../src/client.js"; @@ -53,7 +54,6 @@ describe("Stagehand facade MCP client", () => { it("allowlist excludes non-STAGEHAND/BROWSERBASE host vars", async () => { process.env.STAGEHAND_BROWSER = "local"; process.env.NOT_ALLOWLISTED_SECRET = "must-not-cross"; - const { buildAllowlistedEnv } = await import("../src/client.js"); const env = buildAllowlistedEnv(); expect(env.STAGEHAND_BROWSER).toBe("local"); expect(env).not.toHaveProperty("NOT_ALLOWLISTED_SECRET"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 647e3363eb..4810d60f8c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -459,6 +459,9 @@ importers: '@browserbasehq/stagehand': specifier: workspace:* version: link:../sdk-ts + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../integrations/core '@openai/codex-sdk': specifier: 'catalog:' version: 0.147.0