From 22256df9db4c8acb4c38236a564c94258e160c7c Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 7 Aug 2026 17:11:47 -0700 Subject: [PATCH 1/6] feat(eve): add sandboxed code-mode MCP example --- .../workflows/codemode-framework-examples.yml | 21 + packages/integrations/README.md | 1 + packages/integrations/examples/eve/.gitignore | 2 + packages/integrations/examples/eve/README.md | 169 +++++++ .../integrations/examples/eve/agent/agent.ts | 55 +++ .../eve/agent/connections/stagehand.ts | 17 + .../examples/eve/agent/instructions.md | 3 + .../examples/eve/evals/evals.config.ts | 3 + .../examples/eve/evals/stagehand.eval.ts | 27 ++ .../integrations/examples/eve/package.json | 28 ++ packages/integrations/examples/eve/src/e2e.ts | 12 + .../integrations/examples/eve/src/gateway.ts | 200 ++++++++ .../integrations/examples/eve/src/run-eval.ts | 97 ++++ .../examples/eve/src/sandbox-guest.mjs | 128 +++++ .../integrations/examples/eve/src/sandbox.ts | 119 +++++ .../integrations/examples/eve/src/smoke.ts | 15 + .../integrations/examples/eve/tsconfig.json | 14 + pnpm-lock.yaml | 454 ++++++++++++++++++ pnpm-workspace.yaml | 2 + 19 files changed, 1367 insertions(+) create mode 100644 packages/integrations/examples/eve/.gitignore create mode 100644 packages/integrations/examples/eve/README.md create mode 100644 packages/integrations/examples/eve/agent/agent.ts create mode 100644 packages/integrations/examples/eve/agent/connections/stagehand.ts create mode 100644 packages/integrations/examples/eve/agent/instructions.md create mode 100644 packages/integrations/examples/eve/evals/evals.config.ts create mode 100644 packages/integrations/examples/eve/evals/stagehand.eval.ts create mode 100644 packages/integrations/examples/eve/package.json create mode 100644 packages/integrations/examples/eve/src/e2e.ts create mode 100644 packages/integrations/examples/eve/src/gateway.ts create mode 100644 packages/integrations/examples/eve/src/run-eval.ts create mode 100644 packages/integrations/examples/eve/src/sandbox-guest.mjs create mode 100644 packages/integrations/examples/eve/src/sandbox.ts create mode 100644 packages/integrations/examples/eve/src/smoke.ts create mode 100644 packages/integrations/examples/eve/tsconfig.json diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index 0846df5084..5341822f38 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -128,3 +128,24 @@ jobs: env: CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} STAGEHAND_BROWSER: local + + eve: + name: Eve + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: ./.github/actions/setup-node-pnpm + with: + use-prebuilt-artifacts: "false" + + - uses: ./.github/actions/setup-chrome-verified + id: setup-chrome + + - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve typecheck + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve smoke + env: + CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} + STAGEHAND_BROWSER: local diff --git a/packages/integrations/README.md b/packages/integrations/README.md index 34bd0affb9..e1c5e504a4 100644 --- a/packages/integrations/README.md +++ b/packages/integrations/README.md @@ -50,6 +50,7 @@ The process stays alive across calls and closes when its input stream ends. `SIG - [Mastra](./examples/mastra) discovers the canonical MCP toolset once and reuses one client and browser for the complete agent run. - [CrewAI](./examples/crewai) keeps its context-managed MCP adapter open across every tool call in one crew execution. - [LangChain Deep Agents](./examples/langchain) uses one explicit MCP session so every tool call reaches the same browser. +- [Eve](./examples/eve) keeps its host outside the sandbox and reaches the stdio server through an authenticated Streamable HTTP gateway inside the sandbox. ### Configuration diff --git a/packages/integrations/examples/eve/.gitignore b/packages/integrations/examples/eve/.gitignore new file mode 100644 index 0000000000..0ee2c24870 --- /dev/null +++ b/packages/integrations/examples/eve/.gitignore @@ -0,0 +1,2 @@ +.eve/ +.output/ diff --git a/packages/integrations/examples/eve/README.md b/packages/integrations/examples/eve/README.md new file mode 100644 index 0000000000..fe87979932 --- /dev/null +++ b/packages/integrations/examples/eve/README.md @@ -0,0 +1,169 @@ +# Eve + Stagehand code mode + +Eve accepts remote Streamable HTTP or SSE MCP connections. It does not launch stdio MCP servers +directly. This example keeps the Eve host outside the execution boundary and puts the unchanged +Stagehand stdio server, its browser session, and a small HTTP adapter inside a sandbox. + +```text +Eve host + | + | Streamable HTTP + bearer token + v +Firecracker or gVisor sandbox + |-- authenticated proxy + `-- supergateway (non-first-party) + | + | stdio + v + Stagehand code-mode MCP + | + v + generated JavaScript +``` + +The process boundary between `supergateway` and Stagehand is not the security boundary. Generated +JavaScript inherits the MCP process's filesystem, environment, and network. The Firecracker +microVM or gVisor sandbox is the boundary that protects the Eve host. + +## Configure the Eve connection + +[`agent/connections/stagehand.ts`](./agent/connections/stagehand.ts) is the complete Eve connection: + +```ts +import { defineMcpClientConnection } from "eve/connections"; + +export default defineMcpClientConnection({ + url: process.env.STAGEHAND_MCP_URL!, + description: + "Stagehand browser automation isolated behind an authenticated code-mode MCP gateway.", + auth: { + getToken: async () => ({ token: process.env.STAGEHAND_MCP_TOKEN! }), + }, + tools: { allow: ["code_execute"] }, +}); +``` + +Eve discovers the connection through `connection_search`. The only remote tool it can reveal is +`stagehand__code_execute`. + +## Start the sandbox + +The proposed image name is `ghcr.io/browserbase/stagehand-codemode`. The foundation workflow builds +the image and verifies stdio tool discovery locally; the registry reference becomes available only +after a tag or manual publish workflow runs. Keep the image configurable and, once published, pin +the deployment to an immutable digest: + +```text +STAGEHAND_CODEMODE_IMAGE=ghcr.io/browserbase/stagehand-codemode@sha256: +``` + +[`src/sandbox.ts`](./src/sandbox.ts) defines the provider contract and lifecycle used by the host. +The sandbox adapter must turn `stdioImage` into a command inside the guest. It can mirror the OCI +image into the sandbox root filesystem, or use a guest container runtime and return this command: + +```ts +{ + command: "docker", + args: [ + "run", "--rm", "-i", + "ghcr.io/browserbase/stagehand-codemode@sha256:", + ], +} +``` + +The image itself contains only the Stagehand stdio MCP. It does **not** contain the HTTP gateway. +The trusted host bootstrap writes [`src/sandbox-guest.mjs`](./src/sandbox-guest.mjs) into the +Firecracker or gVisor guest, installs `supergateway@3.4.3` there, and starts the authenticated +proxy. `supergateway` then owns the image command as its stdio child. Pass an adapter that can write +a file, spawn a process, publish a port, and destroy the sandbox; then give the result to Eve: + +```ts +import { createStagehandSandboxGateway } from "./src/sandbox.js"; + +const stagehand = await createStagehandSandboxGateway(sandboxProvider, { + image: process.env.STAGEHAND_CODEMODE_IMAGE, + environment: { STAGEHAND_BROWSER: "browserbase" }, +}); + +try { + // Start the Eve host with STAGEHAND_MCP_URL=stagehand.url and + // STAGEHAND_MCP_TOKEN=stagehand.token. +} finally { + await stagehand.close(); +} +``` + +For Vercel Sandbox specifically, create a Node 24 Firecracker guest with port `3000`, map +`writeTextFile` to `sandbox.writeFiles`, map `spawn` to a detached `sandbox.runCommand`, map +`publicUrl` to `sandbox.domain`, and map `close` to `sandbox.stop`. Vercel's command handle exposes +stdout and stderr but not a writable stdin stream after launch, so the Eve host cannot drive the +stdio MCP directly across the SDK boundary. That is why both `supergateway` and the authentication +proxy run inside the guest. The adapter must also materialize the pinned Stagehand image contents +inside the guest—such as through a mirrored runtime image or an exact source build—and return the +resulting local Node command as `stdioCommand`; do not assume nested Docker is available. + +`createStagehandSandboxGateway` polls the authenticated `/healthz` endpoint before returning. This +prevents Eve from racing the guest bootstrap or public port publication. + +The host starts one authenticated proxy and one stateful `supergateway` process per sandbox. +`supergateway` starts one Stagehand stdio child for each MCP session. `supergateway` is a +non-first-party adapter. The bootstrap pins it to `3.4.3`, sets `--logLevel none`, protects the +public port with a bearer token, and closes the complete sandbox after the Eve run. Do not expose +`supergateway` directly: it does not add inbound authentication. + +For production, bake the audited gateway and its exact dependency tree into the guest image. The +runtime `npm install` in `src/sandbox.ts` makes this provider-neutral example executable, but it +adds a network-time supply-chain dependency during sandbox startup. + +`src/sandbox.ts` accepts only the documented Stagehand and Browserbase environment names. Avoid +putting long-lived credentials there because generated JavaScript can read them. Prefer a sandbox +credential broker and an egress allowlist where the browser provider's HTTP and WebSocket +transports support them. The local test helper inherits the host environment for developer +convenience; it is explicitly not the production boundary. + +## Verification status + +We verified the two critical pieces independently: + +- A real Vercel Firecracker sandbox ran the Stagehand stdio MCP behind the authenticated gateway. + An unauthenticated request returned `401`; an authenticated MCP client initialized, listed only + `code_execute`, and the complete sandbox was destroyed afterward. +- Eve `0.29.4` passed both the deterministic and real-model flows below through the same gateway + implementation on localhost. + +The full composition is still partial. A host Eve `0.29.3` run reached the sandbox gateway, +initialized a stateful MCP session, sent the initialized notification, and opened its authenticated +SSE stream, but did not advance to `tools/list` or `tools/call`. That exact version-mismatched proof +does not establish an Eve `0.29.4` to sandbox `code_execute` loop. Treat the architecture as +supported but keep the composed deployment behind an integration test until that Streamable HTTP +client interoperability gap is closed. + +## Run the deterministic smoke + +From the repository root: + +```bash +pnpm install +pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations +STAGEHAND_BROWSER=local \ + pnpm --filter @browserbasehq/stagehand-integrations-example-eve smoke +``` + +The smoke deliberately runs the gateway on localhost so public CI needs no sandbox credentials. It +is not a sandbox-isolation proof. It does exercise the actual Eve runtime, rejects an unauthorized +request, and then runs authenticated Streamable HTTP `connection_search` followed by one +`stagehand__code_execute` call against a real local browser. + +## Run a real Eve agent + +The real example uses Eve `0.29.4` and a direct Groq model. Set `GROQ_API_KEY`, then run: + +```bash +GROQ_API_KEY= \ +STAGEHAND_BROWSER=local \ + pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e +``` + +To use Browserbase, set `STAGEHAND_BROWSER=browserbase`, `BROWSERBASE_API_KEY`, and optionally +`BROWSERBASE_PROJECT_ID` in the sandbox process. The production architecture remains the sandboxed +one above; the local commands exist only to make framework behavior reproducible in CI. diff --git a/packages/integrations/examples/eve/agent/agent.ts b/packages/integrations/examples/eve/agent/agent.ts new file mode 100644 index 0000000000..dc58caf8ca --- /dev/null +++ b/packages/integrations/examples/eve/agent/agent.ts @@ -0,0 +1,55 @@ +import { groq } from "@ai-sdk/groq"; +import { defineAgent } from "eve"; +import { mockModel } from "eve/evals"; + +const stagehandTool = "stagehand__code_execute"; + +export default defineAgent({ + modelContextWindowTokens: 131_072, + model: + process.env.EVE_STAGEHAND_DETERMINISTIC === "1" + ? mockModel(({ toolResults, tools }) => { + const stagehandResults = toolResults.filter(({ name }) => name === stagehandTool); + + if (!tools.some(({ name }) => name === stagehandTool)) { + return { + toolCalls: [ + { + name: "connection_search", + input: { connection: "stagehand", keywords: "execute browser JavaScript" }, + }, + ], + }; + } + + if (stagehandResults.length === 0) { + return { + toolCalls: [ + { + name: stagehandTool, + input: { + code: ` + await page.goto("https://example.com", { + waitUntil: "domcontentloaded", + }); + await page.evaluate(() => { + document.documentElement.dataset.eveStagehandSmoke = "persistent"; + }); + return { + pageId: page.pageId, + title: await page.title(), + marker: await page.evaluate( + () => document.documentElement.dataset.eveStagehandSmoke, + ), + }; + `, + }, + }, + ], + }; + } + + return `STAGEHAND_RESULT ${JSON.stringify(stagehandResults.at(-1)?.output)}`; + }) + : groq(process.env.EVE_STAGEHAND_MODEL ?? "openai/gpt-oss-120b"), +}); diff --git a/packages/integrations/examples/eve/agent/connections/stagehand.ts b/packages/integrations/examples/eve/agent/connections/stagehand.ts new file mode 100644 index 0000000000..9dfc24d404 --- /dev/null +++ b/packages/integrations/examples/eve/agent/connections/stagehand.ts @@ -0,0 +1,17 @@ +import { defineMcpClientConnection } from "eve/connections"; + +const endpoint = process.env.STAGEHAND_MCP_URL ?? "http://127.0.0.1:3000/mcp"; + +export default defineMcpClientConnection({ + url: endpoint, + description: + "Stagehand browser automation isolated behind an authenticated code-mode MCP gateway.", + auth: { + getToken: async () => { + const token = process.env.STAGEHAND_MCP_TOKEN; + if (!token) throw new Error("STAGEHAND_MCP_TOKEN is required"); + return { token }; + }, + }, + tools: { allow: ["code_execute"] }, +}); diff --git a/packages/integrations/examples/eve/agent/instructions.md b/packages/integrations/examples/eve/agent/instructions.md new file mode 100644 index 0000000000..3a8afb4d38 --- /dev/null +++ b/packages/integrations/examples/eve/agent/instructions.md @@ -0,0 +1,3 @@ +You are a browser agent. Use `connection_search` to discover the Stagehand connection, then use its +`code_execute` tool for browser work. Keep browser state in the existing tool session and return +only the evidence the user requested. diff --git a/packages/integrations/examples/eve/evals/evals.config.ts b/packages/integrations/examples/eve/evals/evals.config.ts new file mode 100644 index 0000000000..3a1a10d6fd --- /dev/null +++ b/packages/integrations/examples/eve/evals/evals.config.ts @@ -0,0 +1,3 @@ +import { defineEvalConfig } from "eve/evals"; + +export default defineEvalConfig({}); diff --git a/packages/integrations/examples/eve/evals/stagehand.eval.ts b/packages/integrations/examples/eve/evals/stagehand.eval.ts new file mode 100644 index 0000000000..c4b5588e76 --- /dev/null +++ b/packages/integrations/examples/eve/evals/stagehand.eval.ts @@ -0,0 +1,27 @@ +import { defineEval } from "eve/evals"; +import { includes } from "eve/evals/expect"; + +export default defineEval({ + description: "Eve discovers and calls Stagehand through an authenticated remote MCP gateway.", + async test(t) { + await t.send( + [ + "Use the Stagehand connection and call code_execute exactly once.", + 'Open example.com and store the exact marker string "persistent" on the page.', + "Report the title and marker.", + ].join(" "), + ); + + t.succeeded(); + t.calledTool("connection_search", { count: 1 }); + t.calledTool("stagehand__code_execute", { + count: 1, + output: (value) => { + const output = JSON.stringify(value); + return output.includes("Example Domain") && output.includes("persistent"); + }, + }); + t.check(t.reply, includes("Example Domain")); + t.check(t.reply, includes("persistent")); + }, +}); diff --git a/packages/integrations/examples/eve/package.json b/packages/integrations/examples/eve/package.json new file mode 100644 index 0000000000..0f64c0deff --- /dev/null +++ b/packages/integrations/examples/eve/package.json @@ -0,0 +1,28 @@ +{ + "name": "@browserbasehq/stagehand-integrations-example-eve", + "version": "4.0.0", + "private": true, + "type": "module", + "scripts": { + "build": "eve build", + "dev": "eve dev", + "e2e": "tsx src/e2e.ts", + "smoke": "tsx src/smoke.ts", + "typecheck": "eve build && tsc --noEmit" + }, + "dependencies": { + "@ai-sdk/groq": "catalog:", + "@browserbasehq/stagehand-integrations": "workspace:*", + "eve": "catalog:", + "supergateway": "catalog:", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsx": "catalog:", + "typescript": "catalog:" + }, + "engines": { + "node": ">=24" + } +} diff --git a/packages/integrations/examples/eve/src/e2e.ts b/packages/integrations/examples/eve/src/e2e.ts new file mode 100644 index 0000000000..7870be9fe8 --- /dev/null +++ b/packages/integrations/examples/eve/src/e2e.ts @@ -0,0 +1,12 @@ +import { runEveStagehandEval } from "./run-eval.js"; + +process.env.STAGEHAND_BROWSER ??= "local"; + +await runEveStagehandEval(false); +process.stdout.write( + `${JSON.stringify({ + status: "PASS", + framework: "eve", + proof: "local authenticated gateway and real Groq-backed Eve agent", + })}\n`, +); diff --git a/packages/integrations/examples/eve/src/gateway.ts b/packages/integrations/examples/eve/src/gateway.ts new file mode 100644 index 0000000000..f48b8811fc --- /dev/null +++ b/packages/integrations/examples/eve/src/gateway.ts @@ -0,0 +1,200 @@ +import assert from "node:assert/strict"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import http from "node:http"; +import { spawn, type ChildProcess } from "node:child_process"; + +const DEFAULT_STDIO_SERVER_PATH = fileURLToPath( + new URL("../../../dist/codemode/stdio-server.mjs", import.meta.url), +); +const SUPERGATEWAY_PATH = fileURLToPath( + new URL("../node_modules/.bin/supergateway", import.meta.url), +); +const MCP_PROTOCOL_VERSION = "2025-11-25"; + +export type StagehandGateway = { + url: string; + token: string; + close: () => Promise; +}; + +export async function startLocalTestGateway(options?: { + stdioServerPath?: string; +}): Promise { + const token = randomBytes(32).toString("hex"); + const upstreamPort = await reservePort(); + const stdioServerPath = options?.stdioServerPath ?? DEFAULT_STDIO_SERVER_PATH; + const stdioCommand = `${shellQuote(process.execPath)} ${shellQuote(stdioServerPath)}`; + const gateway = spawn( + SUPERGATEWAY_PATH, + [ + "--stdio", + stdioCommand, + "--outputTransport", + "streamableHttp", + "--stateful", + "--sessionTimeout", + "600000", + "--protocolVersion", + MCP_PROTOCOL_VERSION, + "--port", + String(upstreamPort), + "--healthEndpoint", + "/healthz", + "--logLevel", + "none", + ], + { + detached: process.platform !== "win32", + // This helper is local-test-only. Production uses the allowlisted guest + // environment in sandbox.ts and destroys the complete sandbox afterward. + env: localTestEnvironment(process.env), + stdio: ["ignore", "ignore", "pipe"], + }, + ); + + let stderr = ""; + gateway.stderr?.setEncoding("utf8"); + gateway.stderr?.on("data", (chunk: string) => { + stderr = `${stderr}${chunk}`.slice(-4_000); + }); + + try { + await waitForHealthyGateway(upstreamPort, gateway, () => stderr); + const proxy = await startAuthenticatedProxy({ token, upstreamPort }); + return { + url: `http://127.0.0.1:${proxy.port}/mcp`, + token, + async close() { + await proxy.close(); + await stopProcessTree(gateway); + }, + }; + } catch (error) { + await stopProcessTree(gateway); + throw error; + } +} + +async function startAuthenticatedProxy(options: { + token: string; + upstreamPort: number; +}): Promise<{ port: number; close: () => Promise }> { + const expectedAuthorization = Buffer.from(`Bearer ${options.token}`); + const server = http.createServer((request, response) => { + if (!isAuthorized(request.headers.authorization, expectedAuthorization)) { + response.writeHead(401, { "content-type": "text/plain" }); + response.end("Unauthorized\n"); + return; + } + + const upstream = http.request( + { + host: "127.0.0.1", + port: options.upstreamPort, + method: request.method, + path: request.url, + headers: forwardedMcpHeaders(request.headers), + }, + (upstreamResponse) => { + response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + upstreamResponse.pipe(response); + }, + ); + upstream.on("error", () => { + if (!response.headersSent) response.writeHead(502); + response.end("Upstream unavailable\n"); + }); + request.pipe(upstream); + }); + + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address === "object"); + + return { + port: address.port, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} + +function isAuthorized(header: string | undefined, expected: Buffer): boolean { + if (!header) return false; + const provided = Buffer.from(header); + return provided.length === expected.length && timingSafeEqual(provided, expected); +} + +function forwardedMcpHeaders(headers: http.IncomingHttpHeaders): http.OutgoingHttpHeaders { + return Object.fromEntries( + ["accept", "content-length", "content-type", "mcp-protocol-version", "mcp-session-id"] + .map((name) => [name, headers[name]] as const) + .filter((entry): entry is [string, string | string[]] => entry[1] !== undefined), + ); +} + +async function reservePort(): Promise { + const server = http.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address === "object"); + const port = address.port; + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + return port; +} + +async function waitForHealthyGateway( + port: number, + process: ChildProcess, + readStderr: () => string, +): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (process.exitCode !== null) { + throw new Error(`supergateway exited before startup: ${readStderr()}`); + } + const response = await fetch(`http://127.0.0.1:${port}/healthz`).catch(() => undefined); + if (response?.ok) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`supergateway did not become healthy: ${readStderr()}`); +} + +async function stopProcessTree(child: ChildProcess): Promise { + if (child.exitCode !== null || child.pid === undefined) return; + signalProcessTree(child, "SIGTERM"); + const stopped = await Promise.race([ + once(child, "exit").then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), + ]); + if (stopped) return; + signalProcessTree(child, "SIGKILL"); + await once(child, "exit").catch(() => undefined); +} + +function signalProcessTree(child: ChildProcess, signal: NodeJS.Signals): void { + try { + if (process.platform === "win32" || child.pid === undefined) child.kill(signal); + else process.kill(-child.pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } +} + +function localTestEnvironment(environment: NodeJS.ProcessEnv): Record { + return Object.fromEntries( + Object.entries(environment).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ); +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'`; +} diff --git a/packages/integrations/examples/eve/src/run-eval.ts b/packages/integrations/examples/eve/src/run-eval.ts new file mode 100644 index 0000000000..ce589bf291 --- /dev/null +++ b/packages/integrations/examples/eve/src/run-eval.ts @@ -0,0 +1,97 @@ +import assert from "node:assert/strict"; +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, rm } from "node:fs/promises"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { startLocalTestGateway } from "./gateway.js"; + +const EVE_BINARY = fileURLToPath(new URL("../node_modules/.bin/eve", import.meta.url)); +const EXAMPLE_ROOT = fileURLToPath(new URL("../", import.meta.url)); +const BUILT_SERVER = fileURLToPath(new URL("../.output/server/index.mjs", import.meta.url)); + +export async function runEveStagehandEval(deterministic: boolean): Promise { + const gateway = await startLocalTestGateway(); + try { + const unauthorized = await fetch(gateway.url); + assert.equal(unauthorized.status, 401, "the gateway must reject requests without its token"); + + const environment = { + ...process.env, + ...(deterministic ? { EVE_STAGEHAND_DETERMINISTIC: "1" } : {}), + STAGEHAND_MCP_URL: gateway.url, + STAGEHAND_MCP_TOKEN: gateway.token, + }; + await runChild(EVE_BINARY, ["build"], environment); + + const port = await reservePort(); + const serverRoot = await mkdtemp(join(tmpdir(), "eve-stagehand-")); + const server = spawn(process.execPath, [BUILT_SERVER], { + cwd: serverRoot, + env: { ...environment, HOST: "127.0.0.1", PORT: String(port) }, + stdio: "inherit", + }); + try { + await waitForAgent(port, server); + await runChild( + EVE_BINARY, + ["eval", "stagehand", "--skip-report", "--url", `http://127.0.0.1:${port}`], + environment, + ); + } finally { + await stopChild(server); + await rm(serverRoot, { recursive: true, force: true }); + } + } finally { + await gateway.close(); + } +} + +async function runChild( + command: string, + args: string[], + environment: NodeJS.ProcessEnv, +): Promise { + const child = spawn(command, args, { + cwd: EXAMPLE_ROOT, + env: environment, + stdio: "inherit", + }); + const [exitCode] = (await once(child, "exit")) as [number | null, NodeJS.Signals | null]; + if (exitCode !== 0) throw new Error(`${command} exited with code ${String(exitCode)}`); +} + +async function reservePort(): Promise { + const server = http.createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address === "object"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + return address.port; +} + +async function waitForAgent(port: number, child: ChildProcess): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (child.exitCode !== null) throw new Error("the built Eve server exited before startup"); + const response = await fetch(`http://127.0.0.1:${port}/eve/v1/health`).catch(() => undefined); + if (response?.ok) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("the built Eve server did not become healthy"); +} + +async function stopChild(child: ChildProcess): Promise { + if (child.exitCode !== null) return; + child.kill("SIGTERM"); + const stopped = await Promise.race([ + once(child, "exit").then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), + ]); + if (!stopped) child.kill("SIGKILL"); +} diff --git a/packages/integrations/examples/eve/src/sandbox-guest.mjs b/packages/integrations/examples/eve/src/sandbox-guest.mjs new file mode 100644 index 0000000000..a70cf1b12c --- /dev/null +++ b/packages/integrations/examples/eve/src/sandbox-guest.mjs @@ -0,0 +1,128 @@ +import assert from "node:assert/strict"; +import { timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import http from "node:http"; +import { spawn } from "node:child_process"; + +const port = Number.parseInt(requiredEnvironment("STAGEHAND_GATEWAY_PORT"), 10); +const token = requiredEnvironment("STAGEHAND_GATEWAY_TOKEN"); +const stdio = JSON.parse(requiredEnvironment("STAGEHAND_STDIO_COMMAND_JSON")); +assert.equal(typeof stdio.command, "string"); +assert.ok(Array.isArray(stdio.args) && stdio.args.every((value) => typeof value === "string")); + +const supergateway = spawn( + "/tmp/stagehand-eve-gateway/node_modules/.bin/supergateway", + [ + "--stdio", + [stdio.command, ...stdio.args].map(shellQuote).join(" "), + "--outputTransport", + "streamableHttp", + "--stateful", + "--sessionTimeout", + "600000", + "--protocolVersion", + "2025-11-25", + "--port", + String(port + 1), + "--healthEndpoint", + "/healthz", + "--logLevel", + "none", + ], + { detached: true, env: process.env, stdio: ["ignore", "ignore", "pipe"] }, +); + +let stderr = ""; +supergateway.stderr.setEncoding("utf8"); +supergateway.stderr.on("data", (chunk) => { + stderr = `${stderr}${chunk}`.slice(-4_000); +}); + +await waitForHealthyGateway(port + 1); + +const expectedAuthorization = Buffer.from(`Bearer ${token}`); +const server = http.createServer((request, response) => { + if (!isAuthorized(request.headers.authorization)) { + response.writeHead(401, { "content-type": "text/plain" }); + response.end("Unauthorized\n"); + return; + } + + const upstream = http.request( + { + host: "127.0.0.1", + port: port + 1, + method: request.method, + path: request.url, + headers: forwardedMcpHeaders(request.headers), + }, + (upstreamResponse) => { + response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + upstreamResponse.pipe(response); + }, + ); + upstream.on("error", () => { + if (!response.headersSent) response.writeHead(502); + response.end("Upstream unavailable\n"); + }); + request.pipe(upstream); +}); + +server.listen(port, "0.0.0.0"); +await once(server, "listening"); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.once(signal, async () => { + await new Promise((resolve) => server.close(resolve)); + stopProcessTree("SIGTERM"); + process.exit(0); + }); +} + +function requiredEnvironment(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function isAuthorized(header) { + if (!header) return false; + const provided = Buffer.from(header); + return ( + provided.length === expectedAuthorization.length && + timingSafeEqual(provided, expectedAuthorization) + ); +} + +function forwardedMcpHeaders(headers) { + return Object.fromEntries( + ["accept", "content-length", "content-type", "mcp-protocol-version", "mcp-session-id"] + .map((name) => [name, headers[name]]) + .filter((entry) => entry[1] !== undefined), + ); +} + +async function waitForHealthyGateway(upstreamPort) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (supergateway.exitCode !== null) { + throw new Error(`supergateway exited before startup: ${stderr}`); + } + const response = await fetch(`http://127.0.0.1:${upstreamPort}/healthz`).catch(() => undefined); + if (response?.ok) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`supergateway did not become healthy: ${stderr}`); +} + +function stopProcessTree(signal) { + if (supergateway.pid === undefined) return; + try { + process.kill(-supergateway.pid, signal); + } catch (error) { + if (error.code !== "ESRCH") throw error; + } +} + +function shellQuote(value) { + return `'${value.replaceAll("'", `'\\''`)}'`; +} diff --git a/packages/integrations/examples/eve/src/sandbox.ts b/packages/integrations/examples/eve/src/sandbox.ts new file mode 100644 index 0000000000..f28c9949cd --- /dev/null +++ b/packages/integrations/examples/eve/src/sandbox.ts @@ -0,0 +1,119 @@ +import { randomBytes } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +export const PROPOSED_STAGEHAND_CODEMODE_IMAGE = "ghcr.io/browserbase/stagehand-codemode"; + +const GUEST_GATEWAY_PATH = "/tmp/stagehand-eve-gateway/gateway.mjs"; +const GUEST_GATEWAY_SOURCE = new URL("./sandbox-guest.mjs", import.meta.url); + +type GuestEnvironment = Partial< + Record< + | "BROWSERBASE_API_KEY" + | "BROWSERBASE_PROJECT_ID" + | "STAGEHAND_BROWSER" + | "STAGEHAND_MODEL_API_KEY" + | "STAGEHAND_MODEL_NAME", + string + > +>; + +export type SandboxProcess = { + wait: () => Promise<{ exitCode: number }>; + kill: (signal: "SIGTERM" | "SIGKILL") => Promise; +}; + +export type SandboxInstance = { + /** + * Command for the Stagehand stdio server inside this sandbox. A provider can + * materialize the OCI image as the sandbox rootfs, or return a nested + * container-runtime command such as `docker run --rm -i @`. + */ + stdioCommand: { command: string; args: string[] }; + publicUrl: (port: number) => Promise; + writeTextFile: (path: string, contents: string) => Promise; + spawn: (options: { + command: string; + args: string[]; + env: Record; + }) => Promise; + close: () => Promise; +}; + +export type SandboxProvider = { + create: (options: { + stdioImage: string; + exposedPorts: number[]; + timeoutMs: number; + }) => Promise; +}; + +export async function createStagehandSandboxGateway( + provider: SandboxProvider, + options: { + image?: string; + timeoutMs?: number; + environment?: GuestEnvironment; + } = {}, +): Promise<{ url: string; token: string; close: () => Promise }> { + const port = 3000; + const token = randomBytes(32).toString("hex"); + const sandbox = await provider.create({ + stdioImage: options.image ?? PROPOSED_STAGEHAND_CODEMODE_IMAGE, + exposedPorts: [port], + timeoutMs: options.timeoutMs ?? 15 * 60_000, + }); + + try { + await sandbox.writeTextFile(GUEST_GATEWAY_PATH, await readFile(GUEST_GATEWAY_SOURCE, "utf8")); + const process = await sandbox.spawn({ + command: "/bin/sh", + args: [ + "-lc", + [ + "set -eu", + "cd /tmp/stagehand-eve-gateway", + "npm init -y >/dev/null 2>&1", + "npm install --ignore-scripts --no-audit --no-fund supergateway@3.4.3 >/dev/null 2>&1", + `exec node ${GUEST_GATEWAY_PATH}`, + ].join("\n"), + ], + env: { + ...options.environment, + STAGEHAND_GATEWAY_PORT: String(port), + STAGEHAND_GATEWAY_TOKEN: token, + STAGEHAND_STDIO_COMMAND_JSON: JSON.stringify(sandbox.stdioCommand), + }, + }); + const baseUrl = (await sandbox.publicUrl(port)).replace(/\/$/, ""); + await waitForGateway(baseUrl, token); + const url = `${baseUrl}/mcp`; + + return { + url, + token, + async close() { + await process.kill("SIGTERM").catch(() => undefined); + const stopped = await Promise.race([ + process.wait().then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), + ]); + if (!stopped) await process.kill("SIGKILL").catch(() => undefined); + await sandbox.close(); + }, + }; + } catch (error) { + await sandbox.close().catch(() => undefined); + throw error; + } +} + +async function waitForGateway(baseUrl: string, token: string): Promise { + for (let attempt = 0; attempt < 150; attempt += 1) { + const response = await fetch(`${baseUrl}/healthz`, { + headers: { authorization: `Bearer ${token}` }, + }).catch(() => undefined); + if (response?.ok) return; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new Error("the sandboxed Stagehand gateway did not become healthy"); +} diff --git a/packages/integrations/examples/eve/src/smoke.ts b/packages/integrations/examples/eve/src/smoke.ts new file mode 100644 index 0000000000..cf09b49020 --- /dev/null +++ b/packages/integrations/examples/eve/src/smoke.ts @@ -0,0 +1,15 @@ +import { runEveStagehandEval } from "./run-eval.js"; + +process.env.STAGEHAND_BROWSER ??= "local"; + +await runEveStagehandEval(true); +process.stdout.write( + `${JSON.stringify({ + status: "PASS", + framework: "eve", + proof: "local authenticated gateway and deterministic Eve agent", + tools: ["connection_search", "stagehand__code_execute"], + codeExecuteCalls: 1, + unauthorizedStatus: 401, + })}\n`, +); diff --git a/packages/integrations/examples/eve/tsconfig.json b/packages/integrations/examples/eve/tsconfig.json new file mode 100644 index 0000000000..3512e1ce7e --- /dev/null +++ b/packages/integrations/examples/eve/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true, + "skipLibCheck": true + }, + "include": ["agent/**/*.ts", "evals/**/*.ts", "src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b6fd94694a..422512de55 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -298,6 +298,9 @@ catalogs: esbuild: specifier: 0.28.1 version: 0.28.1 + eve: + specifier: 0.29.4 + version: 0.29.4 fflate: specifier: ^0.8.3 version: 0.8.3 @@ -325,6 +328,9 @@ catalogs: snakecase-keys: specifier: ^9.0.2 version: 9.0.2 + supergateway: + specifier: 3.4.3 + version: 3.4.3 tsdown: specifier: 0.22.3 version: 0.22.3 @@ -598,6 +604,34 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0)) + packages/integrations/examples/eve: + dependencies: + '@ai-sdk/groq': + specifier: 'catalog:' + version: 4.0.5(zod@4.4.3) + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../.. + eve: + specifier: 'catalog:' + version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@1.21.7)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + supergateway: + specifier: 'catalog:' + version: 3.4.3(bufferutil@4.1.0) + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsx: + specifier: 'catalog:' + version: 4.23.1 + typescript: + specifier: 'catalog:' + version: 5.9.3 + packages/integrations/examples/mastra: dependencies: '@browserbasehq/stagehand-integrations': @@ -3867,6 +3901,10 @@ packages: concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -3934,6 +3972,14 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crossws@0.4.10: + resolution: {integrity: sha512-pz3oubH/dt12KjqsUB0IuXW4nwRDQ583iDsP4555Cpdqx0NoU7pGlWBcayyFI8f/l/idRpgjMEfwuOxSWJYlIA==} + peerDependencies: + srvx: '>=0.11.5' + peerDependenciesMeta: + srvx: + optional: true + css-select@5.2.2: resolution: {integrity: sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==} @@ -3978,6 +4024,29 @@ packages: dateformat@4.6.3: resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + db0@0.3.4: + resolution: {integrity: sha512-RiXXi4WaNzPTHEOu8UPQKMooIbqOEyqA1t7Z6MsdxSCeb8iUC9ko3LcmsLmeUt2SM5bctfArZKkRQggKZz7JNw==} + peerDependencies: + '@electric-sql/pglite': '*' + '@libsql/client': '*' + better-sqlite3: '*' + drizzle-orm: '*' + mysql2: '*' + sqlite3: '*' + peerDependenciesMeta: + '@electric-sql/pglite': + optional: true + '@libsql/client': + optional: true + better-sqlite3: + optional: true + drizzle-orm: + optional: true + mysql2: + optional: true + sqlite3: + optional: true + debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -4211,6 +4280,24 @@ packages: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} + env-runner@0.1.16: + resolution: {integrity: sha512-2LRJM4P2KLX6J83QZZrMqvgCDt/D5ea7wPcI3yYiy5cG/9rX5QwdwZFx0D7ktWnjdRyZxYjttGGorb5nFqb1CA==} + hasBin: true + peerDependencies: + '@netlify/runtime': ^4.1.23 + '@vercel/queue': '>=0.2.0' + miniflare: ^4.20260515.0 + wrangler: ^4.0.0 + peerDependenciesMeta: + '@netlify/runtime': + optional: true + '@vercel/queue': + optional: true + miniflare: + optional: true + wrangler: + optional: true + environment@1.1.0: resolution: {integrity: sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==} engines: {node: '>=18'} @@ -4334,6 +4421,26 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + eve@0.29.4: + resolution: {integrity: sha512-EwOmL37l+Iuu7Umno7at3flFpMs4AtT9cg1J4dt7EZ8ZdxaCvGXwvKGwiulMkG8oXkMQ5CNhxMi2ruHJwrtpwQ==} + engines: {node: '>=24'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.0.0 + ai: ^7.0.38 + braintrust: ^3.0.0 + just-bash: ^3.0.0 + microsandbox: ^0.5.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + braintrust: + optional: true + just-bash: + optional: true + microsandbox: + optional: true + event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} @@ -4391,6 +4498,9 @@ packages: resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} engines: {node: '>= 18'} + exsolve@1.1.1: + resolution: {integrity: sha512-9U/jZUgjnSGyntRr6y5Muu1MJcwFl6kPu7k8qLF0IMNfLqvw0NZ4nnVDq0RVoZ0RvCyumib4Ez3KYrVfilrw+g==} + extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} engines: {node: '>=0.10.0'} @@ -4721,6 +4831,16 @@ packages: resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} engines: {node: '>=6.0'} + h3@2.0.1-rc.22: + resolution: {integrity: sha512-Esv0DMIuPkCTSWCA0vO73vcTqwzH1wjSrAO1TXNu/K3up1sZHa9EKMapbmxCDYBeymC3fVTk4qxp7ogQWQ+KgA==} + engines: {node: '>=20.11.1'} + hasBin: true + peerDependencies: + crossws: ^0.4.1 + peerDependenciesMeta: + crossws: + optional: true + has-bigints@1.1.0: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} @@ -4855,6 +4975,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + httpxy@0.5.5: + resolution: {integrity: sha512-uDjmnPyp1q4Sgzf3w+J/Fc6UqcCEj0x4Wjp7OqK5dGhNeDgpyrAmnS6ey8QWrX3SWDon2DMKf9sBa5X9+CVyMA==} + human-id@4.2.0: resolution: {integrity: sha512-K3GbkIWqyvvlpfhBPlbEvD97TtqBpAYA4kt+cn2lD2x2HuohzZCibcA2nOlnJT6exqvJLggoB5nv2dNf192nEA==} hasBin: true @@ -5757,10 +5880,44 @@ packages: react: '>= 18.3.0 < 19.0.0' react-dom: '>= 18.3.0 < 19.0.0' + nf3@0.3.23: + resolution: {integrity: sha512-RWVLAWozmVD3AaDmaU3qMGB3v+yNlH5d9qqStI4e/WLlNQVnJ4YErGDbYCIrGFyrHdbF6I6Baf0Ae6c7tFYmSg==} + nimma@0.2.3: resolution: {integrity: sha512-1ZOI8J+1PKKGceo/5CT5GfQOG6H8I2BencSK06YarZ2wXwH37BSSUWldqJmMJYA5JfqDqffxDXynt6f11AyKcA==} engines: {node: ^12.20 || >=14.13} + nitro@3.0.260610-beta: + resolution: {integrity: sha512-KPb4L5yaF/Rx/xoGMpgHRJvZhbhGiqbRKOwwPLCH9jKTKTsEUHLjnJas85AeCzaswqa8Wi52eQBtRsODC4PS0Q==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@vercel/queue': ^0.3.0 + dotenv: '*' + giget: '*' + jiti: ^2.7.0 + rollup: ^4.61.1 + vite: 8.1.3 + xml2js: ^0.6.2 + zephyr-agent: ^0.2.0 + peerDependenciesMeta: + '@vercel/queue': + optional: true + dotenv: + optional: true + giget: + optional: true + jiti: + optional: true + rollup: + optional: true + vite: + optional: true + xml2js: + optional: true + zephyr-agent: + optional: true + nlcst-to-string@4.0.0: resolution: {integrity: sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==} @@ -5976,6 +6133,15 @@ packages: resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} engines: {node: '>=12.20.0'} + ocache@0.1.5: + resolution: {integrity: sha512-kNNnkkVQup/QDvmTz8Q84wc2ntiyoVHDxa6eHWKt5qdGAmFRBIxy83rxgCYEjW0x06UJ9E3P6VgM2yY4rOBH4w==} + + ofetch@2.0.0-alpha.3: + resolution: {integrity: sha512-zpYTCs2byOuft65vI3z43Dd6iSdFbOZZLb9/d21aCpx2rGastVU9dOCv0lu4ykc1Ur1anAYjDi3SUvR0vq50JA==} + + ohash@2.0.11: + resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} + ollama-ai-provider-v2@1.5.5: resolution: {integrity: sha512-1YwTFdPjhPNHny/DrOHO+s8oVGGIE5Jib61/KnnjPRNWQhVVimrJJdaAX3e6nNRRDXrY5zbb9cfm2+yVvgsrqw==} engines: {node: '>=18'} @@ -6713,6 +6879,9 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + rou3@0.8.1: + resolution: {integrity: sha512-ePa+XGk00/3HuCqrEnK3LxJW7I0SdNg6EFzKUJG73hMAdDcOUC/i/aSz7LSDwLrGr33kal/rqOGydzwl6U7zBA==} + router@2.2.0: resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} engines: {node: '>= 18'} @@ -6959,6 +7128,11 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + srvx@0.11.22: + resolution: {integrity: sha512-LqZxxBDMKuMAZzFzJnDCkFOrs9MZQZr0LvHiO/SuSZVdQaXD7xQ5UWTUxheJrQPve1qk9MG2B/yttUvJxw8egQ==} + engines: {node: '>=20.16.0'} + hasBin: true + stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -7064,6 +7238,10 @@ packages: engines: {node: '>=16 || 14 >=14.17'} hasBin: true + supergateway@3.4.3: + resolution: {integrity: sha512-lidAbuX84K8gRp+TU+KLGqEu5ne8Ihef53y7+h5L0cFkL8yY1MDPbIGHw1gkPQuaTlsqsikLimt1cc2lt7yVuQ==} + hasBin: true + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -7308,6 +7486,13 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@8.9.0: + resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} + engines: {node: '>=22.19.0'} + + unenv@2.0.0-rc.24: + resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + unicorn-magic@0.3.0: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} @@ -7378,6 +7563,80 @@ packages: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} + unstorage@2.0.0-alpha.7: + resolution: {integrity: sha512-ELPztchk2zgFJnakyodVY3vJWGW9jy//keJ32IOJVGUMyaPydwcA1FtVvWqT0TNRch9H+cMNEGllfVFfScImog==} + peerDependencies: + '@azure/app-configuration': ^1.11.0 + '@azure/cosmos': ^4.9.1 + '@azure/data-tables': ^13.3.2 + '@azure/identity': ^4.13.0 + '@azure/keyvault-secrets': ^4.10.0 + '@azure/storage-blob': ^12.31.0 + '@capacitor/preferences': ^6 || ^7 || ^8 + '@deno/kv': '>=0.13.0' + '@netlify/blobs': ^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0 + '@planetscale/database': ^1.19.0 + '@upstash/redis': ^1.36.2 + '@vercel/blob': '>=0.27.3' + '@vercel/functions': ^2.2.12 || ^3.0.0 + '@vercel/kv': ^1.0.1 + aws4fetch: ^1.0.20 + chokidar: ^4 || ^5 + db0: '>=0.3.4' + idb-keyval: ^6.2.2 + ioredis: ^5.9.3 + lru-cache: ^11.2.6 + mongodb: ^6 || ^7 + ofetch: '*' + uploadthing: ^7.7.4 + peerDependenciesMeta: + '@azure/app-configuration': + optional: true + '@azure/cosmos': + optional: true + '@azure/data-tables': + optional: true + '@azure/identity': + optional: true + '@azure/keyvault-secrets': + optional: true + '@azure/storage-blob': + optional: true + '@capacitor/preferences': + optional: true + '@deno/kv': + optional: true + '@netlify/blobs': + optional: true + '@planetscale/database': + optional: true + '@upstash/redis': + optional: true + '@vercel/blob': + optional: true + '@vercel/functions': + optional: true + '@vercel/kv': + optional: true + aws4fetch: + optional: true + chokidar: + optional: true + db0: + optional: true + idb-keyval: + optional: true + ioredis: + optional: true + lru-cache: + optional: true + mongodb: + optional: true + ofetch: + optional: true + uploadthing: + optional: true + urijs@1.19.11: resolution: {integrity: sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==} @@ -9398,6 +9657,28 @@ snapshots: react: 19.2.3 react-dom: 18.3.1(react@19.2.3) + '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': + dependencies: + '@hono/node-server': 1.19.15(hono@4.12.32) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.1.0 + express: 5.2.1 + express-rate-limit: 8.6.0(express@5.2.1) + hono: 4.12.32 + jose: 6.2.4 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 3.25.76 + zod-to-json-schema: 3.25.2(zod@3.25.76) + transitivePeerDependencies: + - supports-color + '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: '@hono/node-server': 1.19.15(hono@4.12.32) @@ -11090,6 +11371,8 @@ snapshots: concat-map@0.0.1: {} + consola@3.4.2: {} + content-disposition@0.5.4: dependencies: safe-buffer: 5.2.1 @@ -11141,6 +11424,10 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crossws@0.4.10(srvx@0.11.22): + optionalDependencies: + srvx: 0.11.22 + css-select@5.2.2: dependencies: boolbase: 1.0.0 @@ -11183,6 +11470,8 @@ snapshots: dateformat@4.6.3: {} + db0@0.3.4: {} + debug@2.6.9: dependencies: ms: 2.0.0 @@ -11382,6 +11671,13 @@ snapshots: env-paths@2.2.1: {} + env-runner@0.1.16: + dependencies: + crossws: 0.4.10(srvx@0.11.22) + exsolve: 1.1.1 + httpxy: 0.5.5 + srvx: 0.11.22 + environment@1.1.0: {} error-ex@1.3.4: @@ -11622,6 +11918,53 @@ snapshots: etag@1.8.1: {} + eve@0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@1.21.7)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): + dependencies: + ai: 7.0.16(zod@4.4.3) + nitro: 3.0.260610-beta(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@1.21.7)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) + undici: 8.9.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + transitivePeerDependencies: + - '@azure/app-configuration' + - '@azure/cosmos' + - '@azure/data-tables' + - '@azure/identity' + - '@azure/keyvault-secrets' + - '@azure/storage-blob' + - '@capacitor/preferences' + - '@deno/kv' + - '@electric-sql/pglite' + - '@libsql/client' + - '@netlify/blobs' + - '@netlify/runtime' + - '@planetscale/database' + - '@upstash/redis' + - '@vercel/blob' + - '@vercel/functions' + - '@vercel/kv' + - '@vercel/queue' + - aws4fetch + - better-sqlite3 + - chokidar + - dotenv + - drizzle-orm + - giget + - idb-keyval + - ioredis + - jiti + - lru-cache + - miniflare + - mongodb + - mysql2 + - rollup + - sqlite3 + - uploadthing + - vite + - wrangler + - xml2js + - zephyr-agent + event-target-shim@5.0.1: {} events-universal@1.0.1: @@ -11741,6 +12084,8 @@ snapshots: transitivePeerDependencies: - supports-color + exsolve@1.1.1: {} + extend-shallow@2.0.1: dependencies: is-extendable: 0.1.1 @@ -12142,6 +12487,13 @@ snapshots: section-matter: 1.0.0 strip-bom-string: 1.0.0 + h3@2.0.1-rc.22(crossws@0.4.10(srvx@0.11.22)): + dependencies: + rou3: 0.8.1 + srvx: 0.11.22 + optionalDependencies: + crossws: 0.4.10(srvx@0.11.22) + has-bigints@1.1.0: {} has-flag@4.0.0: {} @@ -12393,6 +12745,8 @@ snapshots: transitivePeerDependencies: - supports-color + httpxy@0.5.5: {} + human-id@4.2.0: {} human-signals@8.0.1: {} @@ -13550,6 +13904,8 @@ snapshots: - supports-color - unified + nf3@0.3.23: {} + nimma@0.2.3: dependencies: '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) @@ -13560,6 +13916,59 @@ snapshots: jsonpath-plus: 10.4.0 lodash.topath: 4.5.2 + nitro@3.0.260610-beta(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@1.21.7)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2): + dependencies: + consola: 3.4.2 + crossws: 0.4.10(srvx@0.11.22) + db0: 0.3.4 + env-runner: 0.1.16 + h3: 2.0.1-rc.22(crossws@0.4.10(srvx@0.11.22)) + hookable: 6.1.1 + nf3: 0.3.23 + ocache: 0.1.5 + ofetch: 2.0.0-alpha.3 + ohash: 2.0.11 + rolldown: 1.1.5 + srvx: 0.11.22 + unenv: 2.0.0-rc.24 + unstorage: 2.0.0-alpha.7(aws4fetch@1.0.20)(db0@0.3.4)(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3) + optionalDependencies: + dotenv: 17.4.2 + jiti: 1.21.7 + vite: 8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0) + xml2js: 0.6.2 + transitivePeerDependencies: + - '@azure/app-configuration' + - '@azure/cosmos' + - '@azure/data-tables' + - '@azure/identity' + - '@azure/keyvault-secrets' + - '@azure/storage-blob' + - '@capacitor/preferences' + - '@deno/kv' + - '@electric-sql/pglite' + - '@libsql/client' + - '@netlify/blobs' + - '@netlify/runtime' + - '@planetscale/database' + - '@upstash/redis' + - '@vercel/blob' + - '@vercel/functions' + - '@vercel/kv' + - aws4fetch + - better-sqlite3 + - chokidar + - drizzle-orm + - idb-keyval + - ioredis + - lru-cache + - miniflare + - mongodb + - mysql2 + - sqlite3 + - uploadthing + - wrangler + nlcst-to-string@4.0.0: dependencies: '@types/nlcst': 2.0.3 @@ -13638,6 +14047,14 @@ snapshots: obug@2.1.3: {} + ocache@0.1.5: + dependencies: + ohash: 2.0.11 + + ofetch@2.0.0-alpha.3: {} + + ohash@2.0.11: {} + ollama-ai-provider-v2@1.5.5(zod@4.4.3): dependencies: '@ai-sdk/provider': 2.0.3 @@ -14574,6 +14991,8 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.1.5 '@rolldown/binding-win32-x64-msvc': 1.1.5 + rou3@0.8.1: {} + router@2.2.0: dependencies: debug: 4.4.3(supports-color@8.1.1) @@ -14956,6 +15375,8 @@ snapshots: sprintf-js@1.0.3: {} + srvx@0.11.22: {} + stack-utils@2.0.6: dependencies: escape-string-regexp: 2.0.0 @@ -15088,6 +15509,22 @@ snapshots: tinyglobby: 0.2.17 ts-interface-checker: 0.1.13 + supergateway@3.4.3(bufferutil@4.1.0): + dependencies: + '@modelcontextprotocol/sdk': 1.29.0(zod@3.25.76) + body-parser: 1.20.6 + cors: 2.8.6 + express: 4.22.0 + uuid: 11.1.1 + ws: 8.21.0(bufferutil@4.1.0) + yargs: 17.7.3 + zod: 3.25.76 + transitivePeerDependencies: + - '@cfworker/json-schema' + - bufferutil + - supports-color + - utf-8-validate + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -15384,6 +15821,12 @@ snapshots: undici-types@7.24.6: {} + undici@8.9.0: {} + + unenv@2.0.0-rc.24: + dependencies: + pathe: 2.0.3 + unicorn-magic@0.3.0: {} unified@11.0.5: @@ -15482,6 +15925,13 @@ snapshots: unpipe@1.0.0: {} + unstorage@2.0.0-alpha.7(aws4fetch@1.0.20)(db0@0.3.4)(lru-cache@11.5.2)(ofetch@2.0.0-alpha.3): + optionalDependencies: + aws4fetch: 1.0.20 + db0: 0.3.4 + lru-cache: 11.5.2 + ofetch: 2.0.0-alpha.3 + urijs@1.19.11: {} use-callback-ref@1.3.3(@types/react@19.2.17)(react@19.2.3): @@ -15793,6 +16243,10 @@ snapshots: dependencies: zod: 3.24.0 + zod-to-json-schema@3.25.2(zod@3.25.76): + dependencies: + zod: 3.25.76 + zod-to-json-schema@3.25.2(zod@4.4.3): dependencies: zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8f76f69b5b..359b10a0bc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,6 +9,8 @@ catalog: "@mastra/core": ^1.55.0 "@mastra/mcp": ^1.15.0 "@modelcontextprotocol/sdk": 1.29.0 + eve: 0.29.4 + supergateway: 3.4.3 "@ast-grep/lang-go": 0.0.6 "@ast-grep/lang-python": 0.0.6 "@ast-grep/napi": 0.44.1 From 4c26440257a660fc82fdcee2305ac2c7366d2d39 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 7 Aug 2026 17:29:27 -0700 Subject: [PATCH 2/6] fix(eve): wait for sandbox gateway readiness --- packages/integrations/examples/eve/README.md | 6 ++++-- .../integrations/examples/eve/src/sandbox.ts | 21 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/integrations/examples/eve/README.md b/packages/integrations/examples/eve/README.md index fe87979932..018046aee9 100644 --- a/packages/integrations/examples/eve/README.md +++ b/packages/integrations/examples/eve/README.md @@ -102,8 +102,10 @@ proxy run inside the guest. The adapter must also materialize the pinned Stageha inside the guest—such as through a mirrored runtime image or an exact source build—and return the resulting local Node command as `stdioCommand`; do not assume nested Docker is available. -`createStagehandSandboxGateway` polls the authenticated `/healthz` endpoint before returning. This -prevents Eve from racing the guest bootstrap or public port publication. +`createStagehandSandboxGateway` polls the authenticated `/healthz` endpoint before returning. The +default startup deadline is two minutes, is configurable with `startupTimeoutMs`, and is capped +below the complete sandbox lifetime. This prevents Eve from racing a cold guest bootstrap or +public port publication without permitting an unbounded wait. The host starts one authenticated proxy and one stateful `supergateway` process per sandbox. `supergateway` starts one Stagehand stdio child for each MCP session. `supergateway` is a diff --git a/packages/integrations/examples/eve/src/sandbox.ts b/packages/integrations/examples/eve/src/sandbox.ts index f28c9949cd..64315c8d40 100644 --- a/packages/integrations/examples/eve/src/sandbox.ts +++ b/packages/integrations/examples/eve/src/sandbox.ts @@ -51,16 +51,22 @@ export async function createStagehandSandboxGateway( provider: SandboxProvider, options: { image?: string; + startupTimeoutMs?: number; timeoutMs?: number; environment?: GuestEnvironment; } = {}, ): Promise<{ url: string; token: string; close: () => Promise }> { const port = 3000; const token = randomBytes(32).toString("hex"); + const timeoutMs = options.timeoutMs ?? 15 * 60_000; + const startupTimeoutMs = Math.min(options.startupTimeoutMs ?? 2 * 60_000, timeoutMs - 1_000); + if (startupTimeoutMs <= 0) { + throw new Error("the sandbox timeout must leave at least one second for gateway startup"); + } const sandbox = await provider.create({ stdioImage: options.image ?? PROPOSED_STAGEHAND_CODEMODE_IMAGE, exposedPorts: [port], - timeoutMs: options.timeoutMs ?? 15 * 60_000, + timeoutMs, }); try { @@ -85,7 +91,7 @@ export async function createStagehandSandboxGateway( }, }); const baseUrl = (await sandbox.publicUrl(port)).replace(/\/$/, ""); - await waitForGateway(baseUrl, token); + await waitForGateway(baseUrl, token, startupTimeoutMs); const url = `${baseUrl}/mcp`; return { @@ -107,13 +113,18 @@ export async function createStagehandSandboxGateway( } } -async function waitForGateway(baseUrl: string, token: string): Promise { - for (let attempt = 0; attempt < 150; attempt += 1) { +async function waitForGateway( + baseUrl: string, + token: string, + startupTimeoutMs: number, +): Promise { + const deadline = Date.now() + startupTimeoutMs; + while (Date.now() < deadline) { const response = await fetch(`${baseUrl}/healthz`, { headers: { authorization: `Bearer ${token}` }, }).catch(() => undefined); if (response?.ok) return; - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 250)); } throw new Error("the sandboxed Stagehand gateway did not become healthy"); } From a5119e2f41a078ac6563e8a2b67d0b1084fcafca Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Fri, 7 Aug 2026 17:37:05 -0700 Subject: [PATCH 3/6] docs(eve): record exact sandbox proof gap --- packages/integrations/examples/eve/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/integrations/examples/eve/README.md b/packages/integrations/examples/eve/README.md index 018046aee9..b979c6c4ec 100644 --- a/packages/integrations/examples/eve/README.md +++ b/packages/integrations/examples/eve/README.md @@ -133,12 +133,13 @@ We verified the two critical pieces independently: - Eve `0.29.4` passed both the deterministic and real-model flows below through the same gateway implementation on localhost. -The full composition is still partial. A host Eve `0.29.3` run reached the sandbox gateway, -initialized a stateful MCP session, sent the initialized notification, and opened its authenticated -SSE stream, but did not advance to `tools/list` or `tools/call`. That exact version-mismatched proof -does not establish an Eve `0.29.4` to sandbox `code_execute` loop. Treat the architecture as -supported but keep the composed deployment behind an integration test until that Streamable HTTP -client interoperability gap is closed. +The full composition is still partial. A fresh host Eve `0.29.4` run against the same public +sandbox first made an SSE request without a session (`400`), then initialized a stateful MCP +session (`200`), sent the initialized notification (`202`), and opened its authenticated session +SSE stream (`200`). It did not advance to `tools/list` or `tools/call` before the bounded 150-second +proof timed out. This exact-version run therefore does not establish an Eve-to-sandbox +`code_execute` loop. Treat the architecture as supported but keep the composed deployment behind +an integration test until that Streamable HTTP client interoperability gap is closed. ## Run the deterministic smoke From 0c6f28204e648544a1fc1ad301af796ead104588 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Sat, 8 Aug 2026 03:55:04 +0000 Subject: [PATCH 4/6] refactor(eve): use shared sandboxed code-mode MCP --- .../workflows/codemode-framework-examples.yml | 23 +- packages/integrations/examples/eve/README.md | 199 +++++------------ .../integrations/examples/eve/agent/agent.ts | 49 ++++- .../examples/eve/agent/instructions.md | 4 +- .../examples/eve/evals/stagehand.eval.ts | 26 ++- .../integrations/examples/eve/package.json | 8 +- .../examples/eve/src/contract-server.ts | 107 ++++++++++ .../integrations/examples/eve/src/contract.ts | 37 ++++ packages/integrations/examples/eve/src/e2e.ts | 67 +++++- .../integrations/examples/eve/src/gateway.ts | 200 ------------------ .../integrations/examples/eve/src/run-eval.ts | 105 +++++---- .../examples/eve/src/sandbox-guest.mjs | 128 ----------- .../integrations/examples/eve/src/sandbox.ts | 130 ------------ .../integrations/examples/eve/src/smoke.ts | 15 -- pnpm-lock.yaml | 26 +-- pnpm-workspace.yaml | 1 - 16 files changed, 418 insertions(+), 707 deletions(-) create mode 100644 packages/integrations/examples/eve/src/contract-server.ts create mode 100644 packages/integrations/examples/eve/src/contract.ts delete mode 100644 packages/integrations/examples/eve/src/gateway.ts delete mode 100644 packages/integrations/examples/eve/src/sandbox-guest.mjs delete mode 100644 packages/integrations/examples/eve/src/sandbox.ts delete mode 100644 packages/integrations/examples/eve/src/smoke.ts diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index 7a6d0b9e98..7345114eae 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -199,6 +199,10 @@ jobs: eve: name: Eve + if: >- + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository || + contains(github.event.pull_request.labels.*.name, 'safe-to-test') runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -208,12 +212,17 @@ jobs: with: use-prebuilt-artifacts: "false" - - uses: ./.github/actions/setup-chrome-verified - id: setup-chrome - - - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve typecheck - - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve smoke + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve contract + - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts + - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e env: - CHROME_PATH: ${{ steps.setup-chrome.outputs.chrome-path }} - STAGEHAND_BROWSER: local + STAGEHAND_SANDBOX_ARTIFACTS: ${{ github.workspace }}/packages/integrations/examples/vercel-sandbox/.artifacts + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/packages/integrations/examples/eve/README.md b/packages/integrations/examples/eve/README.md index b979c6c4ec..8f6574daba 100644 --- a/packages/integrations/examples/eve/README.md +++ b/packages/integrations/examples/eve/README.md @@ -1,172 +1,69 @@ -# Eve + Stagehand code mode +# Eve with Stagehand code mode -Eve accepts remote Streamable HTTP or SSE MCP connections. It does not launch stdio MCP servers -directly. This example keeps the Eve host outside the execution boundary and puts the unchanged -Stagehand stdio server, its browser session, and a small HTTP adapter inside a sandbox. +This example gives an Eve agent one browser tool, `stagehand__code_execute`, without running +generated JavaScript in the Eve process. Eve stays on the host while the package-installed +[Vercel Sandbox example](../vercel-sandbox) owns Stagehand, the browser session, and generated code: ```text -Eve host - | - | Streamable HTTP + bearer token - v -Firecracker or gVisor sandbox - |-- authenticated proxy - `-- supergateway (non-first-party) - | - | stdio - v - Stagehand code-mode MCP - | - v - generated JavaScript +Eve connection_search -> authenticated Streamable HTTP -> Vercel Sandbox -> Stagehand MCP + `-> generated JavaScript ``` -The process boundary between `supergateway` and Stagehand is not the security boundary. Generated -JavaScript inherits the MCP process's filesystem, environment, and network. The Firecracker -microVM or gVisor sandbox is the boundary that protects the Eve host. +[`agent/connections/stagehand.ts`](./agent/connections/stagehand.ts) is the complete framework +adapter. It receives the shared sandbox `{ url, token }`, uses Eve's native +`defineMcpClientConnection`, and allows only `code_execute`. Eve qualifies the discovered tool as +`stagehand__code_execute`; the canonical Stagehand guidance stays in the MCP tool description. -## Configure the Eve connection +## Dependency version -[`agent/connections/stagehand.ts`](./agent/connections/stagehand.ts) is the complete Eve connection: +The example pins Eve 0.29.4, the newest release old enough for this repository's dependency-age +policy when the integration was validated. Eve is pre-1.0 and its public API is still evolving, so +upgrade this pin only with the contract and live proofs below. -```ts -import { defineMcpClientConnection } from "eve/connections"; - -export default defineMcpClientConnection({ - url: process.env.STAGEHAND_MCP_URL!, - description: - "Stagehand browser automation isolated behind an authenticated code-mode MCP gateway.", - auth: { - getToken: async () => ({ token: process.env.STAGEHAND_MCP_TOKEN! }), - }, - tools: { allow: ["code_execute"] }, -}); -``` - -Eve discovers the connection through `connection_search`. The only remote tool it can reveal is -`stagehand__code_execute`. - -## Start the sandbox - -The proposed image name is `ghcr.io/browserbase/stagehand-codemode`. The foundation workflow builds -the image and verifies stdio tool discovery locally; the registry reference becomes available only -after a tag or manual publish workflow runs. Keep the image configurable and, once published, pin -the deployment to an immutable digest: - -```text -STAGEHAND_CODEMODE_IMAGE=ghcr.io/browserbase/stagehand-codemode@sha256: -``` - -[`src/sandbox.ts`](./src/sandbox.ts) defines the provider contract and lifecycle used by the host. -The sandbox adapter must turn `stdioImage` into a command inside the guest. It can mirror the OCI -image into the sandbox root filesystem, or use a guest container runtime and return this command: - -```ts -{ - command: "docker", - args: [ - "run", "--rm", "-i", - "ghcr.io/browserbase/stagehand-codemode@sha256:", - ], -} -``` - -The image itself contains only the Stagehand stdio MCP. It does **not** contain the HTTP gateway. -The trusted host bootstrap writes [`src/sandbox-guest.mjs`](./src/sandbox-guest.mjs) into the -Firecracker or gVisor guest, installs `supergateway@3.4.3` there, and starts the authenticated -proxy. `supergateway` then owns the image command as its stdio child. Pass an adapter that can write -a file, spawn a process, publish a port, and destroy the sandbox; then give the result to Eve: - -```ts -import { createStagehandSandboxGateway } from "./src/sandbox.js"; - -const stagehand = await createStagehandSandboxGateway(sandboxProvider, { - image: process.env.STAGEHAND_CODEMODE_IMAGE, - environment: { STAGEHAND_BROWSER: "browserbase" }, -}); - -try { - // Start the Eve host with STAGEHAND_MCP_URL=stagehand.url and - // STAGEHAND_MCP_TOKEN=stagehand.token. -} finally { - await stagehand.close(); -} -``` - -For Vercel Sandbox specifically, create a Node 24 Firecracker guest with port `3000`, map -`writeTextFile` to `sandbox.writeFiles`, map `spawn` to a detached `sandbox.runCommand`, map -`publicUrl` to `sandbox.domain`, and map `close` to `sandbox.stop`. Vercel's command handle exposes -stdout and stderr but not a writable stdin stream after launch, so the Eve host cannot drive the -stdio MCP directly across the SDK boundary. That is why both `supergateway` and the authentication -proxy run inside the guest. The adapter must also materialize the pinned Stagehand image contents -inside the guest—such as through a mirrored runtime image or an exact source build—and return the -resulting local Node command as `stdioCommand`; do not assume nested Docker is available. - -`createStagehandSandboxGateway` polls the authenticated `/healthz` endpoint before returning. The -default startup deadline is two minutes, is configurable with `startupTimeoutMs`, and is capped -below the complete sandbox lifetime. This prevents Eve from racing a cold guest bootstrap or -public port publication without permitting an unbounded wait. - -The host starts one authenticated proxy and one stateful `supergateway` process per sandbox. -`supergateway` starts one Stagehand stdio child for each MCP session. `supergateway` is a -non-first-party adapter. The bootstrap pins it to `3.4.3`, sets `--logLevel none`, protects the -public port with a bearer token, and closes the complete sandbox after the Eve run. Do not expose -`supergateway` directly: it does not add inbound authentication. - -For production, bake the audited gateway and its exact dependency tree into the guest image. The -runtime `npm install` in `src/sandbox.ts` makes this provider-neutral example executable, but it -adds a network-time supply-chain dependency during sandbox startup. - -`src/sandbox.ts` accepts only the documented Stagehand and Browserbase environment names. Avoid -putting long-lived credentials there because generated JavaScript can read them. Prefer a sandbox -credential broker and an egress allowlist where the browser provider's HTTP and WebSocket -transports support them. The local test helper inherits the host environment for developer -convenience; it is explicitly not the production boundary. - -## Verification status - -We verified the two critical pieces independently: - -- A real Vercel Firecracker sandbox ran the Stagehand stdio MCP behind the authenticated gateway. - An unauthenticated request returned `401`; an authenticated MCP client initialized, listed only - `code_execute`, and the complete sandbox was destroyed afterward. -- Eve `0.29.4` passed both the deterministic and real-model flows below through the same gateway - implementation on localhost. - -The full composition is still partial. A fresh host Eve `0.29.4` run against the same public -sandbox first made an SSE request without a session (`400`), then initialized a stateful MCP -session (`200`), sent the initialized notification (`202`), and opened its authenticated session -SSE stream (`200`). It did not advance to `tools/list` or `tools/call` before the bounded 150-second -proof timed out. This exact-version run therefore does not establish an Eve-to-sandbox -`code_execute` loop. Treat the architecture as supported but keep the composed deployment behind -an integration test until that Streamable HTTP client interoperability gap is closed. - -## Run the deterministic smoke +## Secret-free framework contract From the repository root: ```bash pnpm install -pnpm exec turbo run build --filter @browserbasehq/stagehand-integrations -STAGEHAND_BROWSER=local \ - pnpm --filter @browserbasehq/stagehand-integrations-example-eve smoke +pnpm --filter @browserbasehq/stagehand-integrations-example-eve typecheck +pnpm --filter @browserbasehq/stagehand-integrations-example-eve contract ``` -The smoke deliberately runs the gateway on localhost so public CI needs no sandbox credentials. It -is not a sandbox-isolation proof. It does exercise the actual Eve runtime, rejects an unauthorized -request, and then runs authenticated Streamable HTTP `connection_search` followed by one -`stagehand__code_execute` call against a real local browser. +The contract starts an authenticated MCP server built on the official MCP SDK, deliberately returns +405 for the optional GET stream, and runs the real Eve build, server, and eval runtime with a +deterministic model. The eval must: + +1. use `connection_search` exactly once; +2. discover only `stagehand__code_execute`; +3. call it exactly twice through one connection; +4. receive persistent-page-shaped results and the exact expected markers; and +5. reject an unauthenticated request while tolerating authenticated GET 405. -## Run a real Eve agent +This proves Eve's connection behavior without claiming browser or sandbox isolation. -The real example uses Eve `0.29.4` and a direct Groq model. Set `GROQ_API_KEY`, then run: +## Live package-backed proof + +Build and pack the exact Stagehand packages under review, then run the live composition: ```bash -GROQ_API_KEY= \ -STAGEHAND_BROWSER=local \ - pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e +pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode +pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts + +STAGEHAND_SANDBOX_ARTIFACTS="$PWD/packages/integrations/examples/vercel-sandbox/.artifacts" \ +BROWSERBASE_API_KEY= \ +BROWSERBASE_PROJECT_ID= \ +VERCEL_OIDC_TOKEN= \ +OPENAI_API_KEY= \ +pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e ``` -To use Browserbase, set `STAGEHAND_BROWSER=browserbase`, `BROWSERBASE_API_KEY`, and optionally -`BROWSERBASE_PROJECT_ID` in the sandbox process. The production architecture remains the sandboxed -one above; the local commands exist only to make framework behavior reproducible in CI. +For external CI, replace `VERCEL_OIDC_TOKEN` with `VERCEL_TEAM_ID`, `VERCEL_PROJECT_ID`, and +`VERCEL_TOKEN`. `EVE_STAGEHAND_MODEL` selects the direct OpenAI model and defaults to +`gpt-5-mini`. + +The live proof checks unauthenticated 401 and authenticated optional-GET 405, then runs both the +deterministic and real-model Eve evals against one package-installed Vercel Sandbox connection. +Each eval must discover the connection, make two `code_execute` calls, retain a DOM marker between +calls, and observe neither `OPENAI_API_KEY` nor a host-only marker inside generated code. `PASS` is +emitted only after the Eve runtimes stop and the sandbox is destroyed. diff --git a/packages/integrations/examples/eve/agent/agent.ts b/packages/integrations/examples/eve/agent/agent.ts index dc58caf8ca..1b74f21817 100644 --- a/packages/integrations/examples/eve/agent/agent.ts +++ b/packages/integrations/examples/eve/agent/agent.ts @@ -1,4 +1,4 @@ -import { groq } from "@ai-sdk/groq"; +import { openai } from "@ai-sdk/openai"; import { defineAgent } from "eve"; import { mockModel } from "eve/evals"; @@ -33,14 +33,17 @@ export default defineAgent({ waitUntil: "domcontentloaded", }); await page.evaluate(() => { - document.documentElement.dataset.eveStagehandSmoke = "persistent"; + document.documentElement.dataset.eveStagehandDirectMarker = + "eve-direct-persistent"; }); return { pageId: page.pageId, title: await page.title(), - marker: await page.evaluate( - () => document.documentElement.dataset.eveStagehandSmoke, + directMarker: await page.evaluate( + () => document.documentElement.dataset.eveStagehandDirectMarker, ), + modelKeyVisible: process.env.OPENAI_API_KEY ?? null, + hostMarkerVisible: process.env.EVE_HOST_ONLY_MARKER ?? null, }; `, }, @@ -49,7 +52,41 @@ export default defineAgent({ }; } - return `STAGEHAND_RESULT ${JSON.stringify(stagehandResults.at(-1)?.output)}`; + if (stagehandResults.length === 1) { + return { + toolCalls: [ + { + name: stagehandTool, + input: { + code: ` + const directMarker = await page.evaluate( + () => document.documentElement.dataset.eveStagehandDirectMarker, + ); + if (directMarker !== "eve-direct-persistent") { + throw new Error("Eve lost Stagehand browser state between tool calls"); + } + await page.evaluate(() => { + document.documentElement.dataset.eveStagehandModelMarker = + "eve-model-persistent"; + }); + return { + pageId: page.pageId, + title: await page.title(), + directMarker, + modelMarker: await page.evaluate( + () => document.documentElement.dataset.eveStagehandModelMarker, + ), + modelKeyVisible: process.env.OPENAI_API_KEY ?? null, + hostMarkerVisible: process.env.EVE_HOST_ONLY_MARKER ?? null, + }; + `, + }, + }, + ], + }; + } + + return `STAGEHAND_RESULTS ${JSON.stringify(stagehandResults.map(({ output }) => output))}`; }) - : groq(process.env.EVE_STAGEHAND_MODEL ?? "openai/gpt-oss-120b"), + : openai(process.env.EVE_STAGEHAND_MODEL ?? "gpt-5-mini"), }); diff --git a/packages/integrations/examples/eve/agent/instructions.md b/packages/integrations/examples/eve/agent/instructions.md index 3a8afb4d38..f25cb06bcb 100644 --- a/packages/integrations/examples/eve/agent/instructions.md +++ b/packages/integrations/examples/eve/agent/instructions.md @@ -1,3 +1,3 @@ You are a browser agent. Use `connection_search` to discover the Stagehand connection, then use its -`code_execute` tool for browser work. Keep browser state in the existing tool session and return -only the evidence the user requested. +`code_execute` tool for browser work. Follow the requested number and order of calls exactly. Keep +browser state in the existing tool session and return only the evidence the user requested. diff --git a/packages/integrations/examples/eve/evals/stagehand.eval.ts b/packages/integrations/examples/eve/evals/stagehand.eval.ts index c4b5588e76..38c9cff98c 100644 --- a/packages/integrations/examples/eve/evals/stagehand.eval.ts +++ b/packages/integrations/examples/eve/evals/stagehand.eval.ts @@ -6,22 +6,36 @@ export default defineEval({ async test(t) { await t.send( [ - "Use the Stagehand connection and call code_execute exactly once.", - 'Open example.com and store the exact marker string "persistent" on the page.', - "Report the title and marker.", + "Use connection_search to discover the Stagehand connection.", + "Then call stagehand__code_execute exactly twice, in order.", + 'First open https://example.com and store "eve-direct-persistent" in', + "document.documentElement.dataset.eveStagehandDirectMarker.", + "Return the pageId, title, marker, process.env.OPENAI_API_KEY, and", + "process.env.EVE_HOST_ONLY_MARKER from that first call.", + "In the second call, read and verify that same direct marker without navigating.", + 'Then store "eve-model-persistent" in', + "document.documentElement.dataset.eveStagehandModelMarker and return both markers,", + "the current pageId and title, and the same two environment lookups.", + "Report the title and both exact markers after the two calls.", ].join(" "), ); t.succeeded(); t.calledTool("connection_search", { count: 1 }); t.calledTool("stagehand__code_execute", { - count: 1, + count: 2, output: (value) => { const output = JSON.stringify(value); - return output.includes("Example Domain") && output.includes("persistent"); + return ( + output.includes("Example Domain") && + output.includes("eve-direct-persistent") && + output.includes('"modelKeyVisible":null') && + output.includes('"hostMarkerVisible":null') + ); }, }); t.check(t.reply, includes("Example Domain")); - t.check(t.reply, includes("persistent")); + t.check(t.reply, includes("eve-direct-persistent")); + t.check(t.reply, includes("eve-model-persistent")); }, }); diff --git a/packages/integrations/examples/eve/package.json b/packages/integrations/examples/eve/package.json index 0f64c0deff..845a0ed60a 100644 --- a/packages/integrations/examples/eve/package.json +++ b/packages/integrations/examples/eve/package.json @@ -5,16 +5,16 @@ "type": "module", "scripts": { "build": "eve build", + "contract": "tsx src/contract.ts", "dev": "eve dev", "e2e": "tsx src/e2e.ts", - "smoke": "tsx src/smoke.ts", "typecheck": "eve build && tsc --noEmit" }, "dependencies": { - "@ai-sdk/groq": "catalog:", - "@browserbasehq/stagehand-integrations": "workspace:*", + "@ai-sdk/openai": "catalog:", + "@browserbasehq/stagehand-integrations-example-vercel-sandbox": "workspace:*", + "@modelcontextprotocol/sdk": "catalog:", "eve": "catalog:", - "supergateway": "catalog:", "zod": "catalog:" }, "devDependencies": { diff --git a/packages/integrations/examples/eve/src/contract-server.ts b/packages/integrations/examples/eve/src/contract-server.ts new file mode 100644 index 0000000000..e4bb6240a0 --- /dev/null +++ b/packages/integrations/examples/eve/src/contract-server.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import { randomBytes, timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import type { Server } from "node:http"; + +import { createMcpExpressApp } from "@modelcontextprotocol/sdk/server/express.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js"; +import { z } from "zod"; + +export type EveContractServer = { + url: string; + token: string; + codeExecuteCalls: () => number; + close: () => Promise; +}; + +export async function startEveContractServer(): Promise { + const token = randomBytes(32).toString("hex"); + const expectedAuthorization = Buffer.from(`Bearer ${token}`); + let codeExecuteCalls = 0; + const app = createMcpExpressApp(); + + app.use("/mcp", (request, response, next) => { + const authorization = request.headers.authorization; + const provided = authorization === undefined ? undefined : Buffer.from(authorization); + if ( + provided === undefined || + provided.length !== expectedAuthorization.length || + !timingSafeEqual(provided, expectedAuthorization) + ) { + response.status(401).type("text/plain").send("Unauthorized\n"); + return; + } + next(); + }); + + app.post("/mcp", async (request, response) => { + const server = new McpServer({ name: "eve-stagehand-contract", version: "1.0.0" }); + server.registerTool( + "code_execute", + { + description: + "# Stagehand V4 code-mode syntax\nContract-only Stagehand browser execution tool.", + inputSchema: { code: z.string() }, + }, + async () => { + codeExecuteCalls += 1; + const value = { + pageId: "contract-page", + title: "Example Domain", + directMarker: "eve-direct-persistent", + ...(codeExecuteCalls > 1 ? { modelMarker: "eve-model-persistent" } : {}), + modelKeyVisible: null, + hostMarkerVisible: null, + }; + const result = { ok: true, value }; + return { + content: [{ type: "text" as const, text: JSON.stringify(result) }], + structuredContent: result, + }; + }, + ); + + const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined }); + try { + await server.connect(transport); + await transport.handleRequest(request, response, request.body); + response.on("close", () => { + void transport.close(); + void server.close(); + }); + } catch { + if (!response.headersSent) { + response.status(500).json({ + jsonrpc: "2.0", + error: { code: -32603, message: "Contract MCP request failed" }, + id: null, + }); + } + } + }); + app.get("/mcp", (_request, response) => { + response.status(405).set("Allow", "POST").send("Method Not Allowed"); + }); + app.delete("/mcp", (_request, response) => { + response.status(405).set("Allow", "POST").send("Method Not Allowed"); + }); + + const httpServer = app.listen(0, "127.0.0.1"); + await once(httpServer, "listening"); + const address = httpServer.address(); + assert.ok(address && typeof address === "object"); + + return { + url: `http://127.0.0.1:${address.port}/mcp`, + token, + codeExecuteCalls: () => codeExecuteCalls, + close: () => closeServer(httpServer), + }; +} + +function closeServer(server: Server): Promise { + return new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); +} diff --git a/packages/integrations/examples/eve/src/contract.ts b/packages/integrations/examples/eve/src/contract.ts new file mode 100644 index 0000000000..38bf8d5409 --- /dev/null +++ b/packages/integrations/examples/eve/src/contract.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; + +import { startEveContractServer } from "./contract-server.js"; +import { assertEveEndpointContract, runEveStagehandEval } from "./run-eval.js"; + +const server = await startEveContractServer(); +let primaryError: unknown; +try { + await assertEveEndpointContract(server); + await runEveStagehandEval(server, true); + assert.equal(server.codeExecuteCalls(), 2, "Eve must call code_execute twice in one session"); +} catch (error) { + primaryError = error; +} + +let cleanupError: unknown; +await server.close().catch((error: unknown) => { + cleanupError = error; +}); + +if (primaryError !== undefined && cleanupError !== undefined) { + throw new AggregateError([primaryError, cleanupError], "Eve contract and cleanup both failed"); +} +if (primaryError !== undefined) throw primaryError; +if (cleanupError !== undefined) throw cleanupError; + +process.stdout.write( + `${JSON.stringify({ + status: "PASS", + framework: "eve", + proof: "deterministic authenticated Streamable HTTP contract", + connectionSearchCalls: 1, + codeExecuteCalls: 2, + unauthorizedStatus: 401, + optionalGetStatus: 405, + })}\n`, +); diff --git a/packages/integrations/examples/eve/src/e2e.ts b/packages/integrations/examples/eve/src/e2e.ts index 7870be9fe8..212179dd5c 100644 --- a/packages/integrations/examples/eve/src/e2e.ts +++ b/packages/integrations/examples/eve/src/e2e.ts @@ -1,12 +1,71 @@ -import { runEveStagehandEval } from "./run-eval.js"; +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; -process.env.STAGEHAND_BROWSER ??= "local"; +import { createStagehandSandbox } from "@browserbasehq/stagehand-integrations-example-vercel-sandbox"; + +import { assertEveEndpointContract, runEveStagehandEval } from "./run-eval.js"; + +const NO_ERROR = Symbol("no error"); +process.env.EVE_HOST_ONLY_MARKER = `host-${randomUUID()}`; + +const connection = await createStagehandSandbox({ + packageArtifactsPath: requiredEnvironment("STAGEHAND_SANDBOX_ARTIFACTS"), + browserbaseApiKey: requiredEnvironment("BROWSERBASE_API_KEY"), + browserbaseProjectId: requiredEnvironment("BROWSERBASE_PROJECT_ID"), + vercelCredentials: vercelCredentialsFromEnvironment(), +}); + +let primaryError: unknown = NO_ERROR; +try { + await assertEveEndpointContract(connection); + await runEveStagehandEval(connection, true); + await runEveStagehandEval(connection, false); +} catch (error) { + primaryError = error; +} + +let cleanupError: unknown; +await connection.close().catch((error: unknown) => { + cleanupError = error; +}); + +if (primaryError !== NO_ERROR && cleanupError !== undefined) { + throw new AggregateError([primaryError, cleanupError], "Eve E2E and sandbox cleanup both failed"); +} +if (primaryError !== NO_ERROR) throw primaryError; +if (cleanupError !== undefined) throw cleanupError; -await runEveStagehandEval(false); process.stdout.write( `${JSON.stringify({ status: "PASS", framework: "eve", - proof: "local authenticated gateway and real Groq-backed Eve agent", + eveVersion: "0.29.4", + deterministicEval: true, + realModelEval: true, + connectionSearchCallsPerEval: 1, + codeExecuteCallsPerEval: 2, + sessionPersisted: true, + modelCredentialIsolated: true, + unauthorizedStatus: 401, + optionalGetStatus: 405, + cleanup: ["eve-runtime", "vercel-sandbox"], })}\n`, ); + +function requiredEnvironment(name: string): string { + const value = process.env[name]; + assert.ok(value, `Missing ${name}`); + return value; +} + +function vercelCredentialsFromEnvironment(): + | { teamId: string; projectId: string; token: string } + | undefined { + const token = process.env.VERCEL_TOKEN; + if (!token) return undefined; + return { + teamId: requiredEnvironment("VERCEL_TEAM_ID"), + projectId: requiredEnvironment("VERCEL_PROJECT_ID"), + token, + }; +} diff --git a/packages/integrations/examples/eve/src/gateway.ts b/packages/integrations/examples/eve/src/gateway.ts deleted file mode 100644 index f48b8811fc..0000000000 --- a/packages/integrations/examples/eve/src/gateway.ts +++ /dev/null @@ -1,200 +0,0 @@ -import assert from "node:assert/strict"; -import { randomBytes, timingSafeEqual } from "node:crypto"; -import { once } from "node:events"; -import { fileURLToPath } from "node:url"; -import http from "node:http"; -import { spawn, type ChildProcess } from "node:child_process"; - -const DEFAULT_STDIO_SERVER_PATH = fileURLToPath( - new URL("../../../dist/codemode/stdio-server.mjs", import.meta.url), -); -const SUPERGATEWAY_PATH = fileURLToPath( - new URL("../node_modules/.bin/supergateway", import.meta.url), -); -const MCP_PROTOCOL_VERSION = "2025-11-25"; - -export type StagehandGateway = { - url: string; - token: string; - close: () => Promise; -}; - -export async function startLocalTestGateway(options?: { - stdioServerPath?: string; -}): Promise { - const token = randomBytes(32).toString("hex"); - const upstreamPort = await reservePort(); - const stdioServerPath = options?.stdioServerPath ?? DEFAULT_STDIO_SERVER_PATH; - const stdioCommand = `${shellQuote(process.execPath)} ${shellQuote(stdioServerPath)}`; - const gateway = spawn( - SUPERGATEWAY_PATH, - [ - "--stdio", - stdioCommand, - "--outputTransport", - "streamableHttp", - "--stateful", - "--sessionTimeout", - "600000", - "--protocolVersion", - MCP_PROTOCOL_VERSION, - "--port", - String(upstreamPort), - "--healthEndpoint", - "/healthz", - "--logLevel", - "none", - ], - { - detached: process.platform !== "win32", - // This helper is local-test-only. Production uses the allowlisted guest - // environment in sandbox.ts and destroys the complete sandbox afterward. - env: localTestEnvironment(process.env), - stdio: ["ignore", "ignore", "pipe"], - }, - ); - - let stderr = ""; - gateway.stderr?.setEncoding("utf8"); - gateway.stderr?.on("data", (chunk: string) => { - stderr = `${stderr}${chunk}`.slice(-4_000); - }); - - try { - await waitForHealthyGateway(upstreamPort, gateway, () => stderr); - const proxy = await startAuthenticatedProxy({ token, upstreamPort }); - return { - url: `http://127.0.0.1:${proxy.port}/mcp`, - token, - async close() { - await proxy.close(); - await stopProcessTree(gateway); - }, - }; - } catch (error) { - await stopProcessTree(gateway); - throw error; - } -} - -async function startAuthenticatedProxy(options: { - token: string; - upstreamPort: number; -}): Promise<{ port: number; close: () => Promise }> { - const expectedAuthorization = Buffer.from(`Bearer ${options.token}`); - const server = http.createServer((request, response) => { - if (!isAuthorized(request.headers.authorization, expectedAuthorization)) { - response.writeHead(401, { "content-type": "text/plain" }); - response.end("Unauthorized\n"); - return; - } - - const upstream = http.request( - { - host: "127.0.0.1", - port: options.upstreamPort, - method: request.method, - path: request.url, - headers: forwardedMcpHeaders(request.headers), - }, - (upstreamResponse) => { - response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); - upstreamResponse.pipe(response); - }, - ); - upstream.on("error", () => { - if (!response.headersSent) response.writeHead(502); - response.end("Upstream unavailable\n"); - }); - request.pipe(upstream); - }); - - server.listen(0, "127.0.0.1"); - await once(server, "listening"); - const address = server.address(); - assert.ok(address && typeof address === "object"); - - return { - port: address.port, - close: () => - new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ), - }; -} - -function isAuthorized(header: string | undefined, expected: Buffer): boolean { - if (!header) return false; - const provided = Buffer.from(header); - return provided.length === expected.length && timingSafeEqual(provided, expected); -} - -function forwardedMcpHeaders(headers: http.IncomingHttpHeaders): http.OutgoingHttpHeaders { - return Object.fromEntries( - ["accept", "content-length", "content-type", "mcp-protocol-version", "mcp-session-id"] - .map((name) => [name, headers[name]] as const) - .filter((entry): entry is [string, string | string[]] => entry[1] !== undefined), - ); -} - -async function reservePort(): Promise { - const server = http.createServer(); - server.listen(0, "127.0.0.1"); - await once(server, "listening"); - const address = server.address(); - assert.ok(address && typeof address === "object"); - const port = address.port; - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ); - return port; -} - -async function waitForHealthyGateway( - port: number, - process: ChildProcess, - readStderr: () => string, -): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (process.exitCode !== null) { - throw new Error(`supergateway exited before startup: ${readStderr()}`); - } - const response = await fetch(`http://127.0.0.1:${port}/healthz`).catch(() => undefined); - if (response?.ok) return; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - throw new Error(`supergateway did not become healthy: ${readStderr()}`); -} - -async function stopProcessTree(child: ChildProcess): Promise { - if (child.exitCode !== null || child.pid === undefined) return; - signalProcessTree(child, "SIGTERM"); - const stopped = await Promise.race([ - once(child, "exit").then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), - ]); - if (stopped) return; - signalProcessTree(child, "SIGKILL"); - await once(child, "exit").catch(() => undefined); -} - -function signalProcessTree(child: ChildProcess, signal: NodeJS.Signals): void { - try { - if (process.platform === "win32" || child.pid === undefined) child.kill(signal); - else process.kill(-child.pid, signal); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } -} - -function localTestEnvironment(environment: NodeJS.ProcessEnv): Record { - return Object.fromEntries( - Object.entries(environment).filter( - (entry): entry is [string, string] => entry[1] !== undefined, - ), - ); -} - -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'\\''`)}'`; -} diff --git a/packages/integrations/examples/eve/src/run-eval.ts b/packages/integrations/examples/eve/src/run-eval.ts index ce589bf291..bf331f2aad 100644 --- a/packages/integrations/examples/eve/src/run-eval.ts +++ b/packages/integrations/examples/eve/src/run-eval.ts @@ -7,49 +7,64 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { startLocalTestGateway } from "./gateway.js"; - const EVE_BINARY = fileURLToPath(new URL("../node_modules/.bin/eve", import.meta.url)); const EXAMPLE_ROOT = fileURLToPath(new URL("../", import.meta.url)); const BUILT_SERVER = fileURLToPath(new URL("../.output/server/index.mjs", import.meta.url)); -export async function runEveStagehandEval(deterministic: boolean): Promise { - const gateway = await startLocalTestGateway(); - try { - const unauthorized = await fetch(gateway.url); - assert.equal(unauthorized.status, 401, "the gateway must reject requests without its token"); +export type EveStagehandConnection = { + url: URL | string; + token: string; +}; - const environment = { - ...process.env, - ...(deterministic ? { EVE_STAGEHAND_DETERMINISTIC: "1" } : {}), - STAGEHAND_MCP_URL: gateway.url, - STAGEHAND_MCP_TOKEN: gateway.token, - }; - await runChild(EVE_BINARY, ["build"], environment); +export async function assertEveEndpointContract(connection: EveStagehandConnection): Promise { + const unauthorized = await fetch(connection.url); + assert.equal(unauthorized.status, 401, "the Stagehand endpoint must require its bearer token"); - const port = await reservePort(); - const serverRoot = await mkdtemp(join(tmpdir(), "eve-stagehand-")); - const server = spawn(process.execPath, [BUILT_SERVER], { - cwd: serverRoot, - env: { ...environment, HOST: "127.0.0.1", PORT: String(port) }, - stdio: "inherit", - }); - try { - await waitForAgent(port, server); - await runChild( - EVE_BINARY, - ["eval", "stagehand", "--skip-report", "--url", `http://127.0.0.1:${port}`], - environment, - ); - } finally { - await stopChild(server); - await rm(serverRoot, { recursive: true, force: true }); - } + const optionalGet = await fetch(connection.url, { + headers: { authorization: `Bearer ${connection.token}` }, + }); + assert.equal(optionalGet.status, 405, "Eve must tolerate the endpoint's optional GET rejection"); +} + +export async function runEveStagehandEval( + connection: EveStagehandConnection, + deterministic: boolean, +): Promise { + const environment = eveEnvironment(connection, deterministic); + await runChild(EVE_BINARY, ["build"], environment); + + const port = await reservePort(); + const serverRoot = await mkdtemp(join(tmpdir(), "eve-stagehand-")); + const server = spawn(process.execPath, [BUILT_SERVER], { + cwd: serverRoot, + env: { ...environment, HOST: "127.0.0.1", PORT: String(port) }, + stdio: "inherit", + }); + try { + await waitForAgent(port, server); + await runChild( + EVE_BINARY, + ["eval", "stagehand", "--skip-report", "--url", `http://127.0.0.1:${port}`], + environment, + ); } finally { - await gateway.close(); + await stopChild(server); + await rm(serverRoot, { recursive: true, force: true }); } } +function eveEnvironment( + connection: EveStagehandConnection, + deterministic: boolean, +): NodeJS.ProcessEnv { + const environment = { ...process.env }; + if (deterministic) environment.EVE_STAGEHAND_DETERMINISTIC = "1"; + else delete environment.EVE_STAGEHAND_DETERMINISTIC; + environment.STAGEHAND_MCP_URL = connection.url.toString(); + environment.STAGEHAND_MCP_TOKEN = connection.token; + return environment; +} + async function runChild( command: string, args: string[], @@ -60,8 +75,12 @@ async function runChild( env: environment, stdio: "inherit", }); - const [exitCode] = (await once(child, "exit")) as [number | null, NodeJS.Signals | null]; - if (exitCode !== 0) throw new Error(`${command} exited with code ${String(exitCode)}`); + const [exitCode, signal] = (await once(child, "exit")) as [number | null, NodeJS.Signals | null]; + if (exitCode !== 0) { + throw new Error( + `${command} exited with ${signal === null ? `code ${String(exitCode)}` : signal}`, + ); + } } async function reservePort(): Promise { @@ -77,8 +96,10 @@ async function reservePort(): Promise { } async function waitForAgent(port: number, child: ChildProcess): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (child.exitCode !== null) throw new Error("the built Eve server exited before startup"); + for (let attempt = 0; attempt < 200; attempt += 1) { + if (child.exitCode !== null || child.signalCode !== null) { + throw new Error("the built Eve server exited before startup"); + } const response = await fetch(`http://127.0.0.1:${port}/eve/v1/health`).catch(() => undefined); if (response?.ok) return; await new Promise((resolve) => setTimeout(resolve, 50)); @@ -87,11 +108,15 @@ async function waitForAgent(port: number, child: ChildProcess): Promise { } async function stopChild(child: ChildProcess): Promise { - if (child.exitCode !== null) return; + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit").then(() => true); child.kill("SIGTERM"); const stopped = await Promise.race([ - once(child, "exit").then(() => true), + exited, new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), ]); - if (!stopped) child.kill("SIGKILL"); + if (!stopped) { + child.kill("SIGKILL"); + await exited; + } } diff --git a/packages/integrations/examples/eve/src/sandbox-guest.mjs b/packages/integrations/examples/eve/src/sandbox-guest.mjs deleted file mode 100644 index a70cf1b12c..0000000000 --- a/packages/integrations/examples/eve/src/sandbox-guest.mjs +++ /dev/null @@ -1,128 +0,0 @@ -import assert from "node:assert/strict"; -import { timingSafeEqual } from "node:crypto"; -import { once } from "node:events"; -import http from "node:http"; -import { spawn } from "node:child_process"; - -const port = Number.parseInt(requiredEnvironment("STAGEHAND_GATEWAY_PORT"), 10); -const token = requiredEnvironment("STAGEHAND_GATEWAY_TOKEN"); -const stdio = JSON.parse(requiredEnvironment("STAGEHAND_STDIO_COMMAND_JSON")); -assert.equal(typeof stdio.command, "string"); -assert.ok(Array.isArray(stdio.args) && stdio.args.every((value) => typeof value === "string")); - -const supergateway = spawn( - "/tmp/stagehand-eve-gateway/node_modules/.bin/supergateway", - [ - "--stdio", - [stdio.command, ...stdio.args].map(shellQuote).join(" "), - "--outputTransport", - "streamableHttp", - "--stateful", - "--sessionTimeout", - "600000", - "--protocolVersion", - "2025-11-25", - "--port", - String(port + 1), - "--healthEndpoint", - "/healthz", - "--logLevel", - "none", - ], - { detached: true, env: process.env, stdio: ["ignore", "ignore", "pipe"] }, -); - -let stderr = ""; -supergateway.stderr.setEncoding("utf8"); -supergateway.stderr.on("data", (chunk) => { - stderr = `${stderr}${chunk}`.slice(-4_000); -}); - -await waitForHealthyGateway(port + 1); - -const expectedAuthorization = Buffer.from(`Bearer ${token}`); -const server = http.createServer((request, response) => { - if (!isAuthorized(request.headers.authorization)) { - response.writeHead(401, { "content-type": "text/plain" }); - response.end("Unauthorized\n"); - return; - } - - const upstream = http.request( - { - host: "127.0.0.1", - port: port + 1, - method: request.method, - path: request.url, - headers: forwardedMcpHeaders(request.headers), - }, - (upstreamResponse) => { - response.writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); - upstreamResponse.pipe(response); - }, - ); - upstream.on("error", () => { - if (!response.headersSent) response.writeHead(502); - response.end("Upstream unavailable\n"); - }); - request.pipe(upstream); -}); - -server.listen(port, "0.0.0.0"); -await once(server, "listening"); - -for (const signal of ["SIGTERM", "SIGINT"]) { - process.once(signal, async () => { - await new Promise((resolve) => server.close(resolve)); - stopProcessTree("SIGTERM"); - process.exit(0); - }); -} - -function requiredEnvironment(name) { - const value = process.env[name]; - if (!value) throw new Error(`${name} is required`); - return value; -} - -function isAuthorized(header) { - if (!header) return false; - const provided = Buffer.from(header); - return ( - provided.length === expectedAuthorization.length && - timingSafeEqual(provided, expectedAuthorization) - ); -} - -function forwardedMcpHeaders(headers) { - return Object.fromEntries( - ["accept", "content-length", "content-type", "mcp-protocol-version", "mcp-session-id"] - .map((name) => [name, headers[name]]) - .filter((entry) => entry[1] !== undefined), - ); -} - -async function waitForHealthyGateway(upstreamPort) { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (supergateway.exitCode !== null) { - throw new Error(`supergateway exited before startup: ${stderr}`); - } - const response = await fetch(`http://127.0.0.1:${upstreamPort}/healthz`).catch(() => undefined); - if (response?.ok) return; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - throw new Error(`supergateway did not become healthy: ${stderr}`); -} - -function stopProcessTree(signal) { - if (supergateway.pid === undefined) return; - try { - process.kill(-supergateway.pid, signal); - } catch (error) { - if (error.code !== "ESRCH") throw error; - } -} - -function shellQuote(value) { - return `'${value.replaceAll("'", `'\\''`)}'`; -} diff --git a/packages/integrations/examples/eve/src/sandbox.ts b/packages/integrations/examples/eve/src/sandbox.ts deleted file mode 100644 index 64315c8d40..0000000000 --- a/packages/integrations/examples/eve/src/sandbox.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { randomBytes } from "node:crypto"; -import { readFile } from "node:fs/promises"; - -export const PROPOSED_STAGEHAND_CODEMODE_IMAGE = "ghcr.io/browserbase/stagehand-codemode"; - -const GUEST_GATEWAY_PATH = "/tmp/stagehand-eve-gateway/gateway.mjs"; -const GUEST_GATEWAY_SOURCE = new URL("./sandbox-guest.mjs", import.meta.url); - -type GuestEnvironment = Partial< - Record< - | "BROWSERBASE_API_KEY" - | "BROWSERBASE_PROJECT_ID" - | "STAGEHAND_BROWSER" - | "STAGEHAND_MODEL_API_KEY" - | "STAGEHAND_MODEL_NAME", - string - > ->; - -export type SandboxProcess = { - wait: () => Promise<{ exitCode: number }>; - kill: (signal: "SIGTERM" | "SIGKILL") => Promise; -}; - -export type SandboxInstance = { - /** - * Command for the Stagehand stdio server inside this sandbox. A provider can - * materialize the OCI image as the sandbox rootfs, or return a nested - * container-runtime command such as `docker run --rm -i @`. - */ - stdioCommand: { command: string; args: string[] }; - publicUrl: (port: number) => Promise; - writeTextFile: (path: string, contents: string) => Promise; - spawn: (options: { - command: string; - args: string[]; - env: Record; - }) => Promise; - close: () => Promise; -}; - -export type SandboxProvider = { - create: (options: { - stdioImage: string; - exposedPorts: number[]; - timeoutMs: number; - }) => Promise; -}; - -export async function createStagehandSandboxGateway( - provider: SandboxProvider, - options: { - image?: string; - startupTimeoutMs?: number; - timeoutMs?: number; - environment?: GuestEnvironment; - } = {}, -): Promise<{ url: string; token: string; close: () => Promise }> { - const port = 3000; - const token = randomBytes(32).toString("hex"); - const timeoutMs = options.timeoutMs ?? 15 * 60_000; - const startupTimeoutMs = Math.min(options.startupTimeoutMs ?? 2 * 60_000, timeoutMs - 1_000); - if (startupTimeoutMs <= 0) { - throw new Error("the sandbox timeout must leave at least one second for gateway startup"); - } - const sandbox = await provider.create({ - stdioImage: options.image ?? PROPOSED_STAGEHAND_CODEMODE_IMAGE, - exposedPorts: [port], - timeoutMs, - }); - - try { - await sandbox.writeTextFile(GUEST_GATEWAY_PATH, await readFile(GUEST_GATEWAY_SOURCE, "utf8")); - const process = await sandbox.spawn({ - command: "/bin/sh", - args: [ - "-lc", - [ - "set -eu", - "cd /tmp/stagehand-eve-gateway", - "npm init -y >/dev/null 2>&1", - "npm install --ignore-scripts --no-audit --no-fund supergateway@3.4.3 >/dev/null 2>&1", - `exec node ${GUEST_GATEWAY_PATH}`, - ].join("\n"), - ], - env: { - ...options.environment, - STAGEHAND_GATEWAY_PORT: String(port), - STAGEHAND_GATEWAY_TOKEN: token, - STAGEHAND_STDIO_COMMAND_JSON: JSON.stringify(sandbox.stdioCommand), - }, - }); - const baseUrl = (await sandbox.publicUrl(port)).replace(/\/$/, ""); - await waitForGateway(baseUrl, token, startupTimeoutMs); - const url = `${baseUrl}/mcp`; - - return { - url, - token, - async close() { - await process.kill("SIGTERM").catch(() => undefined); - const stopped = await Promise.race([ - process.wait().then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), - ]); - if (!stopped) await process.kill("SIGKILL").catch(() => undefined); - await sandbox.close(); - }, - }; - } catch (error) { - await sandbox.close().catch(() => undefined); - throw error; - } -} - -async function waitForGateway( - baseUrl: string, - token: string, - startupTimeoutMs: number, -): Promise { - const deadline = Date.now() + startupTimeoutMs; - while (Date.now() < deadline) { - const response = await fetch(`${baseUrl}/healthz`, { - headers: { authorization: `Bearer ${token}` }, - }).catch(() => undefined); - if (response?.ok) return; - await new Promise((resolve) => setTimeout(resolve, 250)); - } - throw new Error("the sandboxed Stagehand gateway did not become healthy"); -} diff --git a/packages/integrations/examples/eve/src/smoke.ts b/packages/integrations/examples/eve/src/smoke.ts deleted file mode 100644 index cf09b49020..0000000000 --- a/packages/integrations/examples/eve/src/smoke.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { runEveStagehandEval } from "./run-eval.js"; - -process.env.STAGEHAND_BROWSER ??= "local"; - -await runEveStagehandEval(true); -process.stdout.write( - `${JSON.stringify({ - status: "PASS", - framework: "eve", - proof: "local authenticated gateway and deterministic Eve agent", - tools: ["connection_search", "stagehand__code_execute"], - codeExecuteCalls: 1, - unauthorizedStatus: 401, - })}\n`, -); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f5d0aea73d..39af0cbd21 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -609,18 +609,18 @@ importers: packages/integrations/examples/eve: dependencies: - '@ai-sdk/groq': + '@ai-sdk/openai': specifier: 'catalog:' - version: 4.0.5(zod@4.4.3) - '@browserbasehq/stagehand-integrations': + version: 4.0.8(zod@4.4.3) + '@browserbasehq/stagehand-integrations-example-vercel-sandbox': specifier: workspace:* - version: link:../.. + version: link:../vercel-sandbox + '@modelcontextprotocol/sdk': + specifier: 'catalog:' + version: 1.29.0(zod@4.4.3) eve: specifier: 'catalog:' version: 0.29.4(@opentelemetry/api@1.9.1)(ai@7.0.16(zod@4.4.3))(aws4fetch@1.0.20)(dotenv@17.4.2)(jiti@1.21.7)(lru-cache@11.5.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0))(xml2js@0.6.2) - supergateway: - specifier: 'catalog:' - version: 3.4.3(bufferutil@4.1.0) zod: specifier: 'catalog:' version: 4.4.3 @@ -7502,6 +7502,10 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + undici@8.9.0: resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} engines: {node: '>=22.19.0'} @@ -7513,10 +7517,6 @@ packages: resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} engines: {node: '>=18'} - undici@7.29.0: - resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} - engines: {node: '>=20.18.1'} - unified@11.0.5: resolution: {integrity: sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==} @@ -15878,6 +15878,8 @@ snapshots: undici-types@7.24.6: {} + undici@7.29.0: {} + undici@8.9.0: {} unenv@2.0.0-rc.24: @@ -15886,8 +15888,6 @@ snapshots: unicorn-magic@0.3.0: {} - undici@7.29.0: {} - unified@11.0.5: dependencies: '@types/unist': 3.0.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5925fc6b88..13612cb592 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -9,7 +9,6 @@ catalog: "@mastra/mcp": ^1.15.0 "@modelcontextprotocol/sdk": 1.29.0 eve: 0.29.4 - supergateway: 3.4.3 "@vercel/sandbox": 2.9.2 "@ast-grep/lang-go": 0.0.6 "@ast-grep/lang-python": 0.0.6 From 8e2cbcbed32306543bc7a51cef748fa2daa1f865 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Sat, 8 Aug 2026 04:06:54 +0000 Subject: [PATCH 5/6] fix(eve): gate live proof on credentials --- .../workflows/codemode-framework-examples.yml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index f0ef0e0945..6038eba37c 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -291,7 +291,26 @@ jobs: - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve contract - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts - - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e + - name: Detect Eve live test credentials + id: eve-live-credentials + env: + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + VERCEL_OIDC_TOKEN: ${{ secrets.VERCEL_OIDC_TOKEN }} + VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }} + VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + if [[ -n "$BROWSERBASE_API_KEY" && -n "$BROWSERBASE_PROJECT_ID" && -n "$OPENAI_API_KEY" ]] && \ + [[ -n "$VERCEL_OIDC_TOKEN" || ( -n "$VERCEL_TEAM_ID" && -n "$VERCEL_PROJECT_ID" && -n "$VERCEL_TOKEN" ) ]]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + fi + - name: Run Eve live sandbox proof + if: steps.eve-live-credentials.outputs.available == 'true' + run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e env: STAGEHAND_SANDBOX_ARTIFACTS: ${{ github.workspace }}/packages/integrations/examples/vercel-sandbox/.artifacts BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} From 5ed6194e3b541af98d0f55eedeaf26b6eb03cd4a Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Sat, 8 Aug 2026 04:25:29 +0000 Subject: [PATCH 6/6] fix(eve): share live credential detection --- .github/workflows/codemode-framework-examples.yml | 11 +++-------- packages/integrations/examples/eve/README.md | 1 - 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codemode-framework-examples.yml b/.github/workflows/codemode-framework-examples.yml index 4a69f40b51..a8114c9840 100644 --- a/.github/workflows/codemode-framework-examples.yml +++ b/.github/workflows/codemode-framework-examples.yml @@ -274,10 +274,12 @@ jobs: - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve typecheck - run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve contract - - run: pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode - run: pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts - name: Detect Eve live test credentials id: eve-live-credentials + uses: ./.github/actions/detect-codemode-live-credentials + with: + require-openai: "true" env: BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} @@ -286,13 +288,6 @@ jobs: VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: | - if [[ -n "$BROWSERBASE_API_KEY" && -n "$BROWSERBASE_PROJECT_ID" && -n "$OPENAI_API_KEY" ]] && \ - [[ -n "$VERCEL_OIDC_TOKEN" || ( -n "$VERCEL_TEAM_ID" && -n "$VERCEL_PROJECT_ID" && -n "$VERCEL_TOKEN" ) ]]; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - fi - name: Run Eve live sandbox proof if: steps.eve-live-credentials.outputs.available == 'true' run: pnpm --filter @browserbasehq/stagehand-integrations-example-eve e2e diff --git a/packages/integrations/examples/eve/README.md b/packages/integrations/examples/eve/README.md index 8f6574daba..3ddb74f9b1 100644 --- a/packages/integrations/examples/eve/README.md +++ b/packages/integrations/examples/eve/README.md @@ -47,7 +47,6 @@ This proves Eve's connection behavior without claiming browser or sandbox isolat Build and pack the exact Stagehand packages under review, then run the live composition: ```bash -pnpm exec turbo run build --filter @browserbasehq/stagehand-codemode pnpm --filter @browserbasehq/stagehand-integrations-example-vercel-sandbox pack:artifacts STAGEHAND_SANDBOX_ARTIFACTS="$PWD/packages/integrations/examples/vercel-sandbox/.artifacts" \