From a601c5d2a88ce93063fe9e48db1c69d5393ca942 Mon Sep 17 00:00:00 2001 From: miguel Date: Sun, 16 Aug 2026 16:51:08 -0700 Subject: [PATCH 1/3] feat(integrations): extract Claude Agent SDK session layer from evals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New thin package @browserbasehq/stagehand-integrations-claude-agent-sdk: loadClaudeAgentSdk + runClaudeAgentSession — the query() streaming loop (abort control, message logging, token usage, status/stop-reason) with explicit options and no EVAL_* env reads. This is the code the evals claude_code runner and the claude-code facade example were both maintaining; both now call the package. Deliberately NOT extracted: the tool-surface mount machinery (AgentMount handling, the in-process run MCP tool, permission gate derivation) stays in evals — it abstracts over evals' surface registry and has no integrations consumer. --- .github/workflows/ci.yml | 1 + packages/evals/framework/claudeCodeRunner.ts | 436 ++---------------- packages/evals/package.json | 1 + .../tests/framework/claudeCodeRunner.test.ts | 102 ---- .../claude-agent-sdk/package.json | 36 ++ .../claude-agent-sdk/src/index.ts | 1 + .../claude-agent-sdk/src/session.ts | 348 ++++++++++++++ .../claude-agent-sdk/tests/session.test.ts | 109 +++++ .../claude-agent-sdk/tsconfig.json | 13 + .../claude-agent-sdk/tsdown.config.ts | 15 + .../claude-agent-sdk/vitest.config.ts | 9 + .../integrations/claude-code/package.json | 6 +- .../integrations/claude-code/src/agent.ts | 50 +- pnpm-lock.yaml | 30 +- turbo.json | 18 + vitest.config.ts | 1 + 16 files changed, 661 insertions(+), 515 deletions(-) create mode 100644 packages/integrations/claude-agent-sdk/package.json create mode 100644 packages/integrations/claude-agent-sdk/src/index.ts create mode 100644 packages/integrations/claude-agent-sdk/src/session.ts create mode 100644 packages/integrations/claude-agent-sdk/tests/session.test.ts create mode 100644 packages/integrations/claude-agent-sdk/tsconfig.json create mode 100644 packages/integrations/claude-agent-sdk/tsdown.config.ts create mode 100644 packages/integrations/claude-agent-sdk/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cbd2bebbd..4df628ac67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,7 @@ jobs: packages/extension/artifacts/** packages/sdk-ts/dist/** packages/integrations/core/dist/** + packages/integrations/claude-agent-sdk/dist/** packages/evals/dist/** retention-days: 1 diff --git a/packages/evals/framework/claudeCodeRunner.ts b/packages/evals/framework/claudeCodeRunner.ts index 8e01938544..e4a2379638 100644 --- a/packages/evals/framework/claudeCodeRunner.ts +++ b/packages/evals/framework/claudeCodeRunner.ts @@ -1,20 +1,25 @@ +import { + buildClaudeCodeTranscript, + extractClaudeCodeTokenUsage, + normalizeClaudeModel, + runClaudeAgentSession, + stringifyError, + type ClaudeAgentSdk, + type ClaudeSdkMessage, +} from "@browserbasehq/stagehand-integrations-claude-agent-sdk"; import type { AvailableModel } from "stagehand-v3"; -import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; -import type { TaskResult } from "./types.js"; import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; import { datasetPromptGuidance } from "./externalHarnessPlan.js"; import type { PreparedClaudeCodeToolAdapter } from "./claudeCodeToolAdapter.js"; import { claudeCodeAdapter } from "./harnesses/claudeCodeAdapter.js"; +import type { TaskResult } from "./types.js"; import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; -type ClaudeSdkMessage = Record; -type ClaudeQuery = AsyncIterable; -type MetricValue = { count: number; value: number }; +export type { ClaudeAgentSdk }; +export { isClaudeCodeMaxTurnsError } from "@browserbasehq/stagehand-integrations-claude-agent-sdk"; -export type ClaudeAgentSdk = { - query: (input: { prompt: string; options?: Record }) => ClaudeQuery; -}; +type MetricValue = { count: number; value: number }; export interface ClaudeCodeRunnerInput { plan: ExternalHarnessTaskPlan; @@ -23,15 +28,6 @@ export interface ClaudeCodeRunnerInput { toolAdapter?: PreparedClaudeCodeToolAdapter; signal?: AbortSignal; sdk?: ClaudeAgentSdk; - /** - * Optional verifier integration. When provided, the runner builds a - * Trajectory from the SDK message stream (via claudeCodeAdapter), runs - * V3Evaluator.verify() against the trajectory's embedded TaskSpec, and folds - * the EvaluationResult into the returned TaskResult ({_success} mode follows - * EVAL_SUCCESS_MODE). - * When omitted, the runner falls back to parsing the legacy EVAL_RESULT - * line — preserves current behavior for callers that haven't migrated. - */ verifier?: ExternalHarnessVerifierConfig; } @@ -42,10 +38,8 @@ export interface ParsedClaudeCodeResult { raw: string; } -const CLAUDE_AGENT_SDK_PACKAGE = "@anthropic-ai/claude-agent-sdk"; - export function normalizeClaudeCodeModel(model: AvailableModel): string { - return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; + return normalizeClaudeModel(model); } export function buildClaudeCodePrompt( @@ -79,52 +73,35 @@ export function parseClaudeCodeResult(raw: string): ParsedClaudeCodeResult { markerIndex >= 0 ? [resultText, resultText.split(/\r?\n/, 1)[0]?.trim(), extractFirstJsonObject(resultText)] : [resultText]; - for (const candidate of candidates) { if (!candidate) continue; const parsed = tryParseClaudeCodeJson(candidate); - if (parsed) { - return { - ...parsed, - raw, - }; - } + if (parsed) return { ...parsed, raw }; } - return { success: false, raw }; } function extractFirstJsonObject(value: string): string | undefined { const start = value.indexOf("{"); if (start < 0) return undefined; - let depth = 0; let inString = false; let escaped = false; - for (let index = start; index < value.length; index += 1) { const character = value[index]; if (inString) { - if (escaped) { - escaped = false; - } else if (character === "\\") { - escaped = true; - } else if (character === '"') { - inString = false; - } + if (escaped) escaped = false; + else if (character === "\\") escaped = true; + else if (character === '"') inString = false; continue; } - - if (character === '"') { - inString = true; - } else if (character === "{") { - depth += 1; - } else if (character === "}") { + if (character === '"') inString = true; + else if (character === "{") depth += 1; + else if (character === "}") { depth -= 1; if (depth === 0) return value.slice(start, index + 1); } } - return undefined; } @@ -147,115 +124,53 @@ function tryParseClaudeCodeJson( } } -export function isClaudeCodeMaxTurnsError(value: unknown): boolean { - const message = stringifyError(value); - return /(?:maximum number of turns|max(?:imum)? turns|turn limit)/i.test(message); -} - export async function runClaudeCodeAgent({ plan, model, logger, toolAdapter, signal, - sdk: injectedSdk, + sdk, verifier, }: ClaudeCodeRunnerInput): Promise { - const sdk = injectedSdk ?? (await loadClaudeAgentSdk()); - const abortController = new AbortController(); - if (signal) { - if (signal.aborted) abortController.abort(signal.reason); - signal.addEventListener("abort", () => abortController.abort(signal.reason), { once: true }); - } - - const messages: ClaudeSdkMessage[] = []; const prompt = buildClaudeCodePrompt(plan, toolAdapter?.promptInstructions); - const allowedTools = - toolAdapter?.allowedTools ?? - readCsvEnv("EVAL_CLAUDE_CODE_ALLOWED_TOOLS", ["WebFetch", "WebSearch"]); - const permissionMode = process.env.EVAL_CLAUDE_CODE_PERMISSION_MODE ?? "default"; - const maxTurns = readPositiveIntEnv("EVAL_CLAUDE_CODE_MAX_TURNS", 50); - const pathToClaudeCodeExecutable = process.env.EVAL_CLAUDE_CODE_EXECUTABLE || undefined; - - let resultText = ""; - let resultMessage: ClaudeSdkMessage | undefined; - let iterationError: unknown; - // tool_use id -> tool name, so tool_result blocks (which carry only the id) - // can be attributed for per-step observation triggers. - const toolUseNames = new Map(); - - try { - for await (const message of sdk.query({ - prompt, - options: { - abortController, - allowedTools, - ...(toolAdapter?.canUseTool && { - canUseTool: toolAdapter.canUseTool, - }), - ...(toolAdapter?.cwd && { cwd: toolAdapter.cwd }), - ...(toolAdapter?.env && { env: toolAdapter.env }), - maxTurns, - ...(toolAdapter?.mcpServers && { mcpServers: toolAdapter.mcpServers }), - model: normalizeClaudeCodeModel(model), - pathToClaudeCodeExecutable, - permissionMode, - settingSources: toolAdapter?.settingSources ?? [], - stderr: (data: string) => { - logger.log({ - category: "claude_code", - message: data, - level: 1, - }); - }, - systemPrompt: { - type: "preset", - preset: "claude_code", - append: - "You are being evaluated. Do not edit repository files. Complete the browser task and emit the requested EVAL_RESULT line.", - }, - }, - })) { - messages.push(message); - logClaudeCodeMessage(logger, message); - notifyToolResults(message, toolUseNames, toolAdapter?.onToolResult); - if (message.type === "result") { - resultMessage = message; - if (typeof message.result === "string") { - resultText = message.result; - } else if (Array.isArray(message.errors)) { - resultText = message.errors.join("\n"); - } - } - } - } catch (error) { - iterationError = error; - logger.warn({ - category: "claude_code", - message: `Claude Code stopped before a normal result: ${stringifyError(error)}`, - level: 0, - auxiliary: { - error: { - value: stringifyError(error), - type: "string", - }, + const sessionResult = await runClaudeAgentSession({ + prompt, + model, + logger, + sdk, + signal, + session: { + allowedTools: + toolAdapter?.allowedTools ?? + readCsvEnv("EVAL_CLAUDE_CODE_ALLOWED_TOOLS", ["WebFetch", "WebSearch"]), + permissionMode: process.env.EVAL_CLAUDE_CODE_PERMISSION_MODE ?? "default", + maxTurns: readPositiveIntEnv("EVAL_CLAUDE_CODE_MAX_TURNS", 50), + pathToClaudeCodeExecutable: process.env.EVAL_CLAUDE_CODE_EXECUTABLE || undefined, + cwd: toolAdapter?.cwd, + env: toolAdapter?.env, + mcpServers: toolAdapter?.mcpServers, + canUseTool: toolAdapter?.canUseTool, + settingSources: toolAdapter?.settingSources ?? [], + systemPromptPreset: { + preset: "claude_code", + append: + "You are being evaluated. Do not edit repository files. Complete the browser task and emit the requested EVAL_RESULT line.", }, - }); - } - + }, + onToolResult: toolAdapter?.onToolResult, + }); + const { messages, resultMessage, resultText, iterationError, status, stopReason, tokenUsage } = + sessionResult; const transcriptText = buildClaudeCodeTranscript(messages); const rawResult = [resultText, transcriptText, stringifyError(iterationError)] .filter(Boolean) .join("\n\n"); const parsed = parseClaudeCodeResult(rawResult); - const status = resolveClaudeCodeStatus(resultMessage, iterationError); - const stopReason = buildClaudeCodeStopReason(resultMessage, iterationError); const errorMessage = parsed.summary ?? stopReason ?? (resultText || transcriptText || "Claude Code did not report success"); - const tokenUsage = extractClaudeCodeTokenUsage(resultMessage); - const baseResult: TaskResult = { _success: parsed.success, error: !parsed.success ? errorMessage : undefined, @@ -267,20 +182,10 @@ export async function runClaudeCodeAgent({ logs: logger.getLogs(), metrics: buildClaudeCodeMetrics(resultMessage), }; + if (!verifier) return baseResult; - if (!verifier) { - return baseResult; - } - - // Artifact-grounded grading: capture the terminal page state through the - // tool surface (harness-observed, independent of the agent's self-report) - // before cleanup, and drain the per-step probe observations collected by - // the run tool. const finalObservation = await toolAdapter?.captureEvidence?.().catch((): undefined => undefined); const stepObservations = await toolAdapter?.drainStepObservations?.(); - - // Build a Trajectory from the SDK message stream and grade it with the - // rubric verifier; any failure in that path folds into `verifierError`. return gradeExternalTrajectory({ buildTrajectory: () => claudeCodeAdapter.fromHarnessResult( @@ -309,46 +214,10 @@ export async function runClaudeCodeAgent({ }); } -/** - * Surfaces completed tool_results to the tool adapter. Assistant messages - * register tool_use id -> name; user messages carry the tool_result blocks. - * onToolResult is called in stream order, so observation indexes assigned at - * call time line up with tool_use ordinals in the trajectory adapter. - */ -function notifyToolResults( - message: ClaudeSdkMessage, - toolUseNames: Map, - onToolResult?: (toolName: string) => void, -): void { - const type = String((message as Record).type ?? ""); - const inner = (message as Record).message; - if (!isRecord(inner) || !Array.isArray(inner.content)) return; - - if (type === "assistant") { - for (const block of inner.content) { - if (!isRecord(block) || block.type !== "tool_use") continue; - if (typeof block.id === "string" && typeof block.name === "string") { - toolUseNames.set(block.id, block.name); - } - } - return; - } - - if (type === "user" && onToolResult) { - for (const block of inner.content) { - if (!isRecord(block) || block.type !== "tool_result") continue; - const toolUseId = typeof block.tool_use_id === "string" ? block.tool_use_id : ""; - const name = toolUseNames.get(toolUseId); - if (name) onToolResult(name); - } - } -} - function buildClaudeCodeMetrics( resultMessage: ClaudeSdkMessage | undefined, ): Record { const tokenUsage = extractClaudeCodeTokenUsage(resultMessage); - return { claude_code_turns: metricValue(resultMessage?.num_turns), claude_code_duration_ms: metricValue(resultMessage?.duration_ms), @@ -361,57 +230,8 @@ function buildClaudeCodeMetrics( }; } -function extractClaudeCodeTokenUsage(resultMessage: ClaudeSdkMessage | undefined): { - inputTokens: number; - outputTokens: number; - cacheCreationInputTokens: number; - cacheReadInputTokens: number; - totalTokens: number; -} { - const usage = isRecord(resultMessage?.usage) ? resultMessage.usage : undefined; - - const inputTokens = - readNumber(usage, "input_tokens") ?? sumModelUsage(resultMessage, "inputTokens"); - const outputTokens = - readNumber(usage, "output_tokens") ?? sumModelUsage(resultMessage, "outputTokens"); - const cacheCreationInputTokens = - readNumber(usage, "cache_creation_input_tokens") ?? - sumModelUsage(resultMessage, "cacheCreationInputTokens"); - const cacheReadInputTokens = - readNumber(usage, "cache_read_input_tokens") ?? - sumModelUsage(resultMessage, "cacheReadInputTokens"); - const totalTokens = inputTokens + outputTokens + cacheCreationInputTokens + cacheReadInputTokens; - - return { - inputTokens, - outputTokens, - cacheCreationInputTokens, - cacheReadInputTokens, - totalTokens, - }; -} - -function sumModelUsage(resultMessage: ClaudeSdkMessage | undefined, key: string): number { - if (!isRecord(resultMessage?.modelUsage)) return 0; - - let total = 0; - for (const usage of Object.values(resultMessage.modelUsage)) { - if (!isRecord(usage)) continue; - total += readNumber(usage, key) ?? 0; - } - return total; -} - function metricValue(value: unknown): MetricValue { - return { - count: 1, - value: toFiniteNumber(value), - }; -} - -function readNumber(record: Record | undefined, key: string): number | undefined { - if (!record || !(key in record)) return undefined; - return toFiniteNumber(record[key]); + return { count: 1, value: toFiniteNumber(value) }; } function toFiniteNumber(value: unknown): number { @@ -421,163 +241,9 @@ function toFiniteNumber(value: unknown): number { : typeof value === "string" && value.trim() ? Number(value) : 0; - return Number.isFinite(parsed) ? parsed : 0; } -function resolveClaudeCodeStatus( - resultMessage: ClaudeSdkMessage | undefined, - iterationError: unknown, -): "completed" | "max_turns" | "sdk_error" { - if (isClaudeCodeMaxTurnsError(iterationError) || isClaudeCodeMaxTurnsError(resultMessage)) { - return "max_turns"; - } - if (iterationError || resultMessage?.is_error === true) { - return "sdk_error"; - } - return "completed"; -} - -function buildClaudeCodeStopReason( - resultMessage: ClaudeSdkMessage | undefined, - iterationError: unknown, -): string | undefined { - if (iterationError) return stringifyError(iterationError); - if (resultMessage?.is_error === true) { - const result = resultMessage.result; - if (typeof result === "string" && result.trim()) return result.trim(); - const errors = resultMessage.errors; - if (Array.isArray(errors) && errors.length > 0) { - return errors.map((error) => String(error)).join("\n"); - } - return "Claude Code returned an error result"; - } - return undefined; -} - -function buildClaudeCodeTranscript(messages: ClaudeSdkMessage[]): string { - return messages - .map((message) => summarizeClaudeCodeMessage(message).detail) - .filter((detail): detail is string => Boolean(detail)) - .join("\n"); -} - -function logClaudeCodeMessage(logger: EvalLogger, message: ClaudeSdkMessage): void { - const summary = summarizeClaudeCodeMessage(message); - logger.log({ - category: "claude_code", - message: summary.message, - level: 1, - auxiliary: { - type: { - value: String(message.type ?? "unknown"), - type: "string", - }, - ...(summary.detail && { - detail: { - value: summary.detail, - type: "string", - }, - }), - }, - }); -} - -function summarizeClaudeCodeMessage(message: ClaudeSdkMessage): { - message: string; - detail?: string; -} { - const type = String(message.type ?? "unknown"); - if (type === "assistant") { - const text = extractText(message); - return { - message: text ? `assistant: ${clip(text, 500)}` : "assistant message", - detail: text, - }; - } - if (type === "user") { - const text = extractText(message); - return { - message: text ? `user/tool: ${clip(text, 500)}` : "user/tool message", - detail: text, - }; - } - if (type === "result") { - return { - message: `result: ${String(message.subtype ?? "done")}`, - detail: typeof message.result === "string" ? message.result : undefined, - }; - } - return { - message: `${type} message`, - detail: safeJson(message), - }; -} - -function extractText(message: ClaudeSdkMessage): string | undefined { - const content = message.message; - if (!isRecord(content)) return undefined; - const rawContent = content.content; - if (typeof rawContent === "string") return rawContent; - if (!Array.isArray(rawContent)) return undefined; - const parts: string[] = []; - for (const block of rawContent) { - if (!isRecord(block)) continue; - if (typeof block.text === "string") { - parts.push(block.text); - continue; - } - if (typeof block.name === "string") { - parts.push(`[tool:${block.name}] ${safeJson(block.input) ?? ""}`.trim()); - continue; - } - if (typeof block.type === "string") { - parts.push(`[${block.type}] ${safeJson(block) ?? ""}`.trim()); - } - } - return parts.length > 0 ? parts.join("\n") : undefined; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - -function safeJson(value: unknown): string | undefined { - try { - return JSON.stringify(value); - } catch { - return undefined; - } -} - -function stringifyError(value: unknown): string { - if (!value) return ""; - if (value instanceof Error) return value.message; - if (typeof value === "string") return value; - return safeJson(value) ?? String(value); -} - -function clip(value: string, maxLength: number): string { - return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; -} - -async function loadClaudeAgentSdk(): Promise { - try { - const specifier = CLAUDE_AGENT_SDK_PACKAGE; - const mod = (await import(specifier)) as Partial; - if (typeof mod.query !== "function") { - throw new Error("query export missing"); - } - return { query: mod.query }; - } catch (error) { - throw new EvalsError( - `Claude Code harness requires ${CLAUDE_AGENT_SDK_PACKAGE}. Install it in packages/evals before running --harness claude_code. ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } -} - function readCsvEnv(key: string, fallback: string[]): string[] { const raw = process.env[key]; if (!raw) return fallback; diff --git a/packages/evals/package.json b/packages/evals/package.json index 16e3eca5be..ed97ae8b61 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -24,6 +24,7 @@ "@browserbasehq/sdk": "catalog:", "@browserbasehq/stagehand": "workspace:*", "@browserbasehq/stagehand-integrations": "workspace:*", + "@browserbasehq/stagehand-integrations-claude-agent-sdk": "workspace:*", "@openai/codex-sdk": "catalog:", "ai": "^5.0.133", "browse": "0.9.5", diff --git a/packages/evals/tests/framework/claudeCodeRunner.test.ts b/packages/evals/tests/framework/claudeCodeRunner.test.ts index dce64e8b20..747a6eedde 100644 --- a/packages/evals/tests/framework/claudeCodeRunner.test.ts +++ b/packages/evals/tests/framework/claudeCodeRunner.test.ts @@ -3,8 +3,6 @@ import { describe, expect, it } from "vitest"; import type { AvailableModel } from "stagehand-v3"; import { buildClaudeCodePrompt, - isClaudeCodeMaxTurnsError, - normalizeClaudeCodeModel, parseClaudeCodeResult, runClaudeCodeAgent, } from "../../framework/claudeCodeRunner.js"; @@ -20,13 +18,6 @@ const plan: ExternalHarnessTaskPlan = { }; describe("claude code runner helpers", () => { - it("normalizes provider-prefixed models for Claude Code", () => { - expect(normalizeClaudeCodeModel("anthropic/claude-sonnet-4-20250514" as AvailableModel)).toBe( - "claude-sonnet-4-20250514", - ); - expect(normalizeClaudeCodeModel("claude-opus-4-1" as AvailableModel)).toBe("claude-opus-4-1"); - }); - it("builds a browser task prompt with the required result marker", () => { const prompt = buildClaudeCodePrompt(plan, "Use browse only. Discover usage with browse -h."); @@ -82,61 +73,6 @@ describe("claude code runner helpers", () => { }); }); - it("identifies max-turn SDK errors", () => { - expect(isClaudeCodeMaxTurnsError(new Error("Reached maximum number of turns (20)"))).toBe(true); - expect(isClaudeCodeMaxTurnsError("network failed")).toBe(false); - }); - - it("returns a normal task result when Claude Code reaches max turns after emitting a result", async () => { - const sdk: ClaudeAgentSdk = { - query: async function* () { - yield { - type: "assistant", - message: { - content: [ - { - type: "text", - text: 'EVAL_RESULT: {"success":true,"summary":"already complete","finalAnswer":"done"}', - }, - ], - }, - }; - throw new Error("Reached maximum number of turns (20)"); - }, - }; - - const result = await runClaudeCodeAgent({ - plan, - model: "anthropic/claude-sonnet-4-20250514" as AvailableModel, - logger: new EvalLogger(false), - sdk, - }); - - expect(result._success).toBe(true); - expect(result.error).toBeUndefined(); - expect(result.claudeCodeStatus).toBe("max_turns"); - expect(result.finalAnswer).toBe("done"); - }); - - it("returns a failed task result instead of throwing when max turns prevents a result", async () => { - const sdk: ClaudeAgentSdk = { - query: async function* () { - throw new Error("Reached maximum number of turns (20)"); - }, - }; - - const result = await runClaudeCodeAgent({ - plan, - model: "anthropic/claude-sonnet-4-20250514" as AvailableModel, - logger: new EvalLogger(false), - sdk, - }); - - expect(result._success).toBe(false); - expect(result.claudeCodeStatus).toBe("max_turns"); - expect(String(result.error)).toContain("maximum number of turns"); - }); - it("surfaces verifier integration failures as verifierError on the self-reported result", async () => { const sdk: ClaudeAgentSdk = { query: async function* () { @@ -214,42 +150,4 @@ describe("claude code runner helpers", () => { expect(metrics.claude_code_cache_read_input_tokens.value).toBe(5); expect(metrics.claude_code_total_tokens.value).toBe(140); }); - - it("forwards adapter MCP servers into the Claude Code SDK query", async () => { - let capturedOptions: Record | undefined; - const mcpServers = { - stagehand_browser: { type: "sdk", name: "stagehand_browser" }, - }; - const sdk: ClaudeAgentSdk = { - query: async function* (input) { - capturedOptions = input.options; - yield { - type: "result", - subtype: "success", - result: 'EVAL_RESULT: {"success":true,"summary":"done","finalAnswer":"ok"}', - }; - }, - }; - - await runClaudeCodeAgent({ - plan, - model: "anthropic/claude-sonnet-4-20250514" as AvailableModel, - logger: new EvalLogger(false), - sdk, - toolAdapter: { - toolSurface: "playwright_code", - startupProfile: "runner_provided_local_cdp", - cwd: "/tmp/stagehand-evals-test", - env: {}, - allowedTools: ["Bash", "mcp__stagehand_browser__run"], - settingSources: [], - promptInstructions: "Use run.", - mcpServers, - cleanup: async () => {}, - }, - }); - - expect(capturedOptions?.mcpServers).toBe(mcpServers); - expect(capturedOptions?.allowedTools).toEqual(["Bash", "mcp__stagehand_browser__run"]); - }); }); diff --git a/packages/integrations/claude-agent-sdk/package.json b/packages/integrations/claude-agent-sdk/package.json new file mode 100644 index 0000000000..9e141d5629 --- /dev/null +++ b/packages/integrations/claude-agent-sdk/package.json @@ -0,0 +1,36 @@ +{ + "name": "@browserbasehq/stagehand-integrations-claude-agent-sdk", + "version": "4.0.1", + "private": true, + "description": "Claude Agent SDK harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/claude-agent-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/claude-agent-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@anthropic-ai/claude-agent-sdk": "catalog:", + "@browserbasehq/stagehand-integrations": "workspace:*", + "zod": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=22.18.0" + } +} diff --git a/packages/integrations/claude-agent-sdk/src/index.ts b/packages/integrations/claude-agent-sdk/src/index.ts new file mode 100644 index 0000000000..9df91886ba --- /dev/null +++ b/packages/integrations/claude-agent-sdk/src/index.ts @@ -0,0 +1 @@ +export * from "./session.js"; diff --git a/packages/integrations/claude-agent-sdk/src/session.ts b/packages/integrations/claude-agent-sdk/src/session.ts new file mode 100644 index 0000000000..b05f9e9e30 --- /dev/null +++ b/packages/integrations/claude-agent-sdk/src/session.ts @@ -0,0 +1,348 @@ +import { + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; + +export type ClaudeSdkMessage = Record; +type ClaudeQuery = AsyncIterable; + +export type ClaudeAgentSdk = { + query: (input: { prompt: string; options?: Record }) => ClaudeQuery; +}; + +export type ClaudeSessionConfig = { + cwd?: string; + env?: Record; + allowedTools?: string[]; + maxTurns?: number; + permissionMode?: string; + pathToClaudeCodeExecutable?: string; + settingSources?: string[]; + mcpServers?: Record; + canUseTool?: ( + toolName: string, + input: Record, + ) => Promise>; + systemPromptPreset?: { preset: "claude_code"; append?: string } | string; +}; + +export type ClaudeCodeTokenUsage = { + inputTokens: number; + outputTokens: number; + cacheCreationInputTokens: number; + cacheReadInputTokens: number; + totalTokens: number; +}; + +export type ClaudeSessionResult = { + messages: ClaudeSdkMessage[]; + resultMessage?: ClaudeSdkMessage; + resultText: string; + status: "completed" | "max_turns" | "sdk_error"; + stopReason?: string; + tokenUsage: ClaudeCodeTokenUsage; + iterationError?: unknown; +}; + +export const CLAUDE_AGENT_SDK_PACKAGE = "@anthropic-ai/claude-agent-sdk"; + +export async function loadClaudeAgentSdk(): Promise { + try { + const specifier = CLAUDE_AGENT_SDK_PACKAGE; + const mod = (await import(specifier)) as Partial; + if (typeof mod.query !== "function") { + throw new Error("query export missing"); + } + return { query: mod.query }; + } catch (error) { + const detail = sanitizeErrorMessage(stringifyError(error)); + throw new HarnessAdapterError( + `Claude Agent SDK harness requires ${CLAUDE_AGENT_SDK_PACKAGE}. Install it in the consuming workspace.${detail ? ` ${detail}` : ""}`, + { cause: error }, + ); + } +} + +export function normalizeClaudeModel(model: string): string { + return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; +} + +export async function runClaudeAgentSession(input: { + prompt: string; + model: string; + sdk?: ClaudeAgentSdk; + signal?: AbortSignal; + logger: HarnessLogger; + session: ClaudeSessionConfig; + onToolResult?: (toolName: string) => void | Promise; +}): Promise { + const sdk = input.sdk ?? (await loadClaudeAgentSdk()); + const abortController = new AbortController(); + if (input.signal) { + if (input.signal.aborted) abortController.abort(input.signal.reason); + input.signal.addEventListener("abort", () => abortController.abort(input.signal?.reason), { + once: true, + }); + } + + const messages: ClaudeSdkMessage[] = []; + let resultText = ""; + let resultMessage: ClaudeSdkMessage | undefined; + let iterationError: unknown; + const toolUseNames = new Map(); + const systemPrompt = + typeof input.session.systemPromptPreset === "string" + ? input.session.systemPromptPreset + : input.session.systemPromptPreset + ? { type: "preset", ...input.session.systemPromptPreset } + : undefined; + + try { + for await (const message of sdk.query({ + prompt: input.prompt, + options: { + abortController, + allowedTools: input.session.allowedTools ?? ["WebFetch", "WebSearch"], + ...(input.session.canUseTool && { canUseTool: input.session.canUseTool }), + ...(input.session.cwd && { cwd: input.session.cwd }), + ...(input.session.env && { env: input.session.env }), + maxTurns: input.session.maxTurns ?? 50, + ...(input.session.mcpServers && { mcpServers: input.session.mcpServers }), + model: normalizeClaudeModel(input.model), + pathToClaudeCodeExecutable: input.session.pathToClaudeCodeExecutable, + permissionMode: input.session.permissionMode ?? "default", + settingSources: input.session.settingSources ?? [], + stderr: (data: string) => { + input.logger.log({ category: "claude_code", message: data, level: 1 }); + }, + ...(systemPrompt !== undefined && { systemPrompt }), + }, + })) { + messages.push(message); + logClaudeCodeMessage(input.logger, message); + await notifyToolResults(message, toolUseNames, input.onToolResult); + if (message.type === "result") { + resultMessage = message; + if (typeof message.result === "string") { + resultText = message.result; + } else if (Array.isArray(message.errors)) { + resultText = message.errors.join("\n"); + } + } + } + } catch (error) { + iterationError = error; + input.logger.warn({ + category: "claude_code", + message: `Claude Code stopped before a normal result: ${stringifyError(error)}`, + level: 0, + auxiliary: { + error: { value: stringifyError(error), type: "string" }, + }, + }); + } + + return { + messages, + resultMessage, + resultText, + status: resolveClaudeCodeStatus(resultMessage, iterationError), + stopReason: buildClaudeCodeStopReason(resultMessage, iterationError), + tokenUsage: extractClaudeCodeTokenUsage(resultMessage), + ...(iterationError !== undefined && { iterationError }), + }; +} + +async function notifyToolResults( + message: ClaudeSdkMessage, + toolUseNames: Map, + onToolResult?: (toolName: string) => void | Promise, +): Promise { + const type = String(message.type ?? ""); + const inner = message.message; + if (!isRecord(inner) || !Array.isArray(inner.content)) return; + + if (type === "assistant") { + for (const block of inner.content) { + if (!isRecord(block) || block.type !== "tool_use") continue; + if (typeof block.id === "string" && typeof block.name === "string") { + toolUseNames.set(block.id, block.name); + } + } + return; + } + + if (type === "user" && onToolResult) { + for (const block of inner.content) { + if (!isRecord(block) || block.type !== "tool_result") continue; + const toolUseId = typeof block.tool_use_id === "string" ? block.tool_use_id : ""; + const name = toolUseNames.get(toolUseId); + if (name) await onToolResult(name); + } + } +} + +export function isClaudeCodeMaxTurnsError(value: unknown): boolean { + const message = stringifyError(value); + return /(?:maximum number of turns|max(?:imum)? turns|turn limit)/i.test(message); +} + +export function resolveClaudeCodeStatus( + resultMessage: ClaudeSdkMessage | undefined, + iterationError: unknown, +): "completed" | "max_turns" | "sdk_error" { + if (isClaudeCodeMaxTurnsError(iterationError) || isClaudeCodeMaxTurnsError(resultMessage)) { + return "max_turns"; + } + if (iterationError || resultMessage?.is_error === true) return "sdk_error"; + return "completed"; +} + +export function buildClaudeCodeStopReason( + resultMessage: ClaudeSdkMessage | undefined, + iterationError: unknown, +): string | undefined { + if (iterationError) return stringifyError(iterationError); + if (resultMessage?.is_error === true) { + if (typeof resultMessage.result === "string" && resultMessage.result.trim()) { + return resultMessage.result.trim(); + } + if (Array.isArray(resultMessage.errors) && resultMessage.errors.length > 0) { + return resultMessage.errors.map((error) => String(error)).join("\n"); + } + return "Claude Code returned an error result"; + } + return undefined; +} + +export function extractClaudeCodeTokenUsage( + resultMessage: ClaudeSdkMessage | undefined, +): ClaudeCodeTokenUsage { + const usage = isRecord(resultMessage?.usage) ? resultMessage.usage : undefined; + const inputTokens = + readNumber(usage, "input_tokens") ?? sumModelUsage(resultMessage, "inputTokens"); + const outputTokens = + readNumber(usage, "output_tokens") ?? sumModelUsage(resultMessage, "outputTokens"); + const cacheCreationInputTokens = + readNumber(usage, "cache_creation_input_tokens") ?? + sumModelUsage(resultMessage, "cacheCreationInputTokens"); + const cacheReadInputTokens = + readNumber(usage, "cache_read_input_tokens") ?? + sumModelUsage(resultMessage, "cacheReadInputTokens"); + return { + inputTokens, + outputTokens, + cacheCreationInputTokens, + cacheReadInputTokens, + totalTokens: inputTokens + outputTokens + cacheCreationInputTokens + cacheReadInputTokens, + }; +} + +function sumModelUsage(resultMessage: ClaudeSdkMessage | undefined, key: string): number { + if (!isRecord(resultMessage?.modelUsage)) return 0; + let total = 0; + for (const usage of Object.values(resultMessage.modelUsage)) { + if (isRecord(usage)) total += readNumber(usage, key) ?? 0; + } + return total; +} + +function readNumber(record: Record | undefined, key: string): number | undefined { + if (!record || !(key in record)) return undefined; + return toFiniteNumber(record[key]); +} + +function toFiniteNumber(value: unknown): number { + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && value.trim() + ? Number(value) + : 0; + return Number.isFinite(parsed) ? parsed : 0; +} + +export function buildClaudeCodeTranscript(messages: ClaudeSdkMessage[]): string { + return messages + .map((message) => summarizeClaudeCodeMessage(message).detail) + .filter((detail): detail is string => Boolean(detail)) + .join("\n"); +} + +export function logClaudeCodeMessage(logger: HarnessLogger, message: ClaudeSdkMessage): void { + const summary = summarizeClaudeCodeMessage(message); + logger.log({ + category: "claude_code", + message: summary.message, + level: 1, + auxiliary: { + type: { value: String(message.type ?? "unknown"), type: "string" }, + ...(summary.detail && { detail: { value: summary.detail, type: "string" } }), + }, + }); +} + +export function summarizeClaudeCodeMessage(message: ClaudeSdkMessage): { + message: string; + detail?: string; +} { + const type = String(message.type ?? "unknown"); + if (type === "assistant") { + const text = extractText(message); + return { message: text ? `assistant: ${clip(text, 500)}` : "assistant message", detail: text }; + } + if (type === "user") { + const text = extractText(message); + return { message: text ? `user/tool: ${clip(text, 500)}` : "user/tool message", detail: text }; + } + if (type === "result") { + return { + message: `result: ${String(message.subtype ?? "done")}`, + detail: typeof message.result === "string" ? message.result : undefined, + }; + } + return { message: `${type} message`, detail: safeJson(message) }; +} + +export function extractText(message: ClaudeSdkMessage): string | undefined { + const content = message.message; + if (!isRecord(content)) return undefined; + const rawContent = content.content; + if (typeof rawContent === "string") return rawContent; + if (!Array.isArray(rawContent)) return undefined; + const parts: string[] = []; + for (const block of rawContent) { + if (!isRecord(block)) continue; + if (typeof block.text === "string") parts.push(block.text); + else if (typeof block.name === "string") { + parts.push(`[tool:${block.name}] ${safeJson(block.input) ?? ""}`.trim()); + } else if (typeof block.type === "string") { + parts.push(`[${block.type}] ${safeJson(block) ?? ""}`.trim()); + } + } + return parts.length > 0 ? parts.join("\n") : undefined; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +export function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +export function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + return safeJson(value) ?? String(value); +} + +export function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/integrations/claude-agent-sdk/tests/session.test.ts b/packages/integrations/claude-agent-sdk/tests/session.test.ts new file mode 100644 index 0000000000..f2973d3ebd --- /dev/null +++ b/packages/integrations/claude-agent-sdk/tests/session.test.ts @@ -0,0 +1,109 @@ +/* eslint-disable require-yield */ +import { describe, expect, it } from "vitest"; +import { + isClaudeCodeMaxTurnsError, + normalizeClaudeModel, + runClaudeAgentSession, + type ClaudeAgentSdk, +} from "../src/index.js"; + +const logger = { + log: () => {}, + warn: () => {}, + error: () => {}, +}; + +describe("Claude Agent SDK session", () => { + it("normalizes provider-prefixed models", () => { + expect(normalizeClaudeModel("anthropic/claude-sonnet-4-20250514")).toBe( + "claude-sonnet-4-20250514", + ); + expect(normalizeClaudeModel("claude-opus-4-1")).toBe("claude-opus-4-1"); + }); + + it("classifies max-turn iteration errors without discarding streamed messages", async () => { + const sdk: ClaudeAgentSdk = { + query: async function* () { + yield { type: "assistant", message: { content: [{ type: "text", text: "done" }] } }; + throw new Error("Reached maximum number of turns (20)"); + }, + }; + const result = await runClaudeAgentSession({ + prompt: "task", + model: "anthropic/claude-sonnet-4-20250514", + logger, + sdk, + session: {}, + }); + + expect(result.status).toBe("max_turns"); + expect(result.messages).toHaveLength(1); + expect(result.stopReason).toContain("maximum number of turns"); + expect(isClaudeCodeMaxTurnsError(result.iterationError)).toBe(true); + }); + + it("forwards explicit session options and extracts result usage", async () => { + let capturedOptions: Record | undefined; + const mcpServers = { stagehand: { command: "node", args: ["server.mjs"] } }; + const sdk: ClaudeAgentSdk = { + query: async function* (input) { + capturedOptions = input.options; + yield { + type: "result", + subtype: "success", + result: "complete", + usage: { input_tokens: 10, output_tokens: 4 }, + }; + }, + }; + const result = await runClaudeAgentSession({ + prompt: "task", + model: "anthropic/claude-sonnet-4-20250514", + logger, + sdk, + session: { + allowedTools: ["mcp__stagehand"], + maxTurns: 7, + mcpServers, + systemPromptPreset: "Use Stagehand.", + }, + }); + + expect(capturedOptions).toMatchObject({ + model: "claude-sonnet-4-20250514", + allowedTools: ["mcp__stagehand"], + maxTurns: 7, + mcpServers, + systemPrompt: "Use Stagehand.", + }); + expect(result.resultText).toBe("complete"); + expect(result.tokenUsage.totalTokens).toBe(14); + }); + + it("attributes tool results to their tool-use names", async () => { + const completed: string[] = []; + const sdk: ClaudeAgentSdk = { + query: async function* () { + yield { + type: "assistant", + message: { content: [{ type: "tool_use", id: "tool-1", name: "mcp__stagehand__run" }] }, + }; + yield { + type: "user", + message: { content: [{ type: "tool_result", tool_use_id: "tool-1" }] }, + }; + }, + }; + await runClaudeAgentSession({ + prompt: "task", + model: "claude-sonnet-4-1", + logger, + sdk, + session: {}, + onToolResult: async (toolName) => { + completed.push(toolName); + }, + }); + expect(completed).toEqual(["mcp__stagehand__run"]); + }); +}); diff --git a/packages/integrations/claude-agent-sdk/tsconfig.json b/packages/integrations/claude-agent-sdk/tsconfig.json new file mode 100644 index 0000000000..6bddd941a2 --- /dev/null +++ b/packages/integrations/claude-agent-sdk/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts", "vitest.config.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/integrations/claude-agent-sdk/tsdown.config.ts b/packages/integrations/claude-agent-sdk/tsdown.config.ts new file mode 100644 index 0000000000..3b3b7c3bff --- /dev/null +++ b/packages/integrations/claude-agent-sdk/tsdown.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + }, + format: ["esm"], + platform: "node", + target: "node22", + dts: { + sourcemap: true, + }, + sourcemap: true, + outDir: "dist", +}); diff --git a/packages/integrations/claude-agent-sdk/vitest.config.ts b/packages/integrations/claude-agent-sdk/vitest.config.ts new file mode 100644 index 0000000000..d82214c96a --- /dev/null +++ b/packages/integrations/claude-agent-sdk/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts"], + hookTimeout: 20_000, + testTimeout: 20_000, + }, +}); diff --git a/packages/integrations/claude-code/package.json b/packages/integrations/claude-code/package.json index 94ef08c706..422ac38877 100644 --- a/packages/integrations/claude-code/package.json +++ b/packages/integrations/claude-code/package.json @@ -5,13 +5,13 @@ "type": "module", "scripts": { "start": "node src/agent.ts", - "test": "pnpm -w exec turbo run build --filter @browserbasehq/stagehand-integrations && vitest run", + "test": "pnpm -w exec turbo run build --filter @browserbasehq/stagehand-integrations-claude-agent-sdk && vitest run", "test:unit": "vitest run", "typecheck": "tsc --noEmit" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "catalog:", - "@browserbasehq/stagehand-integrations": "workspace:*" + "@browserbasehq/stagehand-integrations": "workspace:*", + "@browserbasehq/stagehand-integrations-claude-agent-sdk": "workspace:*" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/integrations/claude-code/src/agent.ts b/packages/integrations/claude-code/src/agent.ts index 14f3d95f1e..bdc82ba5e4 100644 --- a/packages/integrations/claude-code/src/agent.ts +++ b/packages/integrations/claude-code/src/agent.ts @@ -1,4 +1,4 @@ -import { query } from "@anthropic-ai/claude-agent-sdk"; +import { runClaudeAgentSession } from "@browserbasehq/stagehand-integrations-claude-agent-sdk"; import { FACADE_AGENT_INSTRUCTIONS, FACADE_TOOLS, @@ -6,25 +6,30 @@ import { import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; +const serverPath = fileURLToPath( + import.meta.resolve("@browserbasehq/stagehand-integrations/facade/stdio-server"), +); + export const STAGEHAND_TOOL_NAMES = FACADE_TOOLS.map((tool) => `mcp__stagehand__${tool.name}`); +const logger = { + log: () => {}, + warn: () => {}, + error: () => {}, +}; + async function main(): Promise { const args = process.argv.slice(2); const instruction = (args[0] === "--" ? args.slice(1) : args).join(" ").trim(); if (!instruction) throw new Error('Usage: pnpm start "your instruction"'); - // Resolved here (not at module load) so a missing build surfaces through - // handleFailure instead of an uncaught module error. - const serverPath = fileURLToPath( - import.meta.resolve("@browserbasehq/stagehand-integrations/facade/stdio-server"), - ); - - const result = query({ + const result = await runClaudeAgentSession({ prompt: instruction, - options: { - model: process.env.CLAUDE_STAGEHAND_MODEL ?? "claude-sonnet-5", + model: process.env.CLAUDE_STAGEHAND_MODEL ?? "claude-sonnet-5", + logger, + session: { maxTurns: 20, - systemPrompt: FACADE_AGENT_INSTRUCTIONS, + systemPromptPreset: FACADE_AGENT_INSTRUCTIONS, mcpServers: { stagehand: { command: process.execPath, @@ -35,24 +40,21 @@ async function main(): Promise { allowedTools: STAGEHAND_TOOL_NAMES, // Headless runs hang on any unanswered permission prompt; allow exactly // the stagehand tools and deny everything else. - canUseTool: async (toolName) => + canUseTool: async (toolName, input) => toolName.startsWith("mcp__stagehand__") - ? { behavior: "allow", updatedInput: undefined } + ? { behavior: "allow", updatedInput: input } : { behavior: "deny", message: "Only stagehand browser tools are permitted." }, }, }); - - for await (const message of result) { - if (message.type === "result") { - if (message.subtype === "success") { - // oxlint-disable-next-line no-console -- CLI example prints the agent result. - console.log(message.result); - return; - } - throw new Error(`Agent did not finish: ${message.subtype}`); - } + if (result.iterationError) throw result.iterationError; + if (!result.resultMessage) { + throw new Error("Agent stream ended without a result message."); + } + if (result.resultMessage.subtype !== "success") { + throw new Error(`Agent did not finish: ${String(result.resultMessage.subtype)}`); } - throw new Error("Agent stream ended without a result message."); + // oxlint-disable-next-line no-console -- CLI example prints the agent result. + console.log(result.resultText); } if (import.meta.main) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 126aa53d84..2e05bbd61d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -263,6 +263,9 @@ importers: '@browserbasehq/stagehand-integrations': specifier: workspace:* version: link:../integrations/core + '@browserbasehq/stagehand-integrations-claude-agent-sdk': + specifier: workspace:* + version: link:../integrations/claude-agent-sdk '@openai/codex-sdk': specifier: 'catalog:' version: 0.147.0 @@ -392,7 +395,7 @@ importers: specifier: 'catalog:' version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) - packages/integrations/claude-code: + packages/integrations/claude-agent-sdk: dependencies: '@anthropic-ai/claude-agent-sdk': specifier: 'catalog:' @@ -400,6 +403,31 @@ importers: '@browserbasehq/stagehand-integrations': specifier: workspace:* version: link:../core + zod: + specifier: 'catalog:' + version: 4.4.3 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + + packages/integrations/claude-code: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@browserbasehq/stagehand-integrations-claude-agent-sdk': + specifier: workspace:* + version: link:../claude-agent-sdk devDependencies: '@types/node': specifier: 'catalog:' diff --git a/turbo.json b/turbo.json index 32b50787d1..41f695758d 100644 --- a/turbo.json +++ b/turbo.json @@ -31,6 +31,11 @@ "inputs": ["$TURBO_DEFAULT$", "!dist/**"], "outputs": ["dist/**"] }, + "@browserbasehq/stagehand-integrations-claude-agent-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, "@browserbasehq/stagehand-evals#build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "!dist/**"], @@ -74,6 +79,9 @@ "@browserbasehq/stagehand-integrations#typecheck": { "dependsOn": ["^build"] }, + "@browserbasehq/stagehand-integrations-claude-agent-sdk#typecheck": { + "dependsOn": ["^build"] + }, "@browserbasehq/stagehand-integrations#test:unit": { "dependsOn": ["^build", "@browserbasehq/stagehand-integrations#build"], "inputs": [ @@ -84,6 +92,16 @@ "$TURBO_ROOT$/vitest.config.ts" ] }, + "@browserbasehq/stagehand-integrations-claude-agent-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-claude-agent-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] + }, "@browserbasehq/stagehand-integrations-example-mastra-facade#typecheck": { "dependsOn": ["^build"] }, diff --git a/vitest.config.ts b/vitest.config.ts index 126b7d3755..943eaf3fdf 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ "packages/docs/tests/**/*.test.ts", "packages/evals/tests/**/*.test.ts", "packages/integrations/core/tests/**/*.test.ts", + "packages/integrations/claude-agent-sdk/tests/**/*.test.ts", "packages/extension/tests/**/*.test.ts", "packages/sdk-ts/tests/**/*.test.ts", "packages/extension/understudy/**/*.test.ts", From 9a5aacbd59b52a30c1db67a438efc889705cc9a0 Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 17 Aug 2026 11:42:55 -0700 Subject: [PATCH 2/3] =?UTF-8?q?fix(claude-agent-sdk):=20review=20fixes=20?= =?UTF-8?q?=E2=80=94=20abort=20forwarder=20cleanup,=20error=20redaction,?= =?UTF-8?q?=20drop=20unused=20vitest=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remove the abort forwarder on session completion so long-lived caller signals don't accumulate listeners (cubic P2) - pass iteration errors through sanitizeErrorMessage before they reach stop reasons and logs (cubic P1) - delete the package vitest.config.ts: tests run through the repo-root config via --root (cubic P3) --- .../integrations/claude-agent-sdk/src/session.ts | 15 +++++++++------ .../claude-agent-sdk/vitest.config.ts | 9 --------- 2 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 packages/integrations/claude-agent-sdk/vitest.config.ts diff --git a/packages/integrations/claude-agent-sdk/src/session.ts b/packages/integrations/claude-agent-sdk/src/session.ts index b05f9e9e30..7a9540bfba 100644 --- a/packages/integrations/claude-agent-sdk/src/session.ts +++ b/packages/integrations/claude-agent-sdk/src/session.ts @@ -79,11 +79,10 @@ export async function runClaudeAgentSession(input: { }): Promise { const sdk = input.sdk ?? (await loadClaudeAgentSdk()); const abortController = new AbortController(); + const forwardAbort = (): void => abortController.abort(input.signal?.reason); if (input.signal) { if (input.signal.aborted) abortController.abort(input.signal.reason); - input.signal.addEventListener("abort", () => abortController.abort(input.signal?.reason), { - once: true, - }); + input.signal.addEventListener("abort", forwardAbort, { once: true }); } const messages: ClaudeSdkMessage[] = []; @@ -135,12 +134,16 @@ export async function runClaudeAgentSession(input: { iterationError = error; input.logger.warn({ category: "claude_code", - message: `Claude Code stopped before a normal result: ${stringifyError(error)}`, + message: `Claude Code stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`, level: 0, auxiliary: { - error: { value: stringifyError(error), type: "string" }, + error: { value: sanitizeErrorMessage(stringifyError(error)), type: "string" }, }, }); + } finally { + // A long-lived caller signal must not accumulate forwarders across + // sessions that complete without aborting. + input.signal?.removeEventListener("abort", forwardAbort); } return { @@ -203,7 +206,7 @@ export function buildClaudeCodeStopReason( resultMessage: ClaudeSdkMessage | undefined, iterationError: unknown, ): string | undefined { - if (iterationError) return stringifyError(iterationError); + if (iterationError) return sanitizeErrorMessage(stringifyError(iterationError)); if (resultMessage?.is_error === true) { if (typeof resultMessage.result === "string" && resultMessage.result.trim()) { return resultMessage.result.trim(); diff --git a/packages/integrations/claude-agent-sdk/vitest.config.ts b/packages/integrations/claude-agent-sdk/vitest.config.ts deleted file mode 100644 index d82214c96a..0000000000 --- a/packages/integrations/claude-agent-sdk/vitest.config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineConfig } from "vitest/config"; - -export default defineConfig({ - test: { - include: ["tests/**/*.test.ts"], - hookTimeout: 20_000, - testTimeout: 20_000, - }, -}); From e82de0a7c2456f78ac339c916be2561387a5596c Mon Sep 17 00:00:00 2001 From: miguel Date: Mon, 17 Aug 2026 11:48:23 -0700 Subject: [PATCH 3/3] test(claude-agent-sdk): cover stop-reason redaction and abort-forwarder detach --- .../claude-agent-sdk/tests/session.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/integrations/claude-agent-sdk/tests/session.test.ts b/packages/integrations/claude-agent-sdk/tests/session.test.ts index f2973d3ebd..40ee2eabbd 100644 --- a/packages/integrations/claude-agent-sdk/tests/session.test.ts +++ b/packages/integrations/claude-agent-sdk/tests/session.test.ts @@ -106,4 +106,37 @@ describe("Claude Agent SDK session", () => { }); expect(completed).toEqual(["mcp__stagehand__run"]); }); + + it("redacts secrets from stop reasons and detaches the abort forwarder", async () => { + const sdk: ClaudeAgentSdk = { + query: async function* () { + yield { type: "assistant", message: { content: [] } }; + throw new Error( + "connect failed: wss://c.example?signingKey=top-secret with sk-abcdef1234567890", + ); + }, + }; + const added: string[] = []; + const removed: string[] = []; + const signal = { + aborted: false, + reason: undefined, + addEventListener: (type: string) => added.push(type), + removeEventListener: (type: string) => removed.push(type), + } as unknown as AbortSignal; + const result = await runClaudeAgentSession({ + prompt: "task", + model: "claude-sonnet-4-5", + sdk, + signal, + logger, + session: {}, + }); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toContain("signingKey=[redacted]"); + expect(result.stopReason).toContain("sk-abcdef[redacted]"); + expect(result.stopReason).not.toContain("top-secret"); + expect(added).toEqual(["abort"]); + expect(removed).toEqual(["abort"]); + }); });