diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88edb0875..e0afd7455 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,6 +171,7 @@ jobs: packages/sdk-ts/dist/** packages/integrations/core/dist/** packages/integrations/claude-agent-sdk/dist/** + packages/integrations/codex-sdk/dist/** packages/evals/dist/** retention-days: 1 diff --git a/packages/evals/framework/codexRunner.ts b/packages/evals/framework/codexRunner.ts index 614bded1d..860fffeeb 100644 --- a/packages/evals/framework/codexRunner.ts +++ b/packages/evals/framework/codexRunner.ts @@ -1,32 +1,32 @@ +import { + buildCodexTranscript, + loadCodexSdk, + normalizeCodexModel, + runCodexSession, + stringifyError, + toFiniteNumber, + validateCodexApprovalPolicy, + validateCodexSandboxMode, + type CodexSdk, + type CodexTokenUsage, +} from "@browserbasehq/stagehand-integrations-codex-sdk"; import type { AvailableModel } from "stagehand-v3"; -import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; -import type { TaskResult } from "./types.js"; -import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; -import { datasetPromptGuidance } from "./externalHarnessPlan.js"; import type { PreparedCodexToolAdapter } from "./codexToolAdapter.js"; +import { datasetPromptGuidance, type ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; import { codexAdapter } from "./harnesses/codexAdapter.js"; +import type { TaskResult } from "./types.js"; import { gradeExternalTrajectory, type ExternalHarnessVerifierConfig } from "./verifierAdapter.js"; -type MetricValue = { count: number; value: number }; -type CodexEvent = Record; -type CodexUsage = { - input_tokens?: number; - cached_input_tokens?: number; - output_tokens?: number; - reasoning_output_tokens?: number; -}; +export type { CodexSdk, CodexThread } from "@browserbasehq/stagehand-integrations-codex-sdk"; +export { + buildCodexTranscript, + loadCodexSdk, + normalizeCodexModel, + runCodexSession, +} from "@browserbasehq/stagehand-integrations-codex-sdk"; -export type CodexThread = { - runStreamed: ( - input: string, - options?: Record, - ) => Promise<{ events: AsyncIterable }>; -}; - -export type CodexSdk = { - startThread: (options?: Record) => CodexThread; -}; +type MetricValue = { count: number; value: number }; export interface CodexRunnerInput { plan: ExternalHarnessTaskPlan; @@ -35,15 +35,6 @@ export interface CodexRunnerInput { toolAdapter?: PreparedCodexToolAdapter; signal?: AbortSignal; sdk?: CodexSdk; - /** - * Optional verifier integration. When provided, the runner builds a - * Trajectory from the codex event stream (via codexAdapter), runs - * V3Evaluator.verify() against the trajectory's embedded TaskSpec, and folds - * the EvaluationResult into the returned TaskResult ({_success} mode follows - * EVAL_SUCCESS_MODE). - * When omitted, the runner falls back to parsing the legacy JSON result — - * preserves current behavior for callers that haven't migrated. - */ verifier?: ExternalHarnessVerifierConfig; } @@ -54,8 +45,6 @@ export interface ParsedCodexResult { raw: string; } -const CODEX_SDK_PACKAGE = "@openai/codex-sdk"; - const EVAL_RESULT_SCHEMA = { type: "object", properties: { @@ -67,21 +56,6 @@ const EVAL_RESULT_SCHEMA = { additionalProperties: false, } as const; -/** Tool-step budget: EVAL_CODEX_MAX_STEPS, falling back to the pre-migration - * AGENT_EVAL_MAX_STEPS knob so carried-forward run configs keep working. */ -function readCodexMaxToolSteps(): number { - for (const key of ["EVAL_CODEX_MAX_STEPS", "AGENT_EVAL_MAX_STEPS"]) { - const parsed = Number.parseInt(process.env[key] ?? "", 10); - if (Number.isFinite(parsed) && parsed > 0) return parsed; - } - return 50; -} - -export function normalizeCodexModel(model: AvailableModel): string { - if (model === ("codex/default" as AvailableModel)) return "gpt-5.4-mini"; - return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; -} - export function buildCodexPrompt(plan: ExternalHarnessTaskPlan, toolInstructions?: string): string { return [ "You are running a browser benchmark task.", @@ -123,7 +97,6 @@ export function parseCodexResult(raw: string): ParsedCodexResult { const parsed = tryParseCodexJson(candidate); if (parsed) return { ...parsed, raw }; } - return { success: false, raw }; } @@ -133,124 +106,45 @@ export async function runCodexAgent({ logger, toolAdapter, signal, - sdk: injectedSdk, + sdk, verifier, }: CodexRunnerInput): Promise { - const sdk = - injectedSdk ?? - (await loadCodexSdk( - toolAdapter?.env, - toolAdapter && "codexConfig" in toolAdapter ? toolAdapter.codexConfig : undefined, - )); const prompt = buildCodexPrompt(plan, toolAdapter?.promptInstructions); - const events: CodexEvent[] = []; - let finalResponse = ""; - let usage: CodexUsage | undefined; - let iterationError: unknown; - let stopReason: string | undefined; - - // Codex has no native turn/step limit, so a run could grind indefinitely. - // Cap tool steps (command executions + MCP tool calls) and abort the - // stream at the budget — the analogue of Claude Code's maxTurns, and it - // gives every published run a citable, reproducible action budget. - const maxToolSteps = readCodexMaxToolSteps(); - const budgetController = new AbortController(); - const forwardAbort = () => budgetController.abort(signal?.reason); - if (signal) { - if (signal.aborted) budgetController.abort(signal.reason); - else signal.addEventListener("abort", forwardAbort, { once: true }); - } - let toolStepCount = 0; - - try { - const thread = sdk.startThread({ - model: normalizeCodexModel(model), - ...(toolAdapter?.cwd && { - workingDirectory: toolAdapter.cwd, - skipGitRepoCheck: true, - }), - sandboxMode: readCodexSandboxMode(), - approvalPolicy: readCodexApprovalPolicy(), + const sessionResult = await runCodexSession({ + prompt, + model, + logger, + sdk: + sdk ?? + (await loadEvalCodexSdk( + toolAdapter?.env, + toolAdapter && "codexConfig" in toolAdapter ? toolAdapter.codexConfig : undefined, + )), + signal, + thread: { + ...(toolAdapter?.cwd && { workingDirectory: toolAdapter.cwd }), + sandboxMode: validateCodexSandboxMode(process.env.EVAL_CODEX_SANDBOX_MODE), + approvalPolicy: validateCodexApprovalPolicy(process.env.EVAL_CODEX_APPROVAL_POLICY), networkAccessEnabled: readBooleanEnv("EVAL_CODEX_NETWORK_ACCESS", true), webSearchMode: "disabled", - }); - const streamed = await thread.runStreamed(prompt, { - outputSchema: EVAL_RESULT_SCHEMA, - signal: budgetController.signal, - }); - - for await (const event of streamed.events) { - events.push(event); - logCodexEvent(logger, event); - - if (event.type === "turn.completed" && isRecord(event.usage)) { - usage = event.usage; - } else if (event.type === "turn.failed") { - stopReason = readCodexErrorMessage(event.error); - } else if (event.type === "error") { - stopReason = typeof event.message === "string" ? event.message : "error"; - } - - const item = isRecord(event.item) ? event.item : undefined; - if ( - event.type === "item.completed" && - item?.type === "agent_message" && - typeof item.text === "string" - ) { - finalResponse = item.text; - } - if ( - event.type === "item.completed" && - (item?.type === "command_execution" || item?.type === "mcp_tool_call") - ) { - toolStepCount += 1; - if (toolStepCount >= maxToolSteps && !budgetController.signal.aborted) { - stopReason = `tool step budget exhausted (${maxToolSteps} steps)`; - budgetController.abort(new Error(stopReason)); - } - } - // MCP-mounted surfaces: each completed MCP tool call consumes one - // observation index, in stream order (mirrors the bridge's - // onRunExecuted for code surfaces). - if ( - event.type === "item.completed" && - item?.type === "mcp_tool_call" && - toolAdapter && - "recordObservation" in toolAdapter - ) { - toolAdapter.recordObservation?.(); - } - } - } catch (error) { - iterationError = error; - logger.warn({ - category: "codex", - message: `Codex stopped before a normal result: ${stringifyError(error)}`, - level: 0, - auxiliary: { - error: { - value: stringifyError(error), - type: "string", - }, - }, - }); - } - - // Long batches share one abort signal; detach the forwarder so completed - // runs don't accumulate listeners on it. - signal?.removeEventListener("abort", forwardAbort); - + skipGitRepoCheck: true, + }, + outputSchema: EVAL_RESULT_SCHEMA, + maxToolSteps: readCodexMaxToolSteps(), + onToolStep: + toolAdapter && "recordObservation" in toolAdapter ? toolAdapter.recordObservation : undefined, + }); + const { events, finalMessage, iterationError, status, stopReason, tokenUsage } = sessionResult; const transcriptText = buildCodexTranscript(events); const iterationErrorMessage = stringifyError(iterationError); - const rawResult = [finalResponse, transcriptText, iterationErrorMessage] + const rawResult = [finalMessage, transcriptText, iterationErrorMessage] .filter(Boolean) .join("\n\n"); const parsed = parseCodexResult(rawResult); - const status = resolveCodexStatus(iterationError, stopReason); const errorMessage = parsed.summary ?? stopReason ?? - (iterationErrorMessage || finalResponse || transcriptText || "Codex did not report success"); + (iterationErrorMessage || finalMessage || transcriptText || "Codex did not report success"); const baseResult: TaskResult = { _success: parsed.success, error: !parsed.success ? errorMessage : undefined, @@ -260,16 +154,10 @@ export async function runCodexAgent({ codexStatus: status, ...(stopReason && { codexStopReason: stopReason }), logs: logger.getLogs(), - metrics: buildCodexMetrics(usage), + metrics: buildCodexMetrics(tokenUsage), }; + if (!verifier) return baseResult; - if (!verifier) { - return baseResult; - } - - // Artifact-grounded grading: capture the terminal page state through the - // tool surface (harness-observed, independent of the agent's self-report) - // before cleanup, mirroring the claude_code runner. const finalObservation = toolAdapter && "captureEvidence" in toolAdapter ? await toolAdapter.captureEvidence?.().catch((): undefined => undefined) @@ -278,9 +166,6 @@ export async function runCodexAgent({ toolAdapter && "drainStepObservations" in toolAdapter ? await toolAdapter.drainStepObservations?.() : undefined; - - // Build a Trajectory from the codex event stream and grade it with the - // rubric verifier; any failure in that path folds into `verifierError`. return gradeExternalTrajectory({ buildTrajectory: () => codexAdapter.fromHarnessResult( @@ -293,17 +178,13 @@ export async function runCodexAgent({ toolAdapter.observedToolMatcher && { observedToolName: toolAdapter.observedToolMatcher, }), - finalAnswer: parsed.finalAnswer ?? finalResponse, + finalAnswer: parsed.finalAnswer ?? finalMessage, status: status === "completed" ? "complete" : "error", usage: { - input_tokens: toFiniteNumber(usage?.input_tokens), - output_tokens: toFiniteNumber(usage?.output_tokens), - ...(usage?.reasoning_output_tokens !== undefined && { - reasoning_tokens: toFiniteNumber(usage.reasoning_output_tokens), - }), - ...(usage?.cached_input_tokens !== undefined && { - cached_input_tokens: toFiniteNumber(usage.cached_input_tokens), - }), + input_tokens: tokenUsage.input_tokens, + output_tokens: tokenUsage.output_tokens, + reasoning_tokens: tokenUsage.reasoning_output_tokens, + cached_input_tokens: tokenUsage.cached_input_tokens, }, }, verifier.taskSpec, @@ -316,6 +197,34 @@ export async function runCodexAgent({ }); } +async function loadEvalCodexSdk( + env?: Record, + extraConfig?: Record, +): Promise { + return loadCodexSdk({ + env, + codexPathOverride: process.env.EVAL_CODEX_PATH, + baseUrl: process.env.EVAL_CODEX_BASE_URL, + apiKey: process.env.OPENAI_API_KEY, + rawReasoning: process.env.EVAL_CODEX_RAW_REASONING === "true", + extraConfig, + }); +} + +function readCodexMaxToolSteps(): number { + for (const key of ["EVAL_CODEX_MAX_STEPS", "AGENT_EVAL_MAX_STEPS"]) { + const parsed = Number.parseInt(process.env[key] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + return 50; +} + +function readBooleanEnv(key: string, fallback: boolean): boolean { + const raw = process.env[key]; + if (!raw) return fallback; + return raw === "true" || raw === "1"; +} + function tryParseCodexJson(candidate: string): Omit | undefined { try { const parsed = JSON.parse(candidate) as { @@ -333,19 +242,11 @@ function tryParseCodexJson(candidate: string): Omit | } } -function resolveCodexStatus( - iterationError: unknown, - stopReason: string | undefined, -): "completed" | "sdk_error" { - return iterationError || stopReason ? "sdk_error" : "completed"; -} - -function buildCodexMetrics(usage: CodexUsage | undefined): Record { - const inputTokens = toFiniteNumber(usage?.input_tokens); - const cachedInputTokens = toFiniteNumber(usage?.cached_input_tokens); - const outputTokens = toFiniteNumber(usage?.output_tokens); - const reasoningOutputTokens = toFiniteNumber(usage?.reasoning_output_tokens); - +function buildCodexMetrics(usage: CodexTokenUsage): Record { + const inputTokens = toFiniteNumber(usage.input_tokens); + const cachedInputTokens = toFiniteNumber(usage.cached_input_tokens); + const outputTokens = toFiniteNumber(usage.output_tokens); + const reasoningOutputTokens = toFiniteNumber(usage.reasoning_output_tokens); return { codex_input_tokens: metricValue(inputTokens), codex_cached_input_tokens: metricValue(cachedInputTokens), @@ -358,195 +259,5 @@ function buildCodexMetrics(usage: CodexUsage | undefined): Record summarizeCodexEvent(event).detail) - .filter((detail): detail is string => Boolean(detail)) - .join("\n"); -} - -function logCodexEvent(logger: EvalLogger, event: CodexEvent): void { - const summary = summarizeCodexEvent(event); - logger.log({ - category: "codex", - message: summary.message, - level: 1, - auxiliary: { - type: { - value: String(event.type ?? "unknown"), - type: "string", - }, - ...(summary.detail && { - detail: { - value: summary.detail, - type: "string", - }, - }), - }, - }); -} - -function summarizeCodexEvent(event: CodexEvent): { - message: string; - detail?: string; -} { - const type = String(event.type ?? "unknown"); - const item = isRecord(event.item) ? event.item : undefined; - if (item?.type === "agent_message" && typeof item.text === "string") { - return { - message: `agent: ${clip(item.text, 500)}`, - detail: item.text, - }; - } - if (item?.type === "command_execution") { - return { - message: `command: ${String(item.command ?? "")} ${String(item.status ?? "")}`.trim(), - detail: safeJson(item), - }; - } - if (item?.type === "mcp_tool_call") { - return { - message: - `mcp: ${String(item.server ?? "")}.${String(item.tool ?? "")} ${String(item.status ?? "")}`.trim(), - detail: safeJson(item), - }; - } - if (item?.type === "error" && typeof item.message === "string") { - return { - message: `error item: ${clip(item.message, 500)}`, - detail: item.message, - }; - } - if (type === "turn.completed") { - return { - message: "turn completed", - detail: safeJson(event.usage), - }; - } - if (type === "turn.failed") { - const message = readCodexErrorMessage(event.error) ?? "turn failed"; - return { - message: `turn failed: ${clip(message, 500)}`, - detail: message, - }; - } - if (type === "error" && typeof event.message === "string") { - return { - message: `error: ${clip(event.message, 500)}`, - detail: event.message, - }; - } - return { - message: `${type} event`, - detail: safeJson(event), - }; -} - -async function loadCodexSdk( - env?: Record, - extraConfig?: Record, -): Promise { - try { - const specifier = CODEX_SDK_PACKAGE; - const mod = (await import(specifier)) as { - Codex?: new (options?: Record) => CodexSdk; - }; - if (typeof mod.Codex !== "function") { - throw new Error("Codex export missing"); - } - return new mod.Codex({ - ...(env && { env }), - ...(process.env.EVAL_CODEX_PATH && { - codexPathOverride: process.env.EVAL_CODEX_PATH, - }), - ...(process.env.EVAL_CODEX_BASE_URL && { - baseUrl: process.env.EVAL_CODEX_BASE_URL, - }), - ...(process.env.OPENAI_API_KEY && { - apiKey: process.env.OPENAI_API_KEY, - }), - config: { - show_raw_agent_reasoning: process.env.EVAL_CODEX_RAW_REASONING === "true", - // Adapter-provided overrides (e.g. mcp_servers for MCP mounts). - ...extraConfig, - }, - }); - } catch (error) { - throw new EvalsError( - `Codex harness requires ${CODEX_SDK_PACKAGE}. Install it in packages/evals before running --harness codex. ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } -} - -function readCodexSandboxMode(): "read-only" | "workspace-write" | "danger-full-access" { - const raw = process.env.EVAL_CODEX_SANDBOX_MODE; - if (raw === "read-only" || raw === "workspace-write" || raw === "danger-full-access") { - return raw; - } - return "workspace-write"; -} - -function readCodexApprovalPolicy(): "never" | "on-request" | "on-failure" | "untrusted" { - const raw = process.env.EVAL_CODEX_APPROVAL_POLICY; - if (raw === "never" || raw === "on-request" || raw === "on-failure" || raw === "untrusted") { - return raw; - } - return "never"; -} - -function readBooleanEnv(key: string, fallback: boolean): boolean { - const raw = process.env[key]; - if (!raw) return fallback; - return raw === "true" || raw === "1"; -} - -function readCodexErrorMessage(value: unknown): string | undefined { - if (!value) return undefined; - if (typeof value === "string") return value; - if (isRecord(value) && typeof value.message === "string") { - return value.message; - } - return safeJson(value); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - -function safeJson(value: unknown): string | undefined { - try { - return JSON.stringify(value); - } catch { - return undefined; - } -} - -function stringifyError(value: unknown): string { - if (!value) return ""; - if (value instanceof Error) return value.message; - if (typeof value === "string") return value; - return safeJson(value) ?? String(value); -} - -function clip(value: string, maxLength: number): string { - return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; + return { count: 1, value: toFiniteNumber(value) }; } diff --git a/packages/evals/package.json b/packages/evals/package.json index ed97ae8b6..ca0723322 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -25,7 +25,7 @@ "@browserbasehq/stagehand": "workspace:*", "@browserbasehq/stagehand-integrations": "workspace:*", "@browserbasehq/stagehand-integrations-claude-agent-sdk": "workspace:*", - "@openai/codex-sdk": "catalog:", + "@browserbasehq/stagehand-integrations-codex-sdk": "workspace:*", "ai": "^5.0.133", "browse": "0.9.5", "dotenv": "^17.3.1", diff --git a/packages/evals/tests/framework/codexRunner.test.ts b/packages/evals/tests/framework/codexRunner.test.ts index 0154ab717..7ccd5fafb 100644 --- a/packages/evals/tests/framework/codexRunner.test.ts +++ b/packages/evals/tests/framework/codexRunner.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import type { AvailableModel } from "stagehand-v3"; import { buildCodexPrompt, - normalizeCodexModel, parseCodexResult, runCodexAgent, type CodexSdk, @@ -18,12 +17,6 @@ const plan: ExternalHarnessTaskPlan = { }; describe("codex runner helpers", () => { - it("normalizes provider-prefixed models for Codex", () => { - expect(normalizeCodexModel("openai/gpt-5.4-mini" as AvailableModel)).toBe("gpt-5.4-mini"); - expect(normalizeCodexModel("gpt-5.4" as AvailableModel)).toBe("gpt-5.4"); - expect(normalizeCodexModel("codex/default" as AvailableModel)).toBe("gpt-5.4-mini"); - }); - it("builds a browser task prompt with structured result instructions", () => { const prompt = buildCodexPrompt(plan, "Use browse only. Discover usage with browse -h."); diff --git a/packages/integrations/codex-sdk/package.json b/packages/integrations/codex-sdk/package.json new file mode 100644 index 000000000..494827df2 --- /dev/null +++ b/packages/integrations/codex-sdk/package.json @@ -0,0 +1,35 @@ +{ + "name": "@browserbasehq/stagehand-integrations-codex-sdk", + "version": "4.0.1", + "private": true, + "description": "Codex SDK harness adapter for Stagehand integrations", + "files": [ + "dist" + ], + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.mts", + "import": "./dist/index.mjs" + } + }, + "scripts": { + "build": "tsdown", + "test": "pnpm run build && vitest run --root ../../.. packages/integrations/codex-sdk/tests", + "test:unit": "vitest run --root ../../.. packages/integrations/codex-sdk/tests", + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "@browserbasehq/stagehand-integrations": "workspace:*", + "@openai/codex-sdk": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "engines": { + "node": ">=22.18.0" + } +} diff --git a/packages/integrations/codex-sdk/src/index.ts b/packages/integrations/codex-sdk/src/index.ts new file mode 100644 index 000000000..9df91886b --- /dev/null +++ b/packages/integrations/codex-sdk/src/index.ts @@ -0,0 +1 @@ +export * from "./session.js"; diff --git a/packages/integrations/codex-sdk/src/session.ts b/packages/integrations/codex-sdk/src/session.ts new file mode 100644 index 000000000..40aa6f4df --- /dev/null +++ b/packages/integrations/codex-sdk/src/session.ts @@ -0,0 +1,336 @@ +import { + HarnessAdapterError, + sanitizeErrorMessage, + type HarnessLogger, +} from "@browserbasehq/stagehand-integrations/harness"; + +export type CodexEvent = Record; + +export type CodexThread = { + runStreamed: ( + input: string, + options?: Record, + ) => Promise<{ events: AsyncIterable }>; +}; + +export type CodexSdk = { + startThread: (options?: Record) => CodexThread; +}; + +export type CodexThreadConfig = { + workingDirectory?: string; + sandboxMode?: "read-only" | "workspace-write" | "danger-full-access"; + approvalPolicy?: "never" | "on-request" | "on-failure" | "untrusted"; + networkAccessEnabled?: boolean; + webSearchMode?: string; + skipGitRepoCheck?: boolean; +}; + +export type CodexTokenUsage = { + input_tokens: number; + cached_input_tokens: number; + output_tokens: number; + reasoning_output_tokens: number; +}; + +export type CodexSessionResult = { + events: CodexEvent[]; + finalMessage: string; + status: "completed" | "sdk_error"; + stopReason?: string; + tokenUsage: CodexTokenUsage; + iterationError?: unknown; +}; + +export const CODEX_SDK_PACKAGE = "@openai/codex-sdk"; + +export async function loadCodexSdk( + options: { + env?: Record; + codexPathOverride?: string; + baseUrl?: string; + apiKey?: string; + rawReasoning?: boolean; + extraConfig?: Record; + } = {}, +): Promise { + let codexCtor: new (options?: Record) => CodexSdk; + try { + const specifier = CODEX_SDK_PACKAGE; + const mod = (await import(specifier)) as { + Codex?: new (options?: Record) => CodexSdk; + }; + if (typeof mod.Codex !== "function") throw new Error("Codex export missing"); + codexCtor = mod.Codex; + } catch (error) { + const detail = sanitizeErrorMessage(stringifyError(error)); + throw new HarnessAdapterError( + `Codex SDK harness requires ${CODEX_SDK_PACKAGE}. Install it in the consuming workspace.${detail ? ` ${detail}` : ""}`, + { cause: error }, + ); + } + // Construction failures are auth/config problems, not missing installs. + try { + return new codexCtor({ + ...(options.env && { env: options.env }), + ...(options.codexPathOverride && { codexPathOverride: options.codexPathOverride }), + ...(options.baseUrl && { baseUrl: options.baseUrl }), + ...(options.apiKey && { apiKey: options.apiKey }), + config: { + show_raw_agent_reasoning: options.rawReasoning === true, + ...options.extraConfig, + }, + }); + } catch (error) { + throw new HarnessAdapterError( + `Failed to initialize the Codex SDK: ${sanitizeErrorMessage(stringifyError(error))}`, + { cause: error }, + ); + } +} + +export function normalizeCodexModel(model: string): string { + if (model === "codex/default") return "gpt-5.4-mini"; + return model.includes("/") ? model.slice(model.indexOf("/") + 1) : model; +} + +export function validateCodexSandboxMode( + value: unknown, +): "read-only" | "workspace-write" | "danger-full-access" { + if (value === "read-only" || value === "workspace-write" || value === "danger-full-access") { + return value; + } + return "workspace-write"; +} + +export function validateCodexApprovalPolicy( + value: unknown, +): "never" | "on-request" | "on-failure" | "untrusted" { + if ( + value === "never" || + value === "on-request" || + value === "on-failure" || + value === "untrusted" + ) { + return value; + } + return "never"; +} + +export async function runCodexSession(input: { + prompt: string; + model: string; + sdk?: CodexSdk; + signal?: AbortSignal; + logger: HarnessLogger; + thread: CodexThreadConfig; + outputSchema?: Record; + maxToolSteps?: number; + onToolStep?: () => void | Promise; +}): Promise { + const sdk = input.sdk ?? (await loadCodexSdk()); + const events: CodexEvent[] = []; + let finalMessage = ""; + let stopReason: string | undefined; + let iterationError: unknown; + let tokenUsage = emptyTokenUsage(); + const maxToolSteps = positiveInteger(input.maxToolSteps, 50); + const budgetController = new AbortController(); + const forwardAbort = () => budgetController.abort(input.signal?.reason); + if (input.signal) { + if (input.signal.aborted) budgetController.abort(input.signal.reason); + else input.signal.addEventListener("abort", forwardAbort, { once: true }); + } + let toolStepCount = 0; + + try { + const thread = sdk.startThread({ + ...(input.model && { model: normalizeCodexModel(input.model) }), + ...(input.thread.workingDirectory && { + workingDirectory: input.thread.workingDirectory, + }), + sandboxMode: validateCodexSandboxMode(input.thread.sandboxMode), + approvalPolicy: validateCodexApprovalPolicy(input.thread.approvalPolicy), + networkAccessEnabled: input.thread.networkAccessEnabled ?? true, + webSearchMode: input.thread.webSearchMode ?? "disabled", + skipGitRepoCheck: input.thread.skipGitRepoCheck ?? true, + }); + const streamed = await thread.runStreamed(input.prompt, { + ...(input.outputSchema && { outputSchema: input.outputSchema }), + signal: budgetController.signal, + }); + + for await (const event of streamed.events) { + events.push(event); + logCodexEvent(input.logger, event); + if (event.type === "turn.completed" && isRecord(event.usage)) { + tokenUsage = extractCodexTokenUsage(event.usage); + } else if (event.type === "turn.failed") { + stopReason = readCodexErrorMessage(event.error); + } else if (event.type === "error") { + stopReason = typeof event.message === "string" ? event.message : "error"; + } + + const item = isRecord(event.item) ? event.item : undefined; + if ( + event.type === "item.completed" && + item?.type === "agent_message" && + typeof item.text === "string" + ) { + finalMessage = item.text; + } + if ( + event.type === "item.completed" && + (item?.type === "command_execution" || item?.type === "mcp_tool_call") + ) { + toolStepCount += 1; + if (toolStepCount >= maxToolSteps && !budgetController.signal.aborted) { + stopReason = `tool step budget exhausted (${maxToolSteps} steps)`; + budgetController.abort(new Error(stopReason)); + } + if (item.type === "mcp_tool_call") await input.onToolStep?.(); + } + } + } catch (error) { + iterationError = error; + input.logger.warn({ + category: "codex", + message: `Codex stopped before a normal result: ${sanitizeErrorMessage(stringifyError(error))}`, + level: 0, + auxiliary: { error: { value: sanitizeErrorMessage(stringifyError(error)), type: "string" } }, + }); + } finally { + input.signal?.removeEventListener("abort", forwardAbort); + } + + return { + events, + finalMessage, + status: resolveCodexStatus(iterationError, stopReason), + ...(stopReason && { stopReason: sanitizeErrorMessage(stopReason) }), + tokenUsage, + ...(iterationError !== undefined && { iterationError }), + }; +} + +export function resolveCodexStatus( + iterationError: unknown, + stopReason?: string, +): "completed" | "sdk_error" { + return iterationError || stopReason ? "sdk_error" : "completed"; +} + +export function extractCodexTokenUsage( + usage: Record | undefined, +): CodexTokenUsage { + return { + input_tokens: toFiniteNumber(usage?.input_tokens), + cached_input_tokens: toFiniteNumber(usage?.cached_input_tokens), + output_tokens: toFiniteNumber(usage?.output_tokens), + reasoning_output_tokens: toFiniteNumber(usage?.reasoning_output_tokens), + }; +} + +function emptyTokenUsage(): CodexTokenUsage { + return extractCodexTokenUsage(undefined); +} + +export function buildCodexTranscript(events: CodexEvent[]): string { + return events + .map((event) => summarizeCodexEvent(event).detail) + .filter((detail): detail is string => Boolean(detail)) + .join("\n"); +} + +export function logCodexEvent(logger: HarnessLogger, event: CodexEvent): void { + const summary = summarizeCodexEvent(event); + logger.log({ + category: "codex", + message: summary.message, + level: 1, + auxiliary: { + type: { value: String(event.type ?? "unknown"), type: "string" }, + ...(summary.detail && { detail: { value: summary.detail, type: "string" } }), + }, + }); +} + +export function summarizeCodexEvent(event: CodexEvent): { message: string; detail?: string } { + const type = String(event.type ?? "unknown"); + const item = isRecord(event.item) ? event.item : undefined; + if (item?.type === "agent_message" && typeof item.text === "string") { + return { message: `agent: ${clip(item.text, 500)}`, detail: item.text }; + } + if (item?.type === "command_execution") { + return { + message: `command: ${String(item.command ?? "")} ${String(item.status ?? "")}`.trim(), + detail: safeJson(item), + }; + } + if (item?.type === "mcp_tool_call") { + return { + message: + `mcp: ${String(item.server ?? "")}.${String(item.tool ?? "")} ${String(item.status ?? "")}`.trim(), + detail: safeJson(item), + }; + } + if (item?.type === "error" && typeof item.message === "string") { + return { message: `error item: ${clip(item.message, 500)}`, detail: item.message }; + } + if (type === "turn.completed") + return { message: "turn completed", detail: safeJson(event.usage) }; + if (type === "turn.failed") { + const message = readCodexErrorMessage(event.error) ?? "turn failed"; + return { message: `turn failed: ${clip(message, 500)}`, detail: message }; + } + if (type === "error" && typeof event.message === "string") { + return { message: `error: ${clip(event.message, 500)}`, detail: event.message }; + } + return { message: `${type} event`, detail: safeJson(event) }; +} + +export function readCodexErrorMessage(value: unknown): string | undefined { + if (!value) return undefined; + if (typeof value === "string") return value; + if (isRecord(value) && typeof value.message === "string") return value.message; + return safeJson(value); +} + +export function toFiniteNumber(value: unknown): number { + const parsed = + typeof value === "number" + ? value + : typeof value === "string" && value.trim() + ? Number(value) + : 0; + return Number.isFinite(parsed) ? parsed : 0; +} + +function positiveInteger(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.max(1, Math.floor(value)) + : fallback; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +export function safeJson(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +export function stringifyError(value: unknown): string { + if (!value) return ""; + if (value instanceof Error) return value.message; + if (typeof value === "string") return value; + return safeJson(value) ?? String(value); +} + +export function clip(value: string, maxLength: number): string { + return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; +} diff --git a/packages/integrations/codex-sdk/tests/session.test.ts b/packages/integrations/codex-sdk/tests/session.test.ts new file mode 100644 index 000000000..343d84f41 --- /dev/null +++ b/packages/integrations/codex-sdk/tests/session.test.ts @@ -0,0 +1,109 @@ +/* eslint-disable require-yield */ +import { describe, expect, it } from "vitest"; +import { normalizeCodexModel, runCodexSession, type CodexSdk } from "../src/index.js"; + +const logger = { log: () => {}, warn: () => {}, error: () => {} }; + +describe("Codex SDK session", () => { + it("normalizes provider-prefixed and default models", () => { + expect(normalizeCodexModel("openai/gpt-5.4-mini")).toBe("gpt-5.4-mini"); + expect(normalizeCodexModel("gpt-5.4")).toBe("gpt-5.4"); + expect(normalizeCodexModel("codex/default")).toBe("gpt-5.4-mini"); + }); + + it("forwards thread options and collects messages and usage", async () => { + let threadOptions: Record | undefined; + let turnOptions: Record | undefined; + const sdk: CodexSdk = { + startThread: (options) => { + threadOptions = options; + return { + runStreamed: async (_prompt, options) => { + turnOptions = options; + return { + events: (async function* () { + yield { + type: "item.completed", + item: { type: "agent_message", text: "complete" }, + }; + yield { + type: "turn.completed", + usage: { input_tokens: 10, output_tokens: 4 }, + }; + })(), + }; + }, + }; + }, + }; + const outputSchema = { type: "object" }; + const result = await runCodexSession({ + prompt: "task", + model: "openai/gpt-5.4-mini", + logger, + sdk, + thread: { workingDirectory: "/tmp/work" }, + outputSchema, + }); + + expect(threadOptions).toMatchObject({ + model: "gpt-5.4-mini", + workingDirectory: "/tmp/work", + sandboxMode: "workspace-write", + approvalPolicy: "never", + networkAccessEnabled: true, + webSearchMode: "disabled", + skipGitRepoCheck: true, + }); + expect(turnOptions?.outputSchema).toBe(outputSchema); + expect(result.finalMessage).toBe("complete"); + expect(result.status).toBe("completed"); + expect(result.tokenUsage).toMatchObject({ input_tokens: 10, output_tokens: 4 }); + }); + + it("reports SDK errors instead of throwing", async () => { + const sdk: CodexSdk = { + startThread: () => ({ + runStreamed: async () => { + throw new Error("codex failed"); + }, + }), + }; + const result = await runCodexSession({ + prompt: "task", + model: "gpt-5.4-mini", + logger, + sdk, + thread: {}, + }); + expect(result.status).toBe("sdk_error"); + expect(String(result.iterationError)).toContain("codex failed"); + }); + + it("aborts when the tool-step budget is exhausted", async () => { + let signal: AbortSignal | undefined; + const sdk: CodexSdk = { + startThread: () => ({ + runStreamed: async (_prompt, options) => { + signal = options?.signal as AbortSignal; + return { + events: (async function* () { + yield { type: "item.completed", item: { type: "command_execution" } }; + })(), + }; + }, + }), + }; + const result = await runCodexSession({ + prompt: "task", + model: "gpt-5.4-mini", + logger, + sdk, + thread: {}, + maxToolSteps: 1, + }); + expect(signal?.aborted).toBe(true); + expect(result.status).toBe("sdk_error"); + expect(result.stopReason).toBe("tool step budget exhausted (1 steps)"); + }); +}); diff --git a/packages/integrations/codex-sdk/tsconfig.json b/packages/integrations/codex-sdk/tsconfig.json new file mode 100644 index 000000000..6bddd941a --- /dev/null +++ b/packages/integrations/codex-sdk/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts", "vitest.config.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/packages/integrations/codex-sdk/tsdown.config.ts b/packages/integrations/codex-sdk/tsdown.config.ts new file mode 100644 index 000000000..3b3b7c3bf --- /dev/null +++ b/packages/integrations/codex-sdk/tsdown.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsdown"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + }, + format: ["esm"], + platform: "node", + target: "node22", + dts: { + sourcemap: true, + }, + sourcemap: true, + outDir: "dist", +}); diff --git a/packages/integrations/codex/package.json b/packages/integrations/codex/package.json index 1fb10b3e4..de43a74fd 100644 --- a/packages/integrations/codex/package.json +++ b/packages/integrations/codex/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@browserbasehq/stagehand-integrations": "workspace:*", - "@openai/codex-sdk": "catalog:" + "@browserbasehq/stagehand-integrations-codex-sdk": "workspace:*" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/integrations/codex/src/agent.ts b/packages/integrations/codex/src/agent.ts index fdb29a9f1..26f71e991 100644 --- a/packages/integrations/codex/src/agent.ts +++ b/packages/integrations/codex/src/agent.ts @@ -1,4 +1,4 @@ -import { Codex, type CodexOptions } from "@openai/codex-sdk"; +import { loadCodexSdk, runCodexSession } from "@browserbasehq/stagehand-integrations-codex-sdk"; import { buildAllowlistedEnv } from "@browserbasehq/stagehand-integrations/harness"; import { fileURLToPath } from "node:url"; @@ -11,7 +11,7 @@ const serverPath = fileURLToPath( * the config override (config.toml shape), the same mechanism the evals codex * harness uses. */ -export function buildCodexConfig(): NonNullable { +export function buildCodexConfig(): Record { return { // Required for headless MCP calls on machines without a global // approvals_reviewer: without it, tool calls die with "user cancelled @@ -30,12 +30,18 @@ export function buildCodexConfig(): NonNullable { }; } +const logger = { + log: () => {}, + warn: () => {}, + error: () => {}, +}; + async function main(): Promise { const args = process.argv.slice(2); const instruction = (args[0] === "--" ? args.slice(1) : args).join(" ").trim(); if (!instruction) throw new Error('Usage: pnpm start "your instruction"'); - const codex = new Codex({ + const sdk = await loadCodexSdk({ ...(process.env.OPENAI_API_KEY ? { apiKey: process.env.OPENAI_API_KEY } : {}), // pnpm can skip the SDK's vendored-binary postinstall; point at a locally // installed codex when that happens (same escape hatch the evals harness @@ -43,38 +49,33 @@ async function main(): Promise { ...(process.env.CODEX_PATH_OVERRIDE ? { codexPathOverride: process.env.CODEX_PATH_OVERRIDE } : {}), - config: buildCodexConfig(), + extraConfig: buildCodexConfig(), }); - const thread = codex.startThread({ + const result = await runCodexSession({ + prompt: instruction, // Codex picks its own harness-tuned default model; override only via env. - ...(process.env.CODEX_STAGEHAND_MODEL ? { model: process.env.CODEX_STAGEHAND_MODEL } : {}), - // The browser work happens in the MCP server; the local sandbox can stay - // read-only. - sandboxMode: "read-only", - // Headless policy chosen empirically: "on-failure" lets MCP tool calls - // complete; "never" and "untrusted" auto-cancel them ("user cancelled - // MCP tool call"). - approvalPolicy: "on-failure", - skipGitRepoCheck: true, + model: process.env.CODEX_STAGEHAND_MODEL ?? "", + sdk, + logger, + thread: { + // The browser work happens in the MCP server; the local sandbox can stay + // read-only. + sandboxMode: "read-only", + // Headless policy chosen empirically: "on-failure" lets MCP tool calls + // complete; "never" and "untrusted" auto-cancel them ("user cancelled + // MCP tool call"). + approvalPolicy: "on-failure", + skipGitRepoCheck: true, + }, }); - - const streamed = await thread.runStreamed(instruction); - let finalResponse = ""; - for await (const event of streamed.events) { - if ( - event.type === "item.completed" && - typeof event.item === "object" && - event.item !== null && - "type" in event.item && - event.item.type === "agent_message" && - "text" in event.item && - typeof event.item.text === "string" - ) { - finalResponse = event.item.text; - } + if (result.status !== "completed") { + throw ( + result.iterationError ?? + new Error(`Agent did not finish: ${result.stopReason ?? result.status}`) + ); } // oxlint-disable-next-line no-console -- CLI example prints the agent result. - console.log(finalResponse); + console.log(result.finalMessage); } if (import.meta.main) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e05bbd61..36e93abc6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -266,9 +266,9 @@ importers: '@browserbasehq/stagehand-integrations-claude-agent-sdk': specifier: workspace:* version: link:../integrations/claude-agent-sdk - '@openai/codex-sdk': - specifier: 'catalog:' - version: 0.147.0 + '@browserbasehq/stagehand-integrations-codex-sdk': + specifier: workspace:* + version: link:../integrations/codex-sdk ai: specifier: ^5.0.133 version: 5.0.220(zod@4.4.3) @@ -440,6 +440,25 @@ importers: version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) packages/integrations/codex: + dependencies: + '@browserbasehq/stagehand-integrations': + specifier: workspace:* + version: link:../core + '@browserbasehq/stagehand-integrations-codex-sdk': + specifier: workspace:* + version: link:../codex-sdk + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.13.2 + typescript: + specifier: 'catalog:' + version: 5.9.3 + vitest: + specifier: 'catalog:' + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@24.13.2)(vite@8.1.3(@types/node@24.13.2)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1)(yaml@2.9.0)) + + packages/integrations/codex-sdk: dependencies: '@browserbasehq/stagehand-integrations': specifier: workspace:* @@ -451,6 +470,9 @@ importers: '@types/node': specifier: 'catalog:' version: 24.13.2 + tsdown: + specifier: 'catalog:' + version: 0.22.3(publint@0.3.21)(tsx@4.23.1)(typescript@5.9.3) typescript: specifier: 'catalog:' version: 5.9.3 diff --git a/turbo.json b/turbo.json index 41f695758..26e43f58c 100644 --- a/turbo.json +++ b/turbo.json @@ -36,6 +36,11 @@ "inputs": ["$TURBO_DEFAULT$", "!dist/**"], "outputs": ["dist/**"] }, + "@browserbasehq/stagehand-integrations-codex-sdk#build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", "!dist/**"], + "outputs": ["dist/**"] + }, "@browserbasehq/stagehand-evals#build": { "dependsOn": ["^build"], "inputs": ["$TURBO_DEFAULT$", "!dist/**"], @@ -82,6 +87,9 @@ "@browserbasehq/stagehand-integrations-claude-agent-sdk#typecheck": { "dependsOn": ["^build"] }, + "@browserbasehq/stagehand-integrations-codex-sdk#typecheck": { + "dependsOn": ["^build"] + }, "@browserbasehq/stagehand-integrations#test:unit": { "dependsOn": ["^build", "@browserbasehq/stagehand-integrations#build"], "inputs": [ @@ -102,6 +110,16 @@ "$TURBO_ROOT$/vitest.config.ts" ] }, + "@browserbasehq/stagehand-integrations-codex-sdk#test:unit": { + "dependsOn": ["^build", "@browserbasehq/stagehand-integrations-codex-sdk#build"], + "inputs": [ + "$TURBO_DEFAULT$", + "tests/**", + "src/**", + "**/*.test.ts", + "$TURBO_ROOT$/vitest.config.ts" + ] + }, "@browserbasehq/stagehand-integrations-example-mastra-facade#typecheck": { "dependsOn": ["^build"] }, diff --git a/vitest.config.ts b/vitest.config.ts index 943eaf3fd..c4452e5f5 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ export default defineConfig({ "packages/evals/tests/**/*.test.ts", "packages/integrations/core/tests/**/*.test.ts", "packages/integrations/claude-agent-sdk/tests/**/*.test.ts", + "packages/integrations/codex-sdk/tests/**/*.test.ts", "packages/extension/tests/**/*.test.ts", "packages/sdk-ts/tests/**/*.test.ts", "packages/extension/understudy/**/*.test.ts",