diff --git a/.changeset/openai-computer-call-loop.md b/.changeset/openai-computer-call-loop.md new file mode 100644 index 0000000000..9232a504f1 --- /dev/null +++ b/.changeset/openai-computer-call-loop.md @@ -0,0 +1,5 @@ +--- +'@tanstack/openai-base': patch +--- + +The computer use loop of `computerUseTool()` runs. A `computer_call` from the model is now a tool call to `computer_use_preview`, with the `action` and the `pending_safety_checks` as its input. The tool result goes back as `computer_call_output`, with the first image of the result as the screenshot. When the call does not fail, the output acknowledges the safety checks of the call. diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 2812e181cd..1c48a41df5 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -996,25 +996,63 @@ Lets the model observe a virtual desktop via screenshots and interact with it using keyboard and mouse events. Provide the display dimensions and the execution environment type. +The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this: + ```typescript import { chat } from "@tanstack/ai"; import { openaiText } from "@tanstack/ai-openai"; import { computerUseTool } from "@tanstack/ai-openai/tools"; +interface SafetyCheck { + id: string; + code?: string; + message?: string; +} + +// Your code: run one action on the desktop, then return a PNG of the screen as base64. +declare function runAction(action: unknown): Promise; +// Your code: show the checks to a person and return their answer. +declare function askPerson(checks: Array): Promise; + const stream = chat({ adapter: openaiText("computer-use-preview"), messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], tools: [ - computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), + { + ...computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", + }), + execute: async (input: { + action: unknown; + pending_safety_checks: Array; + }) => { + const checks = input.pending_safety_checks; + if (checks.length > 0 && !(await askPerson(checks))) { + throw new Error("A person did not allow this action."); + } + const screenshot = await runAction(input.action); + return [ + { + type: "image", + source: { type: "data", value: screenshot, mimeType: "image/png" }, + }, + ]; + }, + }, ], }); ``` +The input of each call has two fields: + +- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`. +- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action. + +The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. + **Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/tools/provider-tools.md b/docs/tools/provider-tools.md index 25da0275c4..addf9bf719 100644 --- a/docs/tools/provider-tools.md +++ b/docs/tools/provider-tools.md @@ -90,7 +90,7 @@ const followUp = chat({ The search/fetch call surfaces as a provider-executed `tool-call` part on the assistant message; the agent loop never tries to run it client-side. -OpenAI `applyPatchTool`, `localShellTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool). +OpenAI `applyPatchTool`, `localShellTool`, `computerUseTool`, and `shellTool({ environment: { type: "local" } })` are different. The model returns the call, and your app runs it. See [OpenAI adapter](../adapters/openai.md#applypatchtool) and [`computerUseTool`](../adapters/openai.md#computerusetool). ### Read web search sources diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 8803b2fb0a..fbaf2720e0 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -51,6 +51,7 @@ import type { } from '@tanstack/ai/adapters' import type { Response, + ResponseComputerToolCallOutputScreenshot, ResponseCreateParams, ResponseFunctionCallOutputItem, ResponseFunctionWebSearch, @@ -400,7 +401,10 @@ export interface OpenAIResponsesToolCallMetadata extends ProviderExecutedToolMet * `additional_tools` has one, and the API needs it on the replayed call. */ namespace?: string - /** Set for shell, local_shell, and apply_patch calls the app must run. */ + /** + * Set for shell, local_shell, apply_patch, and computer calls the app must + * run. + */ openaiUserTool?: OpenAIUserToolName /** Shell `action.max_output_length`, echoed on `shell_call_output`. */ maxOutputLength?: number | null @@ -2913,7 +2917,10 @@ export abstract class OpenAIBaseResponsesTextAdapter< ? userToolCalls.get(message.toolCallId) : undefined const userOutput = owner - ? userToolResultItem(owner, message.content) + ? userToolResultItem(owner, message.content, { + screenshot: () => this.screenshotOf(message.content), + failed: message.error !== undefined, + }) : null if (userOutput) { result.push(userOutput) @@ -3149,6 +3156,22 @@ export abstract class OpenAIBaseResponsesTextAdapter< return result } + /** The first image of a tool result, as a computer screenshot. */ + private screenshotOf( + content: ModelMessage['content'], + ): ResponseComputerToolCallOutputScreenshot { + const image = Array.isArray(content) + ? content.find((part) => part.type === 'image') + : undefined + const input = image ? this.convertContentPartToInput(image) : undefined + if (input?.type !== 'input_image') return { type: 'computer_screenshot' } + return { + type: 'computer_screenshot', + ...(input.image_url ? { image_url: input.image_url } : {}), + ...(input.file_id ? { file_id: input.file_id } : {}), + } + } + /** * Converts a ContentPart to Responses API input content item. * Handles text, image, audio, and document (PDF) content parts. diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index b357ec146e..94e8294e97 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -1,11 +1,20 @@ -import type { ResponseInputItem } from 'openai/resources/responses/responses' +import type { + ResponseComputerToolCall, + ResponseComputerToolCallOutputScreenshot, + ResponseInputItem, +} from 'openai/resources/responses/responses' /** * OpenAI Responses tools the app must run. * A container `shell` runs on the provider. `apply_patch`, `local_shell`, - * and `shell` with `environment.type: "local"` (or no environment) do not. + * `computer_use_preview`, and `shell` with `environment.type: "local"` (or + * no environment) do not. */ -export type OpenAIUserToolName = 'shell' | 'apply_patch' | 'local_shell' +export type OpenAIUserToolName = + | 'shell' + | 'apply_patch' + | 'local_shell' + | 'computer_use_preview' export interface OpenAIUserExecutedCall { name: OpenAIUserToolName @@ -71,12 +80,47 @@ export function hostedShellCallIds( export function readUserToolName(metadata: unknown): OpenAIUserToolName | null { if (!isRecord(metadata)) return null const name = metadata.openaiUserTool - if (name === 'shell' || name === 'apply_patch' || name === 'local_shell') { + if ( + name === 'shell' || + name === 'apply_patch' || + name === 'local_shell' || + name === 'computer_use_preview' + ) { return name } return null } +/** + * A computer action. It goes back to the API as the model sent it, so only + * its `type` is checked. + */ +function isComputerAction( + value: unknown, +): value is NonNullable { + return isRecord(value) && typeof value.type === 'string' +} + +/** The safety checks of a computer call, with only their known fields. */ +function safetyChecks( + value: unknown, +): Array { + if (!Array.isArray(value)) return [] + return value.flatMap((check) => + isRecord(check) && typeof check.id === 'string' + ? [ + { + id: check.id, + ...(typeof check.code === 'string' ? { code: check.code } : {}), + ...(typeof check.message === 'string' + ? { message: check.message } + : {}), + }, + ] + : [], + ) +} + function readItemId(metadata: unknown): string | undefined { if (!isRecord(metadata)) return undefined return typeof metadata.itemId === 'string' && metadata.itemId.length > 0 @@ -135,6 +179,19 @@ export function readUserExecutedCall( } } + if (item.type === 'computer_call') { + if (!callId || !isComputerAction(item.action)) return null + return { + name: 'computer_use_preview', + callId, + ...(itemId ? { itemId } : {}), + input: { + action: item.action, + pending_safety_checks: safetyChecks(item.pending_safety_checks), + }, + } + } + if (item.type === 'local_shell_call') { if (!isRecord(item.action)) return null const command = stringList(item.action.command) @@ -234,6 +291,18 @@ export function userToolRequestItem( } } + if (name === 'computer_use_preview') { + if (!isComputerAction(args.action)) return null + return { + type: 'computer_call', + id: itemId ?? toolCall.id, + call_id: toolCall.id, + status: 'completed', + action: args.action, + pending_safety_checks: safetyChecks(args.pending_safety_checks), + } + } + const commands = stringList(args.commands) if (!commands) return null return { @@ -294,13 +363,38 @@ function shellOutputList(content: unknown): Array<{ return [shellEntry(JSON.stringify(parsed ?? ''))] } -/** Replay the app's tool result as the matching Responses output item. */ +/** + * Replay the app's tool result as the matching Responses output item. + * `computer` is what a `computer_use_preview` result needs: the screenshot, + * and whether the call failed. Only a computer result reads the screenshot. + * The output of a call that did not fail acknowledges the safety checks of + * the call. + */ export function userToolResultItem( toolCall: ToolCallLike, content: unknown, + computer?: { + screenshot: () => ResponseComputerToolCallOutputScreenshot + failed: boolean + }, ): ResponseInputItem | null { const name = readUserToolName(toolCall.metadata) if (!name) return null + + if (name === 'computer_use_preview') { + const checks = computer?.failed + ? [] + : safetyChecks( + parseArguments(toolCall.function.arguments).pending_safety_checks, + ) + return { + type: 'computer_call_output', + call_id: toolCall.id, + output: computer?.screenshot() ?? { type: 'computer_screenshot' }, + ...(checks.length > 0 ? { acknowledged_safety_checks: checks } : {}), + } + } + const parsed = parseContent(content) if (name === 'apply_patch') { diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts new file mode 100644 index 0000000000..b2566dab58 --- /dev/null +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' +import { computerUseTool } from '../src/tools/computer-use-tool' +import type OpenAI from 'openai' +import type { StreamChunk } from '@tanstack/ai' + +const safetyCheck = { + id: 'sc_1', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +async function* completed(output: Array>) { + yield* [ + { + type: 'response.created', + response: { + id: 'resp_1', + model: 'computer-use-preview', + status: 'in_progress', + }, + }, + { + type: 'response.completed', + response: { + id: 'resp_1', + model: 'computer-use-preview', + status: 'completed', + output, + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + }, + ] +} + +const clickCall = { + type: 'computer_call', + id: 'cu_1', + call_id: 'call_cu', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], +} + +const answer = { + type: 'message', + id: 'msg_1', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +class TestAdapter extends OpenAIBaseResponsesTextAdapter {} + +class FileAdapter extends TestAdapter { + override readonly supportsFileSources = true +} + +async function runLoop(execute: (input: unknown) => Promise) { + const create = vi + .fn() + .mockResolvedValueOnce(completed([clickCall])) + .mockResolvedValueOnce(completed([answer])) + const client = { + responses: { create: (params: unknown) => create(params) }, + } as unknown as OpenAI + const adapter = new TestAdapter('computer-use-preview', 'openai', client) + const tool = { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute: vi.fn(execute), + } + const chunks: Array = [] + for await (const chunk of chat({ + adapter, + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [tool], + })) { + chunks.push(chunk) + } + return { create, execute: tool.execute, chunks } +} + +describe('computer use loop', () => { + it('runs the computer call and sends the screenshot back', async () => { + const { create, execute } = await runLoop(async () => [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, + ]) + + expect(execute).toHaveBeenCalledOnce() + expect(execute.mock.calls[0]?.[0]).toEqual({ + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }) + expect(create).toHaveBeenCalledTimes(2) + expect(create.mock.calls[1]?.[0].input).toEqual( + expect.arrayContaining([ + { + type: 'computer_call', + id: 'cu_1', + call_id: 'call_cu', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }, + { + type: 'computer_call_output', + call_id: 'call_cu', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + acknowledged_safety_checks: [safetyCheck], + }, + ]), + ) + }) + + it('reads no screenshot from the result of another app tool', async () => { + const create = vi.fn().mockResolvedValueOnce(completed([answer])) + const client = { + responses: { create: (params: unknown) => create(params) }, + } as unknown as OpenAI + // OpenAI takes file sources, but not a file from another provider, so a + // screenshot read of this result would throw. + const adapter = new FileAdapter('gpt-5.5', 'openai', client) + for await (const _ of chat({ + adapter, + messages: [ + { + role: 'assistant', + content: null, + toolCalls: [ + { + id: 'call_sh', + type: 'function', + function: { + name: 'local_shell', + arguments: JSON.stringify({ command: ['ls'] }), + }, + metadata: { openaiUserTool: 'local_shell', itemId: 'ls_1' }, + }, + ], + }, + { + role: 'tool', + toolCallId: 'call_sh', + content: [ + { + type: 'image', + source: { type: 'file', value: 'file_1', provider: 'gemini' }, + }, + ], + }, + ], + })) { + // Drain the stream. + } + + expect(create.mock.calls[0]?.[0].input).toContainEqual( + expect.objectContaining({ type: 'local_shell_call_output' }), + ) + }) + + it('does not acknowledge the safety checks of a failed call', async () => { + const { create } = await runLoop(async () => { + throw new Error('A person did not allow the click.') + }) + + const input: Array> = + create.mock.calls[1]?.[0].input + const output = input.find((item) => item.type === 'computer_call_output') + expect(output).toEqual({ + type: 'computer_call_output', + call_id: 'call_cu', + output: { type: 'computer_screenshot' }, + }) + }) +}) diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 98d5716a87..c28f175ac5 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -130,6 +130,7 @@ import { Route as ApiMoonshotUsageDetailsRouteImport } from './routes/api.moonsh import { Route as ApiMultimodalToolResultWireRouteImport } from './routes/api.multimodal-tool-result-wire' import { Route as ApiNonStreamingRunErrorRouteImport } from './routes/api.non-streaming-run-error' import { Route as ApiOpenaiCompletedResponseTextRouteImport } from './routes/api.openai-completed-response-text' +import { Route as ApiOpenaiComputerUseWireRouteImport } from './routes/api.openai-computer-use-wire' import { Route as ApiOpenaiImage25ModelsRouteImport } from './routes/api.openai-image-2-5-models' import { Route as ApiOpenaiMalformedToolArgumentsRouteImport } from './routes/api.openai-malformed-tool-arguments' import { Route as ApiOpenaiResponsesUsageRouteImport } from './routes/api.openai-responses-usage' @@ -815,6 +816,12 @@ const ApiOpenaiCompletedResponseTextRoute = path: '/api/openai-completed-response-text', getParentRoute: () => rootRouteImport, } as any) +const ApiOpenaiComputerUseWireRoute = + ApiOpenaiComputerUseWireRouteImport.update({ + id: '/api/openai-computer-use-wire', + path: '/api/openai-computer-use-wire', + getParentRoute: () => rootRouteImport, + } as any) const ApiOpenaiImage25ModelsRoute = ApiOpenaiImage25ModelsRouteImport.update({ id: '/api/openai-image-2-5-models', path: '/api/openai-image-2-5-models', @@ -1219,6 +1226,7 @@ export interface FileRoutesByFullPath { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1395,6 +1403,7 @@ export interface FileRoutesByTo { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1572,6 +1581,7 @@ export interface FileRoutesById { '/api/multimodal-tool-result-wire': typeof ApiMultimodalToolResultWireRoute '/api/non-streaming-run-error': typeof ApiNonStreamingRunErrorRoute '/api/openai-completed-response-text': typeof ApiOpenaiCompletedResponseTextRoute + '/api/openai-computer-use-wire': typeof ApiOpenaiComputerUseWireRoute '/api/openai-image-2-5-models': typeof ApiOpenaiImage25ModelsRoute '/api/openai-malformed-tool-arguments': typeof ApiOpenaiMalformedToolArgumentsRoute '/api/openai-responses-usage': typeof ApiOpenaiResponsesUsageRoute @@ -1750,6 +1760,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -1926,6 +1937,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -2102,6 +2114,7 @@ export interface FileRouteTypes { | '/api/multimodal-tool-result-wire' | '/api/non-streaming-run-error' | '/api/openai-completed-response-text' + | '/api/openai-computer-use-wire' | '/api/openai-image-2-5-models' | '/api/openai-malformed-tool-arguments' | '/api/openai-responses-usage' @@ -2279,6 +2292,7 @@ export interface RootRouteChildren { ApiMultimodalToolResultWireRoute: typeof ApiMultimodalToolResultWireRoute ApiNonStreamingRunErrorRoute: typeof ApiNonStreamingRunErrorRoute ApiOpenaiCompletedResponseTextRoute: typeof ApiOpenaiCompletedResponseTextRoute + ApiOpenaiComputerUseWireRoute: typeof ApiOpenaiComputerUseWireRoute ApiOpenaiImage25ModelsRoute: typeof ApiOpenaiImage25ModelsRoute ApiOpenaiMalformedToolArgumentsRoute: typeof ApiOpenaiMalformedToolArgumentsRoute ApiOpenaiResponsesUsageRoute: typeof ApiOpenaiResponsesUsageRoute @@ -3179,6 +3193,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiOpenaiCompletedResponseTextRouteImport parentRoute: typeof rootRouteImport } + '/api/openai-computer-use-wire': { + id: '/api/openai-computer-use-wire' + path: '/api/openai-computer-use-wire' + fullPath: '/api/openai-computer-use-wire' + preLoaderRoute: typeof ApiOpenaiComputerUseWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/openai-image-2-5-models': { id: '/api/openai-image-2-5-models' path: '/api/openai-image-2-5-models' @@ -3737,6 +3758,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMultimodalToolResultWireRoute: ApiMultimodalToolResultWireRoute, ApiNonStreamingRunErrorRoute: ApiNonStreamingRunErrorRoute, ApiOpenaiCompletedResponseTextRoute: ApiOpenaiCompletedResponseTextRoute, + ApiOpenaiComputerUseWireRoute: ApiOpenaiComputerUseWireRoute, ApiOpenaiImage25ModelsRoute: ApiOpenaiImage25ModelsRoute, ApiOpenaiMalformedToolArgumentsRoute: ApiOpenaiMalformedToolArgumentsRoute, ApiOpenaiResponsesUsageRoute: ApiOpenaiResponsesUsageRoute, diff --git a/testing/e2e/src/routes/api.openai-computer-use-wire.ts b/testing/e2e/src/routes/api.openai-computer-use-wire.ts new file mode 100644 index 0000000000..8e4df804a1 --- /dev/null +++ b/testing/e2e/src/routes/api.openai-computer-use-wire.ts @@ -0,0 +1,135 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createOpenaiChat } from '@tanstack/ai-openai' +import { computerUseTool } from '@tanstack/ai-openai/tools' + +const DUMMY_KEY = 'sk-e2e-test-dummy-key' +const SCREENSHOT = 'iVBORw0KGgo=' + +const safetyCheck = { + id: 'sc_e2e', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', +} + +const computerCall = { + type: 'computer_call', + id: 'cu_e2e', + call_id: 'call_cu_e2e', + status: 'completed', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], +} + +const answer = { + type: 'message', + id: 'msg_cu_e2e', + role: 'assistant', + status: 'completed', + content: [{ type: 'output_text', text: 'Done.', annotations: [] }], +} + +function responseStream( + id: string, + output: Array, +): ReadableStream { + const encoder = new TextEncoder() + const events = [ + { + type: 'response.created', + response: { id, object: 'response', status: 'in_progress', output: [] }, + }, + { + type: 'response.completed', + response: { + id, + object: 'response', + status: 'completed', + output, + usage: { input_tokens: 5, output_tokens: 2, total_tokens: 7 }, + }, + }, + ] + return new ReadableStream({ + start(controller) { + for (const event of events) { + controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`)) + } + controller.enqueue(encoder.encode('data: [DONE]\n\n')) + controller.close() + }, + }) +} + +/** + * Drives the OpenAI Responses adapter through one computer use step. A + * custom `fetch` answers the first request with a `computer_call` and the + * second with text, and records each request body. aimock cannot send a + * `computer_call`, so this route serves the stream itself. + */ +export const Route = createFileRoute('/api/openai-computer-use-wire')({ + server: { + handlers: { + POST: async () => { + const requestBodies: Array = [] + const mockFetch: typeof fetch = async (input, init) => { + const req = + input instanceof Request ? input : new Request(input, init) + requestBodies.push(JSON.parse(await req.text())) + const stream = + requestBodies.length === 1 + ? responseStream('resp_cu_e2e_1', [computerCall]) + : responseStream('resp_cu_e2e_2', [answer]) + return new Response(stream, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + const adapter = createOpenaiChat('computer-use-preview', DUMMY_KEY, { + fetch: mockFetch, + }) + + const toolInputs: Array = [] + try { + for await (const _ of chat({ + ...createChatOptions({ adapter }), + messages: [{ role: 'user', content: 'Click the button.' }], + tools: [ + { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute: async (input: unknown) => { + toolInputs.push(input) + return [ + { + type: 'image' as const, + source: { + type: 'data' as const, + value: SCREENSHOT, + mimeType: 'image/png', + }, + }, + ] + }, + }, + ], + })) { + // Drain the stream. + } + } catch (error) { + return Response.json({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }) + } + + return Response.json({ ok: true, toolInputs, requestBodies }) + }, + }, + }, +}) diff --git a/testing/e2e/tests/openai-computer-use-wire.spec.ts b/testing/e2e/tests/openai-computer-use-wire.spec.ts new file mode 100644 index 0000000000..d0db59ee87 --- /dev/null +++ b/testing/e2e/tests/openai-computer-use-wire.spec.ts @@ -0,0 +1,59 @@ +import { test, expect } from './fixtures' + +/** + * Wire format of OpenAI computer use. The model sends a `computer_call`, the + * app runs it, and the next request sends the call back with its + * `computer_call_output`. + * + * `/api/openai-computer-use-wire` serves the model responses with a custom + * `fetch` and returns the tool inputs and the request bodies as JSON. + */ +test.describe('openai — computer use wire format', () => { + test('runs the computer call and sends the screenshot back', async ({ + request, + }) => { + const res = await request.post('/api/openai-computer-use-wire') + expect(res.ok()).toBe(true) + const { ok, error, toolInputs, requestBodies } = (await res.json()) as { + ok: boolean + error?: string + toolInputs: Array + requestBodies: Array<{ input: Array }> + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + const safetyCheck = { + id: 'sc_e2e', + code: 'malicious_instructions', + message: 'The page asks the model to do something else.', + } + expect(toolInputs).toEqual([ + { + action: { type: 'click', button: 'left', x: 10, y: 20 }, + pending_safety_checks: [safetyCheck], + }, + ]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[1]!.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_e2e', + action: { type: 'click', button: 'left', x: 10, y: 20 }, + }), + { + type: 'computer_call_output', + call_id: 'call_cu_e2e', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + acknowledged_safety_checks: [safetyCheck], + }, + ]), + ) + }) +})