diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md new file mode 100644 index 0000000000..40fbc8cab4 --- /dev/null +++ b/.changeset/openai-computer-tool.md @@ -0,0 +1,6 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. Computer screenshots now go out with `detail: 'original'`, as in OpenAI's computer use guide. diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 1c48a41df5..7a053ba05c 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -993,10 +993,9 @@ const stream = chat({ ### `computerUseTool` Lets the model observe a virtual desktop via screenshots and interact with -it using keyboard and mouse events. Provide the display dimensions and the -execution environment type. +it using keyboard and mouse events. -The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this: +The model asks for actions, for example a click and then some typing. Your app runs the actions, then returns a screenshot of the screen. Add an `execute` to the tool that does this: ```typescript import { chat } from "@tanstack/ai"; @@ -1009,31 +1008,31 @@ interface SafetyCheck { message?: string; } -// Your code: run one action on the desktop, then return a PNG of the screen as base64. -declare function runAction(action: unknown): Promise; +// Your code: run one action on the desktop. +declare function runAction(action: unknown): Promise; +// Your code: return a PNG of the screen as base64. +declare function takeScreenshot(): Promise; // Your code: show the checks to a person and return their answer. declare function askPerson(checks: Array): Promise; const stream = chat({ - adapter: openaiText("computer-use-preview"), - messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], + adapter: openaiText("gpt-6.1-sol"), + messages: [{ role: "user", content: "Open example.com and sign up for the newsletter" }], tools: [ { - ...computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), + ...computerUseTool({ type: "computer" }), execute: async (input: { - action: unknown; + actions: Array; pending_safety_checks: Array; }) => { const checks = input.pending_safety_checks; if (checks.length > 0 && !(await askPerson(checks))) { throw new Error("A person did not allow this action."); } - const screenshot = await runAction(input.action); + for (const action of input.actions) { + await runAction(action); + } + const screenshot = await takeScreenshot(); return [ { type: "image", @@ -1048,12 +1047,25 @@ const stream = chat({ The input of each call has two fields: -- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`. -- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action. +- `actions`: the actions to run, in order, as OpenAI sends them. For example `[{ type: "click", button: "left", x: 120, y: 340 }, { type: "type", text: "ada@example.com" }]`. +- `pending_safety_checks`: the safety checks of the actions. If the list is not empty, ask a person before you run the actions. The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. -**Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The `computer-use-preview` model takes the older tool type. Give it the display size and the environment. Each call then has one `action` field in place of `actions`: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const previewTool = computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", +}); +``` + +**Supported models:** `type: "computer"` works with the models that list computer use, for example `gpt-6.1-sol`. `type: "computer_use_preview"` works with `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..cddd848128 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1209,7 +1209,7 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-19" }, { "label": "Anthropic", diff --git a/packages/ai-openai/src/tools/computer-use-tool.ts b/packages/ai-openai/src/tools/computer-use-tool.ts index 3f65dddb7c..0ad3ecc555 100644 --- a/packages/ai-openai/src/tools/computer-use-tool.ts +++ b/packages/ai-openai/src/tools/computer-use-tool.ts @@ -9,11 +9,11 @@ export { } from '@tanstack/openai-base' // The brand discriminator (`computer_use`) intentionally differs from the -// runtime tool name (`computer_use_preview`). The brand matches the model-meta -// tool-capability union (`tools: ['computer_use', ...]`) used to gate which -// models can construct this tool at compile time, while the runtime name -// matches the OpenAI SDK's literal `'computer_use_preview'`. The base factory -// records the canonical `computer_use` kind separately in adapter metadata. +// runtime tool name (`computer` or `computer_use_preview`). The brand matches +// the model-meta tool-capability union (`tools: ['computer_use', ...]`) used to +// gate which models can construct this tool at compile time, while the runtime +// name matches the tool type in the OpenAI SDK. The base factory records the +// canonical `computer_use` kind separately in adapter metadata. export type OpenAIComputerUseTool = ProviderTool<'openai', 'computer_use'> /** diff --git a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts index ad9ae3205c..cc89d94901 100644 --- a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts @@ -107,6 +107,12 @@ describe('OpenAI per-model tool gating', () => { // OpenAI's GPT-6 model pages list web_search, file_search, image_generation, // code_interpreter, hosted_shell, apply_patch, computer_use and mcp for the // Responses API, and neither web_search_preview nor local_shell. + it('gpt-6.1-sol accepts the computer tool', () => { + typedTools(openaiText('gpt-6.1-sol'), [ + computerUseTool({ type: 'computer' }), + ]) + }) + it('gpt-6-astra accepts the Responses tools its model page lists but rejects the rest', () => { const adapter = openaiText('gpt-6-astra') typedTools(adapter, [ diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index fbaf2720e0..e8a34cd1e7 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1391,6 +1391,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< hasEmittedRunStarted: boolean }, ): AsyncIterable { + // A `computer_call` is a call to the computer tool of this request. + const computerTool = options.tools?.some( + (tool) => + tool.name === 'computer' && + getOpenAIProviderToolKind(tool) === 'computer_use', + ) + ? 'computer' + : 'computer_use_preview' const normalizeToolInput = createToolInputNormalizer( options.tools, (schema, required) => @@ -1619,7 +1627,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< outputIndex: number, bareShell: boolean, ): Array => { - const call = readUserExecutedCall(item, { bareShell }) + const call = readUserExecutedCall(item, { bareShell, computerTool }) if (!call || toolCallMetadata.get(call.callId)?.ended) return [] const tracked = toolCallMetadata.get(call.callId) ?? { callId: call.callId, @@ -3156,10 +3164,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< return result } - /** The first image of a tool result, as a computer screenshot. */ + /** + * The first image of a tool result, as a computer screenshot. It goes out + * with `detail: 'original'`, as in OpenAI's computer use guide, so the + * model sees the screen at full size. The SDK type has no `detail` yet. + */ private screenshotOf( content: ModelMessage['content'], - ): ResponseComputerToolCallOutputScreenshot { + ): ResponseComputerToolCallOutputScreenshot & { detail?: 'original' } { const image = Array.isArray(content) ? content.find((part) => part.type === 'image') : undefined @@ -3169,6 +3181,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< type: 'computer_screenshot', ...(input.image_url ? { image_url: input.image_url } : {}), ...(input.file_id ? { file_id: input.file_id } : {}), + detail: 'original', } } diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index 94e8294e97..d42ae57276 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -1,4 +1,5 @@ import type { + ComputerActionList, ResponseComputerToolCall, ResponseComputerToolCallOutputScreenshot, ResponseInputItem, @@ -7,14 +8,17 @@ import type { /** * OpenAI Responses tools the app must run. * A container `shell` runs on the provider. `apply_patch`, `local_shell`, - * `computer_use_preview`, and `shell` with `environment.type: "local"` (or - * no environment) do not. + * `computer`, `computer_use_preview`, and `shell` with + * `environment.type: "local"` (or no environment) do not. */ export type OpenAIUserToolName = | 'shell' | 'apply_patch' | 'local_shell' - | 'computer_use_preview' + | OpenAIComputerToolName + +/** The names of the computer tool: its two tool types. */ +export type OpenAIComputerToolName = 'computer' | 'computer_use_preview' export interface OpenAIUserExecutedCall { name: OpenAIUserToolName @@ -84,6 +88,7 @@ export function readUserToolName(metadata: unknown): OpenAIUserToolName | null { name === 'shell' || name === 'apply_patch' || name === 'local_shell' || + name === 'computer' || name === 'computer_use_preview' ) { return name @@ -101,6 +106,14 @@ function isComputerAction( return isRecord(value) && typeof value.type === 'string' } +/** The batched actions of a computer call. */ +function isComputerActionList(value: unknown): value is ComputerActionList { + return ( + Array.isArray(value) && + value.every((action) => isRecord(action) && typeof action.type === 'string') + ) +} + /** The safety checks of a computer call, with only their known fields. */ function safetyChecks( value: unknown, @@ -159,10 +172,12 @@ function applyPatchOperation( * `bareShell` is true only once the full response is known. A shell call * with no environment waits for that pass, so a hosted call that later * carries `shell_call_output` is not asked of the app. + * `computerTool` is the name of the computer tool of the request. A + * `computer_call` is a call to it. */ export function readUserExecutedCall( item: unknown, - options: { bareShell: boolean }, + options: { bareShell: boolean; computerTool?: OpenAIComputerToolName }, ): OpenAIUserExecutedCall | null { if (!isRecord(item) || typeof item.type !== 'string') return null const callId = typeof item.call_id === 'string' ? item.call_id : '' @@ -180,13 +195,17 @@ export function readUserExecutedCall( } if (item.type === 'computer_call') { - if (!callId || !isComputerAction(item.action)) return null + const { action, actions } = item + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!callId || (!hasAction && !hasActions)) return null return { - name: 'computer_use_preview', + name: options.computerTool ?? 'computer_use_preview', callId, ...(itemId ? { itemId } : {}), input: { - action: item.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(item.pending_safety_checks), }, } @@ -291,14 +310,18 @@ export function userToolRequestItem( } } - if (name === 'computer_use_preview') { - if (!isComputerAction(args.action)) return null + if (name === 'computer' || name === 'computer_use_preview') { + const { action, actions } = args + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!hasAction && !hasActions) return null return { type: 'computer_call', id: itemId ?? toolCall.id, call_id: toolCall.id, status: 'completed', - action: args.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(args.pending_safety_checks), } } @@ -365,8 +388,8 @@ function shellOutputList(content: unknown): Array<{ /** * Replay the app's tool result as the matching Responses output item. - * `computer` is what a `computer_use_preview` result needs: the screenshot, - * and whether the call failed. Only a computer result reads the screenshot. + * `computer` is what a computer tool result needs: the screenshot, and + * whether the call failed. Only a computer result reads the screenshot. * The output of a call that did not fail acknowledges the safety checks of * the call. */ @@ -381,7 +404,7 @@ export function userToolResultItem( const name = readUserToolName(toolCall.metadata) if (!name) return null - if (name === 'computer_use_preview') { + if (name === 'computer' || name === 'computer_use_preview') { const checks = computer?.failed ? [] : safetyChecks( diff --git a/packages/openai-base/src/tools/computer-use-tool.ts b/packages/openai-base/src/tools/computer-use-tool.ts index 26bec9b451..fb5b6a66b0 100644 --- a/packages/openai-base/src/tools/computer-use-tool.ts +++ b/packages/openai-base/src/tools/computer-use-tool.ts @@ -2,10 +2,17 @@ import { getOpenAIProviderToolMetadata, openAIProviderTool, } from './openai-provider-tool' -import type { ComputerUsePreviewTool as ComputerUseToolConfig } from 'openai/resources/responses/responses' +import type { + ComputerTool, + ComputerUsePreviewTool, +} from 'openai/resources/responses/responses' import type { Tool } from '@tanstack/ai' -export type { ComputerUseToolConfig } +/** + * The computer tool: `{ type: 'computer' }`, or the `computer_use_preview` + * tool of the `computer-use-preview` model. + */ +export type ComputerUseToolConfig = ComputerTool | ComputerUsePreviewTool /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig @@ -17,6 +24,7 @@ export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { const metadata = getOpenAIProviderToolMetadata(tool) as ComputerUseToolConfig + if (metadata.type === 'computer') return { type: 'computer' } return { type: 'computer_use_preview', display_height: metadata.display_height, @@ -26,7 +34,8 @@ export function convertComputerUseToolToAdapterFormat( } /** - * Creates a standard Tool from ComputerUseTool parameters. + * Creates a standard Tool from ComputerUseTool parameters. The tool name is + * the tool type: `computer` or `computer_use_preview`. * * Base (non-branded) factory. Providers that need branded return types should * re-wrap this in their own package. @@ -34,7 +43,7 @@ export function convertComputerUseToolToAdapterFormat( export function computerUseTool(toolData: ComputerUseToolConfig): Tool { return openAIProviderTool( { - name: 'computer_use_preview', + name: toolData.type, description: 'Control a virtual computer', metadata: { ...toolData, diff --git a/packages/openai-base/src/tools/tool-choice.ts b/packages/openai-base/src/tools/tool-choice.ts index 644e49c72e..3c8e825d16 100644 --- a/packages/openai-base/src/tools/tool-choice.ts +++ b/packages/openai-base/src/tools/tool-choice.ts @@ -37,6 +37,7 @@ interface HostedToolChoice { type: | 'file_search' | 'web_search_preview' + | 'computer' | 'computer_use_preview' | 'code_interpreter' | 'image_generation' diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index b2566dab58..f8a5d0b430 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -2,8 +2,9 @@ import { describe, expect, it, vi } from 'vitest' import { chat } from '@tanstack/ai' import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' import { computerUseTool } from '../src/tools/computer-use-tool' +import { convertToolsToProviderFormat } from '../src/tools/tool-converter' import type OpenAI from 'openai' -import type { StreamChunk } from '@tanstack/ai' +import type { StreamChunk, Tool } from '@tanstack/ai' const safetyCheck = { id: 'sc_1', @@ -51,30 +52,47 @@ const answer = { content: [{ type: 'output_text', text: 'Done.', annotations: [] }], } -class TestAdapter extends OpenAIBaseResponsesTextAdapter {} +// The OpenAI adapter sends provider tools in their native shape. +class TestAdapter extends OpenAIBaseResponsesTextAdapter { + protected override convertTools(tools: Array) { + return convertToolsToProviderFormat(tools) + } +} class FileAdapter extends TestAdapter { override readonly supportsFileSources = true } -async function runLoop(execute: (input: unknown) => Promise) { +const previewTool = computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', +}) + +const screenshot = [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, +] + +async function runLoop( + execute: (input: unknown) => Promise, + { + computer = previewTool, + call = clickCall, + }: { computer?: Tool; call?: Record } = {}, +) { const create = vi .fn() - .mockResolvedValueOnce(completed([clickCall])) + .mockResolvedValueOnce(completed([call])) .mockResolvedValueOnce(completed([answer])) const client = { responses: { create: (params: unknown) => create(params) }, } as unknown as OpenAI const adapter = new TestAdapter('computer-use-preview', 'openai', client) - const tool = { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', - }), - execute: vi.fn(execute), - } + const tool = { ...computer, execute: vi.fn(execute) } const chunks: Array = [] for await (const chunk of chat({ adapter, @@ -88,12 +106,7 @@ async function runLoop(execute: (input: unknown) => Promise) { describe('computer use loop', () => { it('runs the computer call and sends the screenshot back', async () => { - const { create, execute } = await runLoop(async () => [ - { - type: 'image', - source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, - }, - ]) + const { create, execute } = await runLoop(async () => screenshot) expect(execute).toHaveBeenCalledOnce() expect(execute.mock.calls[0]?.[0]).toEqual({ @@ -117,6 +130,7 @@ describe('computer use loop', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, @@ -124,6 +138,52 @@ describe('computer use loop', () => { ) }) + it('runs the batched actions of the computer tool', async () => { + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + const { create, execute } = await runLoop(async () => screenshot, { + computer: computerUseTool({ type: 'computer' }), + call: { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + }) + + expect(execute.mock.calls[0]?.[0]).toEqual({ + actions, + pending_safety_checks: [], + }) + const [first, second] = create.mock.calls.map(([params]) => params) + expect(first.tools).toEqual([{ type: 'computer' }]) + expect(second.input).toEqual( + expect.arrayContaining([ + { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + { + type: 'computer_call_output', + call_id: 'call_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', + }, + }, + ]), + ) + }) + it('reads no screenshot from the result of another app tool', async () => { const create = vi.fn().mockResolvedValueOnce(completed([answer])) const client = { diff --git a/packages/openai-base/tests/provider-tool-dispatch.test.ts b/packages/openai-base/tests/provider-tool-dispatch.test.ts index 1f09aaaffd..e7292ae5a5 100644 --- a/packages/openai-base/tests/provider-tool-dispatch.test.ts +++ b/packages/openai-base/tests/provider-tool-dispatch.test.ts @@ -30,6 +30,7 @@ function isRoundTrippedTool(value: unknown): value is Tool { const PROVIDER_TOOL_NAMES = [ 'apply_patch', 'code_interpreter', + 'computer', 'computer_use_preview', 'custom', 'file_search', @@ -57,6 +58,12 @@ const FACTORY_CASES = [ }), wire: { type: 'code_interpreter' }, }, + { + name: 'computer', + kind: 'openai.computer_use', + tool: computerUseTool({ type: 'computer' }), + wire: { type: 'computer' }, + }, { name: 'computer_use_preview', kind: 'openai.computer_use', diff --git a/testing/e2e/src/routes/api.openai-computer-use-wire.ts b/testing/e2e/src/routes/api.openai-computer-use-wire.ts index 8e4df804a1..8461857f5f 100644 --- a/testing/e2e/src/routes/api.openai-computer-use-wire.ts +++ b/testing/e2e/src/routes/api.openai-computer-use-wire.ts @@ -21,6 +21,26 @@ const computerCall = { pending_safety_checks: [safetyCheck], } +// The computer tool of the newer models sends a batch of actions. +const batchCall = { + type: 'computer_call', + id: 'cu_e2e_batch', + call_id: 'call_cu_e2e_batch', + status: 'completed', + actions: [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ], + pending_safety_checks: [], +} + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: SCREENSHOT, mimeType: 'image/png' }, + }, +] + const answer = { type: 'message', id: 'msg_cu_e2e', @@ -66,11 +86,16 @@ function responseStream( * custom `fetch` answers the first request with a `computer_call` and the * second with text, and records each request body. aimock cannot send a * `computer_call`, so this route serves the stream itself. + * + * `?tool=computer` uses the `computer` tool of `gpt-6.1-sol`, with a batch + * of actions. Anything else uses `computer_use_preview`. */ export const Route = createFileRoute('/api/openai-computer-use-wire')({ server: { handlers: { - POST: async () => { + POST: async ({ request }) => { + const batch = + new URL(request.url).searchParams.get('tool') === 'computer' const requestBodies: Array = [] const mockFetch: typeof fetch = async (input, init) => { const req = @@ -78,7 +103,9 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ requestBodies.push(JSON.parse(await req.text())) const stream = requestBodies.length === 1 - ? responseStream('resp_cu_e2e_1', [computerCall]) + ? responseStream('resp_cu_e2e_1', [ + batch ? batchCall : computerCall, + ]) : responseStream('resp_cu_e2e_2', [answer]) return new Response(stream, { status: 200, @@ -86,39 +113,45 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ }) } - const adapter = createOpenaiChat('computer-use-preview', DUMMY_KEY, { - fetch: mockFetch, - }) - const toolInputs: Array = [] + const execute = async (input: unknown) => { + toolInputs.push(input) + return screenshot + } + const messages = [ + { role: 'user' as const, content: 'Click the button.' }, + ] try { - for await (const _ of chat({ - ...createChatOptions({ adapter }), - messages: [{ role: 'user', content: 'Click the button.' }], - tools: [ - { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', + const stream = batch + ? chat({ + ...createChatOptions({ + adapter: createOpenaiChat('gpt-6.1-sol', DUMMY_KEY, { + fetch: mockFetch, + }), + }), + messages, + tools: [{ ...computerUseTool({ type: 'computer' }), execute }], + }) + : chat({ + ...createChatOptions({ + adapter: createOpenaiChat('computer-use-preview', DUMMY_KEY, { + fetch: mockFetch, + }), }), - execute: async (input: unknown) => { - toolInputs.push(input) - return [ - { - type: 'image' as const, - source: { - type: 'data' as const, - value: SCREENSHOT, - mimeType: 'image/png', - }, - }, - ] - }, - }, - ], - })) { + messages, + tools: [ + { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute, + }, + ], + }) + for await (const _ of stream) { // Drain the stream. } } catch (error) { diff --git a/testing/e2e/tests/openai-computer-use-wire.spec.ts b/testing/e2e/tests/openai-computer-use-wire.spec.ts index d0db59ee87..b700e7023f 100644 --- a/testing/e2e/tests/openai-computer-use-wire.spec.ts +++ b/testing/e2e/tests/openai-computer-use-wire.spec.ts @@ -50,10 +50,54 @@ test.describe('openai — computer use wire format', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, ]), ) }) + + test('runs the batched actions of the computer tool', async ({ request }) => { + const res = await request.post( + '/api/openai-computer-use-wire?tool=computer', + ) + expect(res.ok()).toBe(true) + const { ok, error, toolInputs, requestBodies } = (await res.json()) as { + ok: boolean + error?: string + toolInputs: Array + requestBodies: Array<{ tools?: Array; input: Array }> + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + expect(toolInputs).toEqual([{ actions, pending_safety_checks: [] }]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[0]!.tools).toEqual([{ type: 'computer' }]) + expect(requestBodies[1]!.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_e2e_batch', + actions, + }), + { + type: 'computer_call_output', + call_id: 'call_cu_e2e_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', + }, + }, + ]), + ) + }) })