From fa2729d166667fd398dca364222da06cc2bbe896 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 18:41:28 +0200 Subject: [PATCH 1/3] feat(openai-base, ai-openai): add the computer tool computerUseTool({ type: 'computer' }) sends the computer tool of the newer OpenAI models. Its tool name is computer. A computer_call is a call to the computer tool of the request, with the batched actions and the pending safety checks as input, and the replay sends the actions back as they came. The computer_use_preview tool works as before. --- .changeset/openai-computer-tool.md | 6 ++ docs/adapters/openai.md | 48 ++++++---- docs/config.json | 2 +- .../ai-openai/src/tools/computer-use-tool.ts | 10 +- .../tests/tools-per-model-type-safety.test.ts | 6 ++ .../src/adapters/responses-text.ts | 10 +- .../src/adapters/responses-user-tools.ts | 49 +++++++--- .../src/tools/computer-use-tool.ts | 17 +++- packages/openai-base/src/tools/tool-choice.ts | 1 + .../tests/computer-use-loop.test.ts | 96 +++++++++++++++---- .../tests/provider-tool-dispatch.test.ts | 7 ++ 11 files changed, 191 insertions(+), 61 deletions(-) create mode 100644 .changeset/openai-computer-tool.md diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md new file mode 100644 index 0000000000..a2c1d63e9c --- /dev/null +++ b/.changeset/openai-computer-tool.md @@ -0,0 +1,6 @@ +--- +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +--- + +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. diff --git a/docs/adapters/openai.md b/docs/adapters/openai.md index 6646c49201..1a89ed4b82 100644 --- a/docs/adapters/openai.md +++ b/docs/adapters/openai.md @@ -993,10 +993,9 @@ const stream = chat({ ### `computerUseTool` Lets the model observe a virtual desktop via screenshots and interact with -it using keyboard and mouse events. Provide the display dimensions and the -execution environment type. +it using keyboard and mouse events. -The model asks for one action at a time, for example a click. Your app runs the action, then returns a screenshot of the screen. Add an `execute` to the tool that does this: +The model asks for actions, for example a click and then some typing. Your app runs the actions, then returns a screenshot of the screen. Add an `execute` to the tool that does this: ```typescript import { chat } from "@tanstack/ai"; @@ -1009,31 +1008,31 @@ interface SafetyCheck { message?: string; } -// Your code: run one action on the desktop, then return a PNG of the screen as base64. -declare function runAction(action: unknown): Promise; +// Your code: run one action on the desktop. +declare function runAction(action: unknown): Promise; +// Your code: return a PNG of the screen as base64. +declare function takeScreenshot(): Promise; // Your code: show the checks to a person and return their answer. declare function askPerson(checks: Array): Promise; const stream = chat({ - adapter: openaiText("computer-use-preview"), - messages: [{ role: "user", content: "Open Chrome and navigate to example.com" }], + adapter: openaiText("gpt-6.1-sol"), + messages: [{ role: "user", content: "Open example.com and sign up for the newsletter" }], tools: [ { - ...computerUseTool({ - type: "computer_use_preview", - display_width: 1024, - display_height: 768, - environment: "browser", - }), + ...computerUseTool({ type: "computer" }), execute: async (input: { - action: unknown; + actions: Array; pending_safety_checks: Array; }) => { const checks = input.pending_safety_checks; if (checks.length > 0 && !(await askPerson(checks))) { throw new Error("A person did not allow this action."); } - const screenshot = await runAction(input.action); + for (const action of input.actions) { + await runAction(action); + } + const screenshot = await takeScreenshot(); return [ { type: "image" as const, @@ -1048,12 +1047,25 @@ const stream = chat({ The input of each call has two fields: -- `action`: the action to run, as OpenAI sends it. For example `{ type: "click", button: "left", x: 120, y: 340 }`. -- `pending_safety_checks`: the safety checks of the action. If the list is not empty, ask a person before you run the action. +- `actions`: the actions to run, in order, as OpenAI sends them. For example `[{ type: "click", button: "left", x: 120, y: 340 }, { type: "type", text: "ada@example.com" }]`. +- `pending_safety_checks`: the safety checks of the actions. If the list is not empty, ask a person before you run the actions. The adapter sends the first image of the result back as the screenshot. When the call does not fail, the adapter also tells OpenAI that the safety checks are accepted. When the call fails, they stay unaccepted. -**Supported models:** `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). +The `computer-use-preview` model takes the older tool type. Give it the display size and the environment. Each call then has one `action` field in place of `actions`: + +```typescript +import { computerUseTool } from "@tanstack/ai-openai/tools"; + +const previewTool = computerUseTool({ + type: "computer_use_preview", + display_width: 1024, + display_height: 768, + environment: "browser", +}); +``` + +**Supported models:** `type: "computer"` works with the models that list computer use, for example `gpt-6.1-sol`. `type: "computer_use_preview"` works with `computer-use-preview`. See [Provider Tools](../tools/provider-tools.md#which-models-support-which-tools). ### `localShellTool` diff --git a/docs/config.json b/docs/config.json index 5fed936ceb..cddd848128 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1209,7 +1209,7 @@ "label": "OpenAI", "to": "adapters/openai", "addedAt": "2026-04-15", - "updatedAt": "2026-10-12" + "updatedAt": "2026-10-19" }, { "label": "Anthropic", diff --git a/packages/ai-openai/src/tools/computer-use-tool.ts b/packages/ai-openai/src/tools/computer-use-tool.ts index 3f65dddb7c..0ad3ecc555 100644 --- a/packages/ai-openai/src/tools/computer-use-tool.ts +++ b/packages/ai-openai/src/tools/computer-use-tool.ts @@ -9,11 +9,11 @@ export { } from '@tanstack/openai-base' // The brand discriminator (`computer_use`) intentionally differs from the -// runtime tool name (`computer_use_preview`). The brand matches the model-meta -// tool-capability union (`tools: ['computer_use', ...]`) used to gate which -// models can construct this tool at compile time, while the runtime name -// matches the OpenAI SDK's literal `'computer_use_preview'`. The base factory -// records the canonical `computer_use` kind separately in adapter metadata. +// runtime tool name (`computer` or `computer_use_preview`). The brand matches +// the model-meta tool-capability union (`tools: ['computer_use', ...]`) used to +// gate which models can construct this tool at compile time, while the runtime +// name matches the tool type in the OpenAI SDK. The base factory records the +// canonical `computer_use` kind separately in adapter metadata. export type OpenAIComputerUseTool = ProviderTool<'openai', 'computer_use'> /** diff --git a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts index ad9ae3205c..cc89d94901 100644 --- a/packages/ai-openai/tests/tools-per-model-type-safety.test.ts +++ b/packages/ai-openai/tests/tools-per-model-type-safety.test.ts @@ -107,6 +107,12 @@ describe('OpenAI per-model tool gating', () => { // OpenAI's GPT-6 model pages list web_search, file_search, image_generation, // code_interpreter, hosted_shell, apply_patch, computer_use and mcp for the // Responses API, and neither web_search_preview nor local_shell. + it('gpt-6.1-sol accepts the computer tool', () => { + typedTools(openaiText('gpt-6.1-sol'), [ + computerUseTool({ type: 'computer' }), + ]) + }) + it('gpt-6-astra accepts the Responses tools its model page lists but rejects the rest', () => { const adapter = openaiText('gpt-6-astra') typedTools(adapter, [ diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 1628d75202..4e8d55794d 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1391,6 +1391,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< hasEmittedRunStarted: boolean }, ): AsyncIterable { + // A `computer_call` is a call to the computer tool of this request. + const computerTool = options.tools?.some( + (tool) => + tool.name === 'computer' && + getOpenAIProviderToolKind(tool) === 'computer_use', + ) + ? 'computer' + : 'computer_use_preview' const normalizeToolInput = createToolInputNormalizer( options.tools, (schema, required) => @@ -1619,7 +1627,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< outputIndex: number, bareShell: boolean, ): Array => { - const call = readUserExecutedCall(item, { bareShell }) + const call = readUserExecutedCall(item, { bareShell, computerTool }) if (!call || toolCallMetadata.get(call.callId)?.ended) return [] const tracked = toolCallMetadata.get(call.callId) ?? { callId: call.callId, diff --git a/packages/openai-base/src/adapters/responses-user-tools.ts b/packages/openai-base/src/adapters/responses-user-tools.ts index 01a02b2a8e..903f907a05 100644 --- a/packages/openai-base/src/adapters/responses-user-tools.ts +++ b/packages/openai-base/src/adapters/responses-user-tools.ts @@ -1,4 +1,5 @@ import type { + ComputerActionList, ResponseComputerToolCall, ResponseComputerToolCallOutputScreenshot, ResponseInputItem, @@ -7,14 +8,17 @@ import type { /** * OpenAI Responses tools the app must run. * A container `shell` runs on the provider. `apply_patch`, `local_shell`, - * `computer_use_preview`, and `shell` with `environment.type: "local"` (or - * no environment) do not. + * `computer`, `computer_use_preview`, and `shell` with + * `environment.type: "local"` (or no environment) do not. */ export type OpenAIUserToolName = | 'shell' | 'apply_patch' | 'local_shell' - | 'computer_use_preview' + | OpenAIComputerToolName + +/** The names of the computer tool: its two tool types. */ +export type OpenAIComputerToolName = 'computer' | 'computer_use_preview' export interface OpenAIUserExecutedCall { name: OpenAIUserToolName @@ -84,6 +88,7 @@ export function readUserToolName(metadata: unknown): OpenAIUserToolName | null { name === 'shell' || name === 'apply_patch' || name === 'local_shell' || + name === 'computer' || name === 'computer_use_preview' ) { return name @@ -101,6 +106,14 @@ function isComputerAction( return isRecord(value) && typeof value.type === 'string' } +/** The batched actions of a computer call. */ +function isComputerActionList(value: unknown): value is ComputerActionList { + return ( + Array.isArray(value) && + value.every((action) => isRecord(action) && typeof action.type === 'string') + ) +} + /** The safety checks of a computer call, with only their known fields. */ function safetyChecks( value: unknown, @@ -159,10 +172,12 @@ function applyPatchOperation( * `bareShell` is true only once the full response is known. A shell call * with no environment waits for that pass, so a hosted call that later * carries `shell_call_output` is not asked of the app. + * `computerTool` is the name of the computer tool of the request. A + * `computer_call` is a call to it. */ export function readUserExecutedCall( item: unknown, - options: { bareShell: boolean }, + options: { bareShell: boolean; computerTool?: OpenAIComputerToolName }, ): OpenAIUserExecutedCall | null { if (!isRecord(item) || typeof item.type !== 'string') return null const callId = typeof item.call_id === 'string' ? item.call_id : '' @@ -180,13 +195,17 @@ export function readUserExecutedCall( } if (item.type === 'computer_call') { - if (!callId || !isComputerAction(item.action)) return null + const { action, actions } = item + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!callId || (!hasAction && !hasActions)) return null return { - name: 'computer_use_preview', + name: options.computerTool ?? 'computer_use_preview', callId, ...(itemId ? { itemId } : {}), input: { - action: item.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(item.pending_safety_checks), }, } @@ -291,14 +310,18 @@ export function userToolRequestItem( } } - if (name === 'computer_use_preview') { - if (!isComputerAction(args.action)) return null + if (name === 'computer' || name === 'computer_use_preview') { + const { action, actions } = args + const hasAction = isComputerAction(action) + const hasActions = isComputerActionList(actions) + if (!hasAction && !hasActions) return null return { type: 'computer_call', id: itemId ?? toolCall.id, call_id: toolCall.id, status: 'completed', - action: args.action, + ...(hasAction ? { action } : {}), + ...(hasActions ? { actions } : {}), pending_safety_checks: safetyChecks(args.pending_safety_checks), } } @@ -365,8 +388,8 @@ function shellOutputList(content: unknown): Array<{ /** * Replay the app's tool result as the matching Responses output item. - * `computer` is what a `computer_use_preview` result needs: the screenshot, - * and whether the call failed. The output of a call that did not fail + * `computer` is what a computer tool result needs: the screenshot, and + * whether the call failed. The output of a call that did not fail * acknowledges the safety checks of the call. */ export function userToolResultItem( @@ -380,7 +403,7 @@ export function userToolResultItem( const name = readUserToolName(toolCall.metadata) if (!name) return null - if (name === 'computer_use_preview') { + if (name === 'computer' || name === 'computer_use_preview') { const checks = computer?.failed ? [] : safetyChecks( diff --git a/packages/openai-base/src/tools/computer-use-tool.ts b/packages/openai-base/src/tools/computer-use-tool.ts index 26bec9b451..fb5b6a66b0 100644 --- a/packages/openai-base/src/tools/computer-use-tool.ts +++ b/packages/openai-base/src/tools/computer-use-tool.ts @@ -2,10 +2,17 @@ import { getOpenAIProviderToolMetadata, openAIProviderTool, } from './openai-provider-tool' -import type { ComputerUsePreviewTool as ComputerUseToolConfig } from 'openai/resources/responses/responses' +import type { + ComputerTool, + ComputerUsePreviewTool, +} from 'openai/resources/responses/responses' import type { Tool } from '@tanstack/ai' -export type { ComputerUseToolConfig } +/** + * The computer tool: `{ type: 'computer' }`, or the `computer_use_preview` + * tool of the `computer-use-preview` model. + */ +export type ComputerUseToolConfig = ComputerTool | ComputerUsePreviewTool /** @deprecated Renamed to `ComputerUseToolConfig`. Will be removed in a future release. */ export type ComputerUseTool = ComputerUseToolConfig @@ -17,6 +24,7 @@ export function convertComputerUseToolToAdapterFormat( tool: Tool, ): ComputerUseToolConfig { const metadata = getOpenAIProviderToolMetadata(tool) as ComputerUseToolConfig + if (metadata.type === 'computer') return { type: 'computer' } return { type: 'computer_use_preview', display_height: metadata.display_height, @@ -26,7 +34,8 @@ export function convertComputerUseToolToAdapterFormat( } /** - * Creates a standard Tool from ComputerUseTool parameters. + * Creates a standard Tool from ComputerUseTool parameters. The tool name is + * the tool type: `computer` or `computer_use_preview`. * * Base (non-branded) factory. Providers that need branded return types should * re-wrap this in their own package. @@ -34,7 +43,7 @@ export function convertComputerUseToolToAdapterFormat( export function computerUseTool(toolData: ComputerUseToolConfig): Tool { return openAIProviderTool( { - name: 'computer_use_preview', + name: toolData.type, description: 'Control a virtual computer', metadata: { ...toolData, diff --git a/packages/openai-base/src/tools/tool-choice.ts b/packages/openai-base/src/tools/tool-choice.ts index 644e49c72e..3c8e825d16 100644 --- a/packages/openai-base/src/tools/tool-choice.ts +++ b/packages/openai-base/src/tools/tool-choice.ts @@ -37,6 +37,7 @@ interface HostedToolChoice { type: | 'file_search' | 'web_search_preview' + | 'computer' | 'computer_use_preview' | 'code_interpreter' | 'image_generation' diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index f15e0674c6..e8950ea1ac 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -2,8 +2,9 @@ import { describe, expect, it, vi } from 'vitest' import { chat } from '@tanstack/ai' import { OpenAIBaseResponsesTextAdapter } from '../src/adapters/responses-text' import { computerUseTool } from '../src/tools/computer-use-tool' +import { convertToolsToProviderFormat } from '../src/tools/tool-converter' import type OpenAI from 'openai' -import type { StreamChunk } from '@tanstack/ai' +import type { StreamChunk, Tool } from '@tanstack/ai' const safetyCheck = { id: 'sc_1', @@ -51,26 +52,43 @@ const answer = { content: [{ type: 'output_text', text: 'Done.', annotations: [] }], } -class TestAdapter extends OpenAIBaseResponsesTextAdapter {} +// The OpenAI adapter sends provider tools in their native shape. +class TestAdapter extends OpenAIBaseResponsesTextAdapter { + protected override convertTools(tools: Array) { + return convertToolsToProviderFormat(tools) + } +} + +const previewTool = computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', +}) -async function runLoop(execute: (input: unknown) => Promise) { +const screenshot = [ + { + type: 'image', + source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, + }, +] + +async function runLoop( + execute: (input: unknown) => Promise, + { + computer = previewTool, + call = clickCall, + }: { computer?: Tool; call?: Record } = {}, +) { const create = vi .fn() - .mockResolvedValueOnce(completed([clickCall])) + .mockResolvedValueOnce(completed([call])) .mockResolvedValueOnce(completed([answer])) const client = { responses: { create: (params: unknown) => create(params) }, } as unknown as OpenAI const adapter = new TestAdapter('computer-use-preview', 'openai', client) - const tool = { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', - }), - execute: vi.fn(execute), - } + const tool = { ...computer, execute: vi.fn(execute) } const chunks: Array = [] for await (const chunk of chat({ adapter, @@ -84,12 +102,7 @@ async function runLoop(execute: (input: unknown) => Promise) { describe('computer use loop', () => { it('runs the computer call and sends the screenshot back', async () => { - const { create, execute } = await runLoop(async () => [ - { - type: 'image', - source: { type: 'data', value: 'iVBORw0KGgo=', mimeType: 'image/png' }, - }, - ]) + const { create, execute } = await runLoop(async () => screenshot) expect(execute).toHaveBeenCalledOnce() expect(execute.mock.calls[0]?.[0]).toEqual({ @@ -120,6 +133,51 @@ describe('computer use loop', () => { ) }) + it('runs the batched actions of the computer tool', async () => { + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + const { create, execute } = await runLoop(async () => screenshot, { + computer: computerUseTool({ type: 'computer' }), + call: { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + }) + + expect(execute.mock.calls[0]?.[0]).toEqual({ + actions, + pending_safety_checks: [], + }) + const [first, second] = create.mock.calls.map(([params]) => params) + expect(first.tools).toEqual([{ type: 'computer' }]) + expect(second.input).toEqual( + expect.arrayContaining([ + { + type: 'computer_call', + id: 'cu_2', + call_id: 'call_batch', + status: 'completed', + actions, + pending_safety_checks: [], + }, + { + type: 'computer_call_output', + call_id: 'call_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + }, + }, + ]), + ) + }) + it('does not acknowledge the safety checks of a failed call', async () => { const { create } = await runLoop(async () => { throw new Error('A person did not allow the click.') diff --git a/packages/openai-base/tests/provider-tool-dispatch.test.ts b/packages/openai-base/tests/provider-tool-dispatch.test.ts index 1f09aaaffd..e7292ae5a5 100644 --- a/packages/openai-base/tests/provider-tool-dispatch.test.ts +++ b/packages/openai-base/tests/provider-tool-dispatch.test.ts @@ -30,6 +30,7 @@ function isRoundTrippedTool(value: unknown): value is Tool { const PROVIDER_TOOL_NAMES = [ 'apply_patch', 'code_interpreter', + 'computer', 'computer_use_preview', 'custom', 'file_search', @@ -57,6 +58,12 @@ const FACTORY_CASES = [ }), wire: { type: 'code_interpreter' }, }, + { + name: 'computer', + kind: 'openai.computer_use', + tool: computerUseTool({ type: 'computer' }), + wire: { type: 'computer' }, + }, { name: 'computer_use_preview', kind: 'openai.computer_use', From becc71f594a783a66c589666b5586edd5ee27164 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 19 Oct 2026 19:05:44 +0200 Subject: [PATCH 2/3] feat(openai-base): send computer screenshots with the original detail OpenAI's computer use guide sends each screenshot with detail: 'original', so the model sees the screen at full size. The SDK type has no detail field yet, so the adapter adds it to its own screenshot type. --- .changeset/openai-computer-tool.md | 2 +- packages/openai-base/src/adapters/responses-text.ts | 9 +++++++-- packages/openai-base/tests/computer-use-loop.test.ts | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.changeset/openai-computer-tool.md b/.changeset/openai-computer-tool.md index a2c1d63e9c..40fbc8cab4 100644 --- a/.changeset/openai-computer-tool.md +++ b/.changeset/openai-computer-tool.md @@ -3,4 +3,4 @@ '@tanstack/ai-openai': minor --- -`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. +`computerUseTool({ type: 'computer' })` adds the computer tool of the newer OpenAI models, for example `gpt-6.1-sol`. The tool name is `computer`. Each call has the batched `actions` of the model and the `pending_safety_checks`, and the replay sends the actions back as they came. `computerUseTool({ type: 'computer_use_preview', ... })` for the `computer-use-preview` model works as before. The Responses tool choice type also accepts `computer`. Computer screenshots now go out with `detail: 'original'`, as in OpenAI's computer use guide. diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index 4e8d55794d..49f14ed0c1 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -3164,10 +3164,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< return result } - /** The first image of a tool result, as a computer screenshot. */ + /** + * The first image of a tool result, as a computer screenshot. It goes out + * with `detail: 'original'`, as in OpenAI's computer use guide, so the + * model sees the screen at full size. The SDK type has no `detail` yet. + */ private screenshotOf( content: ModelMessage['content'], - ): ResponseComputerToolCallOutputScreenshot { + ): ResponseComputerToolCallOutputScreenshot & { detail?: 'original' } { const image = Array.isArray(content) ? content.find((part) => part.type === 'image') : undefined @@ -3177,6 +3181,7 @@ export abstract class OpenAIBaseResponsesTextAdapter< type: 'computer_screenshot', ...(input.image_url ? { image_url: input.image_url } : {}), ...(input.file_id ? { file_id: input.file_id } : {}), + detail: 'original', } } diff --git a/packages/openai-base/tests/computer-use-loop.test.ts b/packages/openai-base/tests/computer-use-loop.test.ts index e8950ea1ac..0df6b47bfb 100644 --- a/packages/openai-base/tests/computer-use-loop.test.ts +++ b/packages/openai-base/tests/computer-use-loop.test.ts @@ -126,6 +126,7 @@ describe('computer use loop', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, @@ -172,6 +173,7 @@ describe('computer use loop', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, }, ]), From 7d482c80e8c4e664446b8b5df8a1fea2969e2f94 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Tue, 20 Oct 2026 08:24:03 +0200 Subject: [PATCH 3/3] test(e2e): check the computer tool batch and the screenshot detail on the wire The route takes ?tool=computer for the computer tool of gpt-6.1-sol. The spec checks the batched actions, the computer tool entry, and detail: 'original' on each screenshot. --- .../routes/api.openai-computer-use-wire.ts | 97 +++++++++++++------ .../tests/openai-computer-use-wire.spec.ts | 44 +++++++++ 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/testing/e2e/src/routes/api.openai-computer-use-wire.ts b/testing/e2e/src/routes/api.openai-computer-use-wire.ts index 8e4df804a1..8461857f5f 100644 --- a/testing/e2e/src/routes/api.openai-computer-use-wire.ts +++ b/testing/e2e/src/routes/api.openai-computer-use-wire.ts @@ -21,6 +21,26 @@ const computerCall = { pending_safety_checks: [safetyCheck], } +// The computer tool of the newer models sends a batch of actions. +const batchCall = { + type: 'computer_call', + id: 'cu_e2e_batch', + call_id: 'call_cu_e2e_batch', + status: 'completed', + actions: [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ], + pending_safety_checks: [], +} + +const screenshot = [ + { + type: 'image' as const, + source: { type: 'data' as const, value: SCREENSHOT, mimeType: 'image/png' }, + }, +] + const answer = { type: 'message', id: 'msg_cu_e2e', @@ -66,11 +86,16 @@ function responseStream( * custom `fetch` answers the first request with a `computer_call` and the * second with text, and records each request body. aimock cannot send a * `computer_call`, so this route serves the stream itself. + * + * `?tool=computer` uses the `computer` tool of `gpt-6.1-sol`, with a batch + * of actions. Anything else uses `computer_use_preview`. */ export const Route = createFileRoute('/api/openai-computer-use-wire')({ server: { handlers: { - POST: async () => { + POST: async ({ request }) => { + const batch = + new URL(request.url).searchParams.get('tool') === 'computer' const requestBodies: Array = [] const mockFetch: typeof fetch = async (input, init) => { const req = @@ -78,7 +103,9 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ requestBodies.push(JSON.parse(await req.text())) const stream = requestBodies.length === 1 - ? responseStream('resp_cu_e2e_1', [computerCall]) + ? responseStream('resp_cu_e2e_1', [ + batch ? batchCall : computerCall, + ]) : responseStream('resp_cu_e2e_2', [answer]) return new Response(stream, { status: 200, @@ -86,39 +113,45 @@ export const Route = createFileRoute('/api/openai-computer-use-wire')({ }) } - const adapter = createOpenaiChat('computer-use-preview', DUMMY_KEY, { - fetch: mockFetch, - }) - const toolInputs: Array = [] + const execute = async (input: unknown) => { + toolInputs.push(input) + return screenshot + } + const messages = [ + { role: 'user' as const, content: 'Click the button.' }, + ] try { - for await (const _ of chat({ - ...createChatOptions({ adapter }), - messages: [{ role: 'user', content: 'Click the button.' }], - tools: [ - { - ...computerUseTool({ - type: 'computer_use_preview', - display_width: 1024, - display_height: 768, - environment: 'browser', + const stream = batch + ? chat({ + ...createChatOptions({ + adapter: createOpenaiChat('gpt-6.1-sol', DUMMY_KEY, { + fetch: mockFetch, + }), + }), + messages, + tools: [{ ...computerUseTool({ type: 'computer' }), execute }], + }) + : chat({ + ...createChatOptions({ + adapter: createOpenaiChat('computer-use-preview', DUMMY_KEY, { + fetch: mockFetch, + }), }), - execute: async (input: unknown) => { - toolInputs.push(input) - return [ - { - type: 'image' as const, - source: { - type: 'data' as const, - value: SCREENSHOT, - mimeType: 'image/png', - }, - }, - ] - }, - }, - ], - })) { + messages, + tools: [ + { + ...computerUseTool({ + type: 'computer_use_preview', + display_width: 1024, + display_height: 768, + environment: 'browser', + }), + execute, + }, + ], + }) + for await (const _ of stream) { // Drain the stream. } } catch (error) { diff --git a/testing/e2e/tests/openai-computer-use-wire.spec.ts b/testing/e2e/tests/openai-computer-use-wire.spec.ts index d0db59ee87..b700e7023f 100644 --- a/testing/e2e/tests/openai-computer-use-wire.spec.ts +++ b/testing/e2e/tests/openai-computer-use-wire.spec.ts @@ -50,10 +50,54 @@ test.describe('openai — computer use wire format', () => { output: { type: 'computer_screenshot', image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', }, acknowledged_safety_checks: [safetyCheck], }, ]), ) }) + + test('runs the batched actions of the computer tool', async ({ request }) => { + const res = await request.post( + '/api/openai-computer-use-wire?tool=computer', + ) + expect(res.ok()).toBe(true) + const { ok, error, toolInputs, requestBodies } = (await res.json()) as { + ok: boolean + error?: string + toolInputs: Array + requestBodies: Array<{ tools?: Array; input: Array }> + } + + if (!ok) { + throw new Error(`Route failed: ${error}`) + } + + const actions = [ + { type: 'click', button: 'left', x: 10, y: 20 }, + { type: 'type', text: 'hello' }, + ] + expect(toolInputs).toEqual([{ actions, pending_safety_checks: [] }]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[0]!.tools).toEqual([{ type: 'computer' }]) + expect(requestBodies[1]!.input).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'computer_call', + call_id: 'call_cu_e2e_batch', + actions, + }), + { + type: 'computer_call_output', + call_id: 'call_cu_e2e_batch', + output: { + type: 'computer_screenshot', + image_url: 'data:image/png;base64,iVBORw0KGgo=', + detail: 'original', + }, + }, + ]), + ) + }) })